diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index ca26cd7..201fc97 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -2,7 +2,9 @@ import type { AppConfig, AuthService, Feature, JobHandler, JobQueue, ServerConte import { createApp, createJobQueue, loadConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import { connect, migrate } from "@stc/db"; +import { importFeature } from "@stc/feature-import"; import { learnerFeature } from "@stc/feature-learner"; +import { resolve } from "node:path"; import indexHtml from "../../web/index.html"; // Used until a feature supplies an `auth` service (the learner package does). @@ -49,8 +51,12 @@ export async function startServer( if (import.meta.main) { const config = loadConfig(); + // Contract request (import.md): core config should include /courses itself. + const generated = resolve(config.dataDir, "courses"); + if (!config.courseDirs.some((c) => resolve(c) === generated)) config.courseDirs.push(generated); const { server } = await startServer(config, [ learnerFeature, + (ctx) => importFeature(ctx), ]); console.log(`STCourses listening on ${server.url}`); } diff --git a/apps/web/src/features.ts b/apps/web/src/features.ts index cdef4be..3b82a55 100644 --- a/apps/web/src/features.ts +++ b/apps/web/src/features.ts @@ -1,5 +1,6 @@ import type { WebFeature } from "@stc/contracts"; import { learnerWeb } from "@stc/feature-learner/web"; +import { importWeb } from "../../../packages/feature-import/src/web"; // Phase 1 packages append one import line each and push into this list. -export const webFeatures: WebFeature[] = [learnerWeb]; +export const webFeatures: WebFeature[] = [learnerWeb, importWeb]; diff --git a/docs/contract-requests/import.md b/docs/contract-requests/import.md new file mode 100644 index 0000000..4831f36 --- /dev/null +++ b/docs/contract-requests/import.md @@ -0,0 +1,12 @@ +# Contract requests from WP-B (Importer) + +1. **`loadConfig` should add `/courses` to `courseDirs`.** The importer writes courses there. Worked around in `apps/server/src/main.ts` (the main guard pushes it) and in `packages/feature-import/dev/serve.ts`. `/api/import/info` reports `coursesDirServed` and the UI warns when it is false. +2. **`startServer` accepts feature factories** (`Feature | ((ctx) => Feature)`). A `Feature` needs `ServerContext`, but `ServerContext.jobs` is built from the features' job handlers. `importFeature(ctx)` only reads `ctx` while a job or request runs, so `main.ts` now builds `ctx` first, resolves the factories, then assigns `ctx.jobs`. The edit is about ten lines in `apps/server/src/main.ts`; Phase 2 should keep or replace it. +3. **Web entry for features.** `apps/web` must not pull a package's server code into the browser bundle. `@stc/feature-import` exposes a browser-safe entry as `./web` (package.json `exports`), but the root tsconfig `paths` (`@stc/*` to `packages/*/src`) cannot resolve `@stc/feature-import/web`, so `apps/web/src/features.ts` imports it by relative path. Suggest a tsconfig `paths` entry per feature, or a `src/web.ts` convention. +4. **Derived cache layout (no contract defines one).** Written by the importer, to be read by the learner and AI packages: + - `/cache/decks///p-001.jpg ...` (1280px wide JPEGs) and `text.json` (`[{ page, text }]`); + - `/cache/covers//frame-N.jpg` and `deck-.jpg` (cover candidates); + - `/cache/transcripts//.json` (`{ lessonId, text, segments: [{ start, end, text }] }`). + Suggest a `cacheDir`/`deckPageUrl` helper in core, and a media route for `cache/decks` so the deck viewer can load pages. +5. **Video lessons have no description field.** Packt clips come with a one-line description. It is dropped, because `transcript` is the only Markdown a video lesson can carry and the transcription job writes it. +6. **`STC_IMPORT_ROOTS`** (":"-separated, default `/import-inbox`, created when `importFeature` is built) is read inside the import package. Core config could own it later. diff --git a/docs/superpowers/plans/2026-10-30-import.md b/docs/superpowers/plans/2026-10-30-import.md new file mode 100644 index 0000000..ba21859 --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-import.md @@ -0,0 +1,56 @@ +# WP-B Importer Implementation Plan + +> Executed inline by the WP-B agent, task by task, TDD. Spec: `docs/superpowers/specs/2026-10-30-stcourses-design.md` sections 5, 8, 13. + +**Goal:** `packages/feature-import` turns a Packt video zip, a companion code repo, a slide PDF or a folder of videos into an ordinary course folder under `/courses//`, as background jobs with visible progress, plus an owner-only admin UI at `/admin/import`. + +**Architecture:** +- `importFeature(ctx)` returns a `Feature` with job handlers `import.packt-video`, `import.code-repo`, `import.pdf-deck`, `import.video-folder`, `import.transcribe` and owner-only `/api/import/*` routes. `ctx` is read lazily (inside handlers), because the job queue is built from the handlers. +- Courses are built in `/tmp/` and renamed into `/courses/` in one step. Attach jobs (code, deck) only touch courses under `/courses` and write the manifest last, through temp file + rename. +- Media: **copied** (`cp --reflink=auto`, so instant on CoW filesystems), or hard-linked with `options.media = "link"` (falls back to copy across filesystems). Symlinks out of the course are not an option: the media route 404s on paths that escape the course root. The originals are only ever read. +- PDFs: PyMuPDF through `py/pdf_tool.py` run as `python -I` in a subprocess (`STC_PYTHON`, default `python3`). Chosen over pdfjs/mupdf-wasm because it rasterises and extracts text natively, is already installed, is AGPL-compatible and costs no idle memory in the server. If it is missing, the deck is still attached and the job reports a warning. +- Derived data goes to `/cache/`: `decks///p-001.jpg…` + `text.json`, `covers//*.jpg`, `transcripts//.json`. + +## Global Constraints + +Inherited unchanged from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun conventions, AGPL, design tokens, no licensed content in the repo, commit trailer). + +## Review Focus + +1. Original files are never modified; attach jobs refuse courses outside `/courses` (overlays, authored repos). +2. Paths with spaces/non-ASCII (`S08 - Containerizing Applications`) survive unzip, copy and manifest round trip. +3. Packt parsing is structural (chapter blocks plus the section summary lists), not regexes on "Video N.M:" prefixes. +4. Failure leaves no half-built course visible; a failed PDF/ffmpeg step degrades to a warning where the course is still valid. +5. Routes: 401 without a user, 403 without the owner role. + +## Tasks + +### Task 1: Utilities (slug, HTML entities, natural sort, exec, manifest IO, sources) +- Files: `src/util/*.ts`, `test/util.test.ts` +- Interfaces: `slugify`, `uniqueCourseId`, `decodeEntities`, `naturalSort`, `run`, `readRawManifest`, `writeManifest` (validates with `ManifestSchema`), `openSource(path)` (zip via `unzip`, or directory). +- [ ] Failing tests for each helper; [ ] implement; [ ] commit. + +### Task 2: Fixtures and media tooling +- Files: `test/fixtures.ts` (generates fake Packt zip, repo zip, PDF, video folder in a temp dir), `src/media.ts` (ffprobe duration, ffmpeg frame, copy/link), `src/pdf.ts`, `py/pdf_tool.py`. +- [ ] Failing tests (frame extraction, duration, PDF pages+text); [ ] implement; [ ] commit. + +### Task 3: Packt parser and `import.packt-video` +- Files: `src/packt.ts`, `src/jobs/packt-video.ts`, `test/packt.test.ts` +- [ ] Parser test on the synthetic index (summary-only titles, missing chapter); [ ] job test (2 sections, 3 clips, restricted manifest, source filled, course loads `ok`, cover candidates); [ ] implement; [ ] commit. + +### Task 4: `import.video-folder`, `import.pdf-deck` +- [ ] Tests: natural order, deck cache pages/text, attach to lesson and module, refuses unmanaged course; [ ] implement; [ ] commit. + +### Task 5: `import.code-repo` +- [ ] Tests: folder `S02 - …` attaches to module 2 by number, by name when no number, PDFs become deck/attachment, git URL validation; [ ] implement; [ ] commit. + +### Task 6: `import.transcribe` +- [ ] Tests with a fake Whisper server; skipped when `STC_TRANSCRIBE_URL` unset; [ ] implement; [ ] commit. + +### Task 7: Routes and `importFeature` +- [ ] Tests: 401/403/owner, enqueue validation, queue integration (SQLite and Postgres), covers; [ ] implement; wire `main.ts`; [ ] commit. + +### Task 8: Web UI `importWeb` +- [ ] Page with form, job list with progress, cover picker; register in `apps/web/src/features.ts`; screenshot check; [ ] commit. + +### Task 9: Manual check against `/home/user/example_course_content`, report. diff --git a/packages/feature-import/dev/serve.ts b/packages/feature-import/dev/serve.ts new file mode 100644 index 0000000..0896835 --- /dev/null +++ b/packages/feature-import/dev/serve.ts @@ -0,0 +1,23 @@ +// Dev server for the import UI while real sign-in (learner package) is not merged: everyone is the owner. +// Usage: STC_PORT=3462 STC_DATA_DIR=/tmp/stc-import-data STC_PYTHON=… bun packages/feature-import/dev/serve.ts +import { resolve } from "node:path"; +import type { AuthService, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue, loadConfig } from "@stc/core"; +import { createCourseRepository } from "@stc/course-format"; +import { connect, migrate } from "@stc/db"; +import indexHtml from "../../../apps/web/index.html"; +import { importFeature } from "../src"; + +const config = loadConfig(); +config.courseDirs.push(resolve(config.dataDir, "courses")); +const db = await connect(config.databaseUrl); +await migrate(db); +const auth: AuthService = { userFromRequest: async () => ({ id: "dev", email: "dev@localhost", name: "Dev owner", roles: ["owner"], groups: [] }) }; +const ctx = { config, db, courses: await createCourseRepository(config.courseDirs), auth } as ServerContext; +const feature = importFeature(ctx); +const jobs = createJobQueue(db, feature.jobs!); +ctx.jobs = jobs; +const app = createApp(ctx, [feature]); +jobs.start(); +const server = Bun.serve({ port: config.port, routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, fetch: (req) => app.fetch(req) }); +console.log(`import dev server on ${server.url}`); diff --git a/packages/feature-import/package.json b/packages/feature-import/package.json index cab5a10..c758331 100644 --- a/packages/feature-import/package.json +++ b/packages/feature-import/package.json @@ -3,5 +3,7 @@ "version": "0.0.0", "private": true, "type": "module", - "license": "AGPL-3.0-only" + "license": "AGPL-3.0-only", + "main": "src/index.ts", + "exports": { ".": "./src/index.ts", "./web": "./src/web.ts" } } diff --git a/packages/feature-import/py/pdf_tool.py b/packages/feature-import/py/pdf_tool.py new file mode 100644 index 0000000..0bff2b7 --- /dev/null +++ b/packages/feature-import/py/pdf_tool.py @@ -0,0 +1,50 @@ +"""PDF helper for the STCourses importer. Run as: python -I pdf_tool.py render [--width 1280] [--quality 80] + +Writes outdir/p-001.jpg ... and outdir/text.json ([{"page": 1, "text": "..."}]). +Prints one "page N/M" line per page on stdout and a final JSON line {"pages": M, "title": "..."}. +""" +import json +import os +import sys + +try: + import pymupdf as fitz +except ImportError: # older PyMuPDF + import fitz + + +def render(pdf, outdir, width, quality): + os.makedirs(outdir, exist_ok=True) + doc = fitz.open(pdf) + n = len(doc) + texts = [] + for i, page in enumerate(doc, start=1): + zoom = width / max(page.rect.width, 1) + pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom), alpha=False) + pix.save(os.path.join(outdir, "p-%03d.jpg" % i), jpg_quality=quality) + texts.append({"page": i, "text": page.get_text().strip()}) + print("page %d/%d" % (i, n), flush=True) + with open(os.path.join(outdir, "text.json"), "w", encoding="utf-8") as f: + json.dump(texts, f, ensure_ascii=False) + print(json.dumps({"pages": n, "title": (doc.metadata or {}).get("title") or ""}), flush=True) + + +def main(argv): + if len(argv) < 4 or argv[1] != "render": + print("usage: pdf_tool.py render [--width N] [--quality N]", file=sys.stderr) + return 2 + width, quality = 1280, 80 + rest = argv[4:] + while rest: + flag, val = rest[0], rest[1] if len(rest) > 1 else "" + if flag == "--width": + width = int(val) + elif flag == "--quality": + quality = int(val) + rest = rest[2:] + render(argv[2], argv[3], width, quality) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/packages/feature-import/src/context.ts b/packages/feature-import/src/context.ts new file mode 100644 index 0000000..caeb3d2 --- /dev/null +++ b/packages/feature-import/src/context.ts @@ -0,0 +1,102 @@ +import { mkdir, readdir, rename, stat } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; +import type { ServerContext } from "@stc/contracts"; +import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest"; +import { slugify, uniqueId } from "./util/text"; +import { extractFrame, type Tools } from "./media"; + +export interface ImportEnv { [k: string]: string | undefined } +export interface ImportOptions { env?: ImportEnv } + +export interface Deps { + ctx: ServerContext; + env: ImportEnv; + tools: Tools; + python: string; +} + +export const coursesDir = (d: Deps) => resolve(d.ctx.config.dataDir, "courses"); +export const cacheDir = (d: Deps) => resolve(d.ctx.config.dataDir, "cache"); +export const deckCacheDir = (d: Deps, courseId: string, deckId: string) => join(cacheDir(d), "decks", courseId, deckId); +export const coverCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "covers", courseId); +export const transcriptCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "transcripts", courseId); + +export const isPlainId = (s: string) => /^[a-z0-9][a-z0-9-]*$/.test(s); + +/** A fresh course id from a title, unique among managed folders and every course the server already knows. */ +export async function freshCourseId(d: Deps, wanted: string | undefined, title: string): Promise { + if (wanted !== undefined) { + if (!isPlainId(wanted)) throw new Error(`invalid courseId "${wanted}" (use lowercase letters, digits and dashes)`); + if (await exists(join(coursesDir(d), wanted)) || d.ctx.courses.get(wanted)) throw new Error(`a course with id "${wanted}" already exists`); + return wanted; + } + const taken = new Set(d.ctx.courses.list().map((c) => c.id)); + for (const e of await readdir(coursesDir(d)).catch(() => [] as string[])) taken.add(e); + return uniqueId(slugify(title), (x) => taken.has(x)); +} + +export async function exists(p: string): Promise { + return stat(p).then(() => true, () => false); +} + +/** Scratch folder on the same filesystem as the courses, so finished work is moved in with one rename. */ +export async function stageDir(d: Deps, label: string): Promise { + const dir = resolve(d.ctx.config.dataDir, "tmp", `${label}-${Bun.randomUUIDv7()}`); + await mkdir(dir, { recursive: true }); + return dir; +} + +export async function publishCourse(d: Deps, stage: string, id: string): Promise { + await mkdir(coursesDir(d), { recursive: true }); + const dest = join(coursesDir(d), id); + await rename(stage, dest); + return dest; +} + +export interface ManagedCourse { id: string; dir: string; manifest: RawManifest } + +/** An existing course the importer may add to. Only folders directly under /courses qualify: overlays and authored repos are never edited. */ +export async function managedCourse(d: Deps, courseId: string | undefined): Promise { + if (!courseId) throw new Error("options.courseId is required: choose the course to add to"); + const dir = join(coursesDir(d), courseId); + const rel = relative(coursesDir(d), dir); + if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) { + throw new Error(`course "${courseId}" is not an imported course under ${coursesDir(d)}; the importer only edits courses it created`); + } + const manifest = await readRawManifest(dir); + if (manifest.id !== courseId) throw new Error(`course folder "${courseId}" has manifest id "${manifest.id}"`); + return { id: courseId, dir, manifest }; +} + +/** Reloads the repository and turns load problems into warnings. */ +export async function finish(d: Deps, id: string, warnings: string[]): Promise { + await d.ctx.courses.reload(id); + const c = d.ctx.courses.get(id); + if (!c) { + warnings.push(`course "${id}" is not visible to the server: add ${coursesDir(d)} to STC_COURSE_DIRS`); + return; + } + for (const diag of c.diagnostics) warnings.push(`${diag.level}: ${diag.message}${diag.path ? ` (${diag.path})` : ""}`); +} + +export async function saveManifest(dir: string, manifest: RawManifest): Promise { + await writeManifest(dir, manifest); +} + +export function sourceBlock(kind: string, origin: string) { + return { kind, origin, importedAt: new Date().toISOString() }; +} + +/** Writes up to `count` frame candidates from evenly spaced clips; returns the candidate file names. Failures are skipped. */ +export async function frameCandidates(d: Deps, courseId: string, clips: string[], count = 3): Promise { + const out: string[] = []; + const n = Math.min(count, clips.length); + for (let i = 0; i < n; i++) { + const name = `frame-${i + 1}.jpg`; + try { + await extractFrame(d.tools, clips[Math.floor((i * clips.length) / n)]!, join(coverCacheDir(d, courseId), name), 0.2); + out.push(name); + } catch { /* a bad clip must not fail the import */ } + } + return out; +} diff --git a/packages/feature-import/src/deck.ts b/packages/feature-import/src/deck.ts new file mode 100644 index 0000000..9844060 --- /dev/null +++ b/packages/feature-import/src/deck.ts @@ -0,0 +1,49 @@ +import { basename, extname, join } from "node:path"; +import { copyFile, mkdir } from "node:fs/promises"; +import { coverCacheDir, deckCacheDir, exists, type Deps, type ManagedCourse } from "./context"; +import { putFile } from "./media"; +import { renderPdf } from "./pdf"; +import { slugify, uniqueId } from "./util/text"; + +export interface AttachedDeck { deckId: string; file: string; pages?: number; warning?: string } + +/** All lesson and module ids of a raw manifest, for validating "attach to". */ +export function attachTargets(manifest: Record): Set { + const ids = new Set(); + for (const m of manifest.modules ?? []) { + ids.add(m.id); + for (const l of m.lessons ?? []) ids.add(l.id); + } + return ids; +} + +/** + * Copies a PDF into the course (media/decks/), registers it in the (in-memory) manifest, and builds the derived cache: + * page images + text.json under /cache/decks/// and the first page as a cover candidate. + * A failure of the PDF tool only degrades to a warning: the deck stays attached and the cache can be rebuilt. + * The caller writes the manifest. + */ +export async function attachDeck(d: Deps, course: ManagedCourse, pdfPath: string, attachTo: string, opts: { title?: string; mode?: "copy" | "link"; onProgress?: (fraction: number) => void } = {}): Promise { + if (!attachTarget(course, attachTo)) throw new Error(`attachTo "${attachTo}" is not a lesson or module of course "${course.id}"`); + const decks: any[] = (course.manifest.decks ??= []); + const baseName = basename(pdfPath, extname(pdfPath)); + const deckId = uniqueId(`d-${slugify(baseName)}`, (x) => decks.some((k) => k.id === x)); + let file = `media/decks/${basename(pdfPath)}`; + for (let n = 2; await exists(join(course.dir, file)); n++) file = `media/decks/${baseName}-${n}${extname(pdfPath)}`; + await putFile(pdfPath, join(course.dir, file), opts.mode ?? "copy"); + decks.push({ id: deckId, file, title: opts.title ?? baseName.replace(/[_]+/g, " ").trim(), attachTo }); + + const out: AttachedDeck = { deckId, file }; + try { + const r = await renderPdf(d.python, join(course.dir, file), deckCacheDir(d, course.id, deckId), { onPage: (p, t) => opts.onProgress?.(p / t) }); + out.pages = r.pages; + await mkdir(coverCacheDir(d, course.id), { recursive: true }); + await copyFile(join(deckCacheDir(d, course.id, deckId), "p-001.jpg"), join(coverCacheDir(d, course.id), `deck-${deckId}.jpg`)); + } catch (e) { + out.warning = `deck "${basename(pdfPath)}" is attached, but page images and text could not be built (${e instanceof Error ? e.message.split("\n")[0] : e}); STC_PYTHON must point to Python with PyMuPDF`; + } + return out; +} + +const attachTarget = (c: ManagedCourse, id: string) => attachTargets(c.manifest).has(id); + diff --git a/packages/feature-import/src/index.ts b/packages/feature-import/src/index.ts new file mode 100644 index 0000000..367fe43 --- /dev/null +++ b/packages/feature-import/src/index.ts @@ -0,0 +1,39 @@ +import type { Feature, JobHandler, ServerContext } from "@stc/contracts"; +import type { Deps, ImportOptions } from "./context"; +import { importCodeRepo } from "./jobs/code-repo"; +import { importPacktVideo } from "./jobs/packt-video"; +import { importPdfDeck } from "./jobs/pdf-deck"; +import { transcribeCourse } from "./jobs/transcribe"; +import { importVideoFolder } from "./jobs/video-folder"; +import { ensureRoots } from "./roots"; +import { importRoutes } from "./routes"; + +export type { ImportOptions } from "./context"; +export { deckCacheDir, coverCacheDir, transcriptCacheDir } from "./context"; + +/** + * `ctx` is only read when a job runs or a request arrives, never while this function runs, so the server can pass + * its context object before the job queue (which needs these handlers) exists. + * Environment: STC_PYTHON (Python with PyMuPDF, default python3), STC_FFMPEG / STC_FFPROBE, STC_TRANSCRIBE_URL (+ _KEY, _MODEL). + */ +export function importFeature(ctx: ServerContext, opts: ImportOptions = {}): Feature { + const deps = (): Deps => { + const env = opts.env ?? process.env; + return { ctx, env, tools: { ffmpeg: env.STC_FFMPEG || "ffmpeg", ffprobe: env.STC_FFPROBE || "ffprobe" }, python: env.STC_PYTHON || "python3" }; + }; + const job = (fn: (d: Deps, p: unknown, jc: Parameters[1]) => Promise): JobHandler => (p, jc) => fn(deps(), p, jc); + ensureRoots({ ctx, env: opts.env ?? process.env }); + return { + name: "import", + routes: importRoutes(deps), + jobs: { + "import.packt-video": job(importPacktVideo), + "import.code-repo": job(importCodeRepo), + "import.pdf-deck": job(importPdfDeck), + "import.video-folder": job(importVideoFolder), + "import.transcribe": job(transcribeCourse), + }, + }; +} + +export { importWeb } from "./web"; diff --git a/packages/feature-import/src/jobs/code-repo.ts b/packages/feature-import/src/jobs/code-repo.ts new file mode 100644 index 0000000..29aff79 --- /dev/null +++ b/packages/feature-import/src/jobs/code-repo.ts @@ -0,0 +1,135 @@ +import { cp, lstat, mkdir, readdir } from "node:fs/promises"; +import { basename, extname, join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context"; +import { attachDeck } from "../deck"; +import { putFile } from "../media"; +import { run } from "../util/exec"; +import { assertAllowedPath } from "../roots"; +import { openSource } from "../util/source"; +import { containment } from "../util/text"; +import { optStr, parsePayload, withStage } from "./common"; + +const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)[^\s]+$/; +const SKIP_DIRS = new Set([".git", "node_modules", "__pycache__", ".venv"]); +const NUMBER = /^\s*(?:s|sec(?:tion)?|mod(?:ule)?)?\s*0*(\d+)\b\s*[-–—:.)_]*\s*/i; + +export function folderNumber(name: string): number | null { + const m = NUMBER.exec(name); + return m ? Number(m[1]) : null; +} +export function folderName(name: string): string { + return name.replace(NUMBER, "").trim(); +} + +function moduleNumber(m: { id: string; title: string }, index: number): number { + const byId = /^m0*(\d+)$/i.exec(m.id); + if (byId) return Number(byId[1]); + return folderNumber(m.title) ?? index + 1; +} + +/** Single-directory wrappers ("Repo-main/…") of a zip are skipped. */ +async function effectiveRoot(dir: string): Promise { + for (let i = 0; i < 3; i++) { + const entries = (await readdir(dir, { withFileTypes: true })).filter((e) => !e.name.startsWith(".") && e.name !== "__MACOSX"); + if (entries.length === 1 && entries[0]!.isDirectory()) dir = join(dir, entries[0]!.name); + else break; + } + return dir; +} + +async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise { + for (const e of await readdir(dir, { withFileTypes: true })) { + if (SKIP_DIRS.has(e.name)) continue; + if (e.isDirectory() && depth < 2) await findPdfs(join(dir, e.name), depth + 1, out); + else if (e.isFile() && extname(e.name).toLowerCase() === ".pdf") out.push(join(dir, e.name)); + } + return out; +} + +export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "code-repo"); + const warnings: string[] = []; + const course = await managedCourse(d, optStr(options, "courseId")); + if (!GIT_URL.test(path)) await assertAllowedPath(d, path); + const stage = await stageDir(d, "repo"); + + return withStage(stage, async () => { + let root: string; + if (GIT_URL.test(path)) { + await jc.reportProgress(0.05, "Cloning the repository"); + await run(["git", "clone", "--depth", "1", "--", path, join(stage, "repo")], { signal: jc.signal, env: { GIT_TERMINAL_PROMPT: "0" } }); + root = join(stage, "repo"); + } else if (/\.zip$/i.test(path)) { + await jc.reportProgress(0.05, "Unpacking the repository zip"); + const src = await openSource(path); + await src.extract(src.entries.filter((e) => !e.startsWith("__MACOSX/")), join(stage, "repo"), jc.signal); + root = await effectiveRoot(join(stage, "repo")); // GitHub-style zips wrap everything in one folder + } else { + root = path; // an extracted folder is only read + } + + const modules: any[] = course.manifest.modules ?? []; + const folders = (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")).map((e) => e.name).sort(); + const attachments: any[] = (course.manifest.attachments ??= []); + const claimed = new Set(); + let matched = 0; + + for (const [i, folder] of folders.entries()) { + await jc.reportProgress(0.1 + 0.8 * (i / Math.max(1, folders.length)), `Matching ${folder}`); + const num = folderNumber(folder); + const name = folderName(folder); + let mod = num !== null ? modules.find((m, idx) => moduleNumber(m, idx) === num) : undefined; + if (mod && name && containment(name, mod.title) < 0.3 && containment(mod.title, name) < 0.3) { + warnings.push(`"${folder}" was matched to module ${num} ("${mod.title}") by number only; the names differ`); + } + if (!mod && name) { + let best = 0; + for (const m of modules) { + const score = containment(name, m.title); + if (score > best) { best = score; mod = m; } + } + if (best < 0.6) mod = undefined; + } + if (!mod) { warnings.push(`folder "${folder}" matches no module and was skipped`); continue; } + if (claimed.has(mod.id)) { warnings.push(`folder "${folder}" also matches module "${mod.id}", which already got code; skipped`); continue; } + if (mod.code) { warnings.push(`module "${mod.id}" already has a code folder; "${folder}" skipped`); continue; } + claimed.add(mod.id); + matched++; + + const srcDir = join(root, folder); + const dest = `code/${folder}`; + await mkdir(join(course.dir, "code"), { recursive: true }); + await cp(srcDir, join(course.dir, dest), { + recursive: true, + filter: async (s) => { + const base = basename(s); + if (SKIP_DIRS.has(base)) return false; + const st = await lstat(s); + if (st.isSymbolicLink()) return false; // a link could point outside the course + return !(st.isFile() && extname(base).toLowerCase() === ".pdf"); + }, + }); + mod.code = { dir: dest }; + + for (const pdf of await findPdfs(srcDir)) { + const title = basename(pdf, ".pdf").replace(/_/g, " ").trim(); + if (/cheat|quick.?ref|reference|handout/i.test(basename(pdf))) { + let rel = `media/files/${basename(pdf)}`; + await putFile(pdf, join(course.dir, rel)); + attachments.push({ file: rel, title, module: mod.id }); + } else { + const r = await attachDeck(d, course, pdf, mod.id, { title }); + if (r.warning) warnings.push(r.warning); + } + } + } + if (attachments.length === 0) delete course.manifest.attachments; + if (matched === 0) warnings.push("no folder of the repository matched a module; nothing was attached"); + + await saveManifest(course.dir, course.manifest); + await finish(d, course.id, warnings); + await jc.reportProgress(1, `Attached code to ${matched} module(s)`); + return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings }; + }); +} diff --git a/packages/feature-import/src/jobs/common.ts b/packages/feature-import/src/jobs/common.ts new file mode 100644 index 0000000..f8158fc --- /dev/null +++ b/packages/feature-import/src/jobs/common.ts @@ -0,0 +1,48 @@ +import type { ImportResult, ImportSource, JobContext } from "@stc/contracts"; +import { copyFile, mkdir, rm } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { coverCacheDir, exists, type Deps } from "../context"; + +export type Opts = Record; + +export function parsePayload(payload: unknown, kind: ImportSource["kind"]): { path: string; options: Opts } { + const p = payload as Partial | null; + if (!p || typeof p.path !== "string" || p.path === "") throw new Error("payload.path is required"); + if (p.kind !== undefined && p.kind !== kind) throw new Error(`payload.kind "${p.kind}" does not match job kind "${kind}"`); + return { path: p.path, options: (p.options ?? {}) as Opts }; +} + +export const optStr = (o: Opts, k: string): string | undefined => (typeof o[k] === "string" && (o[k] as string).trim() !== "" ? (o[k] as string).trim() : undefined); +export const optBool = (o: Opts, k: string): boolean => o[k] === true || o[k] === "true"; + +/** Runs a job body with a scratch folder that is always removed, and a result shape shared by all import jobs. */ +export async function withStage(stage: string, fn: () => Promise): Promise { + try { + return await fn(); + } finally { + await rm(stage, { recursive: true, force: true }); + } +} + +/** Copies the first cover candidate into the course (relative path returned) so the library has a cover straight away. */ +export async function adoptCover(d: Deps, courseId: string, courseDir: string, candidate: string | undefined): Promise { + if (!candidate) return undefined; + const src = join(coverCacheDir(d, courseId), candidate); + if (!(await exists(src))) return undefined; + const rel = "media/cover.jpg"; + await mkdir(dirname(join(courseDir, rel)), { recursive: true }); + await copyFile(src, join(courseDir, rel)); + return rel; +} + +export async function maybeQueueTranscription(d: Deps, courseId: string, wanted: boolean, warnings: string[]): Promise { + if (!wanted) return; + if (!d.env.STC_TRANSCRIBE_URL) { + warnings.push("transcription requested but STC_TRANSCRIBE_URL is not set; skipped"); + return; + } + await d.ctx.jobs.enqueue("import.transcribe", { courseId }); +} + +export type Reporter = JobContext["reportProgress"]; +export type { ImportResult }; diff --git a/packages/feature-import/src/jobs/packt-video.ts b/packages/feature-import/src/jobs/packt-video.ts new file mode 100644 index 0000000..0103736 --- /dev/null +++ b/packages/feature-import/src/jobs/packt-video.ts @@ -0,0 +1,89 @@ +import { mkdir, rename, rm } from "node:fs/promises"; +import { basename, dirname, join, posix } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context"; +import { probeDuration } from "../media"; +import { parsePacktIndex } from "../packt"; +import { assertAllowedPath } from "../roots"; +import { openSource } from "../util/source"; +import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common"; + +export async function importPacktVideo(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "packt-video"); + const warnings: string[] = []; + await assertAllowedPath(d, path); + await jc.reportProgress(0.02, "Reading the package"); + const src = await openSource(path); + + // The player page is the shallowest index.html that lists sections. + const candidates = src.entries.filter((e) => /(^|\/)index\.html$/i.test(e)).sort((a, b) => a.split("/").length - b.split("/").length); + let indexEntry: string | undefined; + let index: ReturnType | undefined; + for (const c of candidates) { + try { index = parsePacktIndex(await src.readText(c)); indexEntry = c; break; } catch { /* try the next */ } + } + if (!index || indexEntry === undefined) throw new Error("no Packt player index.html (with \"Section N: …\" headings) found in the package"); + warnings.push(...index.warnings); + const base = dirname(indexEntry) === "." ? "" : dirname(indexEntry); + const at = (rel: string) => posix.normalize(base ? `${base}/${rel}` : rel); + + const title = optStr(options, "title") ?? index.title; + const id = await freshCourseId(d, optStr(options, "courseId"), title); + const stage = await stageDir(d, "packt"); + + return withStage(stage, async () => { + const known = new Set(src.entries); + const modules: any[] = []; + const used = new Set(); + const clipFiles: string[] = []; + const total = index.sections.reduce((n, s) => n + s.clips.length, 0) || 1; + let done = 0; + for (const sec of index.sections) { + const mid = `m${sec.number}`; + const entries: { clip: (typeof sec.clips)[number]; entry: string }[] = []; + for (const clip of sec.clips) { + const entry = at(clip.video); + if (!known.has(entry)) { warnings.push(`video for ${sec.number}.${clip.number} is missing from the package: ${clip.video}`); continue; } + entries.push({ clip, entry }); + used.add(entry); + } + await jc.reportProgress(0.05 + 0.65 * (done / total), `Copying section ${sec.number}: ${sec.title}`); + const tmp = join(stage, ".extract"); + await src.extract(entries.map((e) => e.entry), tmp, jc.signal); + const lessons: any[] = []; + for (const { clip, entry } of entries) { + const rel = `media/${basename(entry)}`; + await mkdir(join(stage, "media"), { recursive: true }); + await rename(join(tmp, entry), join(stage, rel)); + clipFiles.push(join(stage, rel)); + const duration = await probeDuration(d.tools, join(stage, rel)); + lessons.push({ id: `${mid}-l${clip.number}`, title: clip.title, type: "video", file: rel, ...(duration ? { duration } : {}) }); + done++; + } + await rm(tmp, { recursive: true, force: true }); + modules.push({ id: mid, title: sec.title.replace(/^Section\s+\d+\s*[:.-]\s*/i, ""), lessons }); + } + const unused = src.entries.filter((e) => /\.(mp4|webm|m4v|mov)$/i.test(e) && !used.has(e)); + if (unused.length) warnings.push(`${unused.length} video file(s) in the package are not listed in index.html and were left out`); + + await jc.reportProgress(0.75, "Making cover candidates"); + const covers = await frameCandidates(d, id, clipFiles); + if (covers.length === 0 && clipFiles.length) warnings.push("could not extract cover frames (is ffmpeg available?)"); + const cover = await adoptCover(d, id, stage, covers[0]); + + const manifest = { + schema: 1, id, title, + visibility: "restricted", + ...(cover ? { cover } : {}), + source: sourceBlock("packt-video", basename(path)), + modules, + }; + await saveManifest(stage, manifest); + await jc.reportProgress(0.9, "Publishing the course"); + const dir = await publishCourse(d, stage, id); + await finish(d, id, warnings); + await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings); + await jc.reportProgress(1, `Imported ${total} clips`); + return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings }; + }); +} diff --git a/packages/feature-import/src/jobs/pdf-deck.ts b/packages/feature-import/src/jobs/pdf-deck.ts new file mode 100644 index 0000000..855815a --- /dev/null +++ b/packages/feature-import/src/jobs/pdf-deck.ts @@ -0,0 +1,28 @@ +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, managedCourse, saveManifest, type Deps } from "../context"; +import { assertAllowedPath } from "../roots"; +import { attachDeck } from "../deck"; +import { optStr, parsePayload } from "./common"; + +export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "pdf-deck"); + const warnings: string[] = []; + await assertAllowedPath(d, path); + if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`); + const course = await managedCourse(d, optStr(options, "courseId")); + const attachTo = optStr(options, "attachTo"); + if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to"); + await jc.reportProgress(0.05, "Copying the deck"); + const r = await attachDeck(d, course, path, attachTo, { + title: optStr(options, "title"), + mode: options.media === "link" ? "link" : "copy", + onProgress: (f) => void jc.reportProgress(0.1 + 0.8 * f, "Rendering slides"), + }); + if (r.warning) warnings.push(r.warning); + await saveManifest(course.dir, course.manifest); + await finish(d, course.id, warnings); + await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached"); + return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings }; +} diff --git a/packages/feature-import/src/jobs/transcribe.ts b/packages/feature-import/src/jobs/transcribe.ts new file mode 100644 index 0000000..7afd802 --- /dev/null +++ b/packages/feature-import/src/jobs/transcribe.ts @@ -0,0 +1,82 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { JobContext } from "@stc/contracts"; +import { finish, managedCourse, saveManifest, stageDir, transcriptCacheDir, type Deps } from "../context"; +import { run } from "../util/exec"; +import { optStr, withStage } from "./common"; + +export interface TranscribeResult { courseId: string; skipped?: boolean; reason?: string; transcribed: number; warnings: string[] } +interface Segment { start: number; end: number; text: string } + +const stamp = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`; + +/** Groups timed segments into paragraphs of roughly 30 seconds, each starting with a [m:ss] marker. */ +export function segmentsToMarkdown(title: string, segments: Segment[], fallbackText: string): string { + if (segments.length === 0) return `# ${title}\n\n${fallbackText.trim()}\n`; + const paras: string[] = []; + let cur: string[] = []; + let startedAt = segments[0]!.start; + for (const s of segments) { + if (cur.length && s.start - startedAt >= 30) { + paras.push(`[${stamp(startedAt)}] ${cur.join(" ")}`); + cur = []; startedAt = s.start; + } + cur.push(s.text.trim()); + } + if (cur.length) paras.push(`[${stamp(startedAt)}] ${cur.join(" ")}`); + return `# ${title}\n\n${paras.join("\n\n")}\n`; +} + +/** + * Optional job: sends each video lesson's audio to a Whisper-compatible endpoint (STC_TRANSCRIBE_URL, OpenAI + * /v1/audio/transcriptions shape) and attaches the result as the lesson's transcript. The model never runs in this process. + */ +export async function transcribeCourse(d: Deps, payload: unknown, jc: JobContext): Promise { + const p = (payload ?? {}) as Record; + const url = d.env.STC_TRANSCRIBE_URL; + const courseId = optStr(p, "courseId") ?? ""; + if (!url) return { courseId, skipped: true, reason: "STC_TRANSCRIBE_URL is not set", transcribed: 0, warnings: [] }; + const course = await managedCourse(d, courseId); + const only = Array.isArray(p.lessonIds) ? new Set(p.lessonIds.map(String)) : null; + const lessons = (course.manifest.modules as any[]).flatMap((m) => m.lessons as any[]).filter((l) => l.type === "video" && !l.transcript && (!only || only.has(l.id))); + const warnings: string[] = []; + const stage = await stageDir(d, "transcribe"); + let done = 0; + return withStage(stage, async () => { + try { + for (const [i, l] of lessons.entries()) { + if (jc.signal.aborted) break; + await jc.reportProgress(i / Math.max(1, lessons.length), `Transcribing ${l.title}`); + try { + const audio = join(stage, `${l.id}.ogg`); + await run([d.tools.ffmpeg, "-v", "error", "-y", "-i", join(course.dir, l.file), "-vn", "-ac", "1", "-ar", "16000", "-c:a", "libopus", "-b:a", "24k", audio], { signal: jc.signal }); + const form = new FormData(); + form.set("file", Bun.file(audio), `${l.id}.ogg`); + form.set("model", d.env.STC_TRANSCRIBE_MODEL || "whisper-1"); + form.set("response_format", "verbose_json"); + const res = await fetch(url, { + method: "POST", body: form, signal: jc.signal, + headers: d.env.STC_TRANSCRIBE_KEY ? { Authorization: `Bearer ${d.env.STC_TRANSCRIBE_KEY}` } : {}, + }); + if (!res.ok) throw new Error(`HTTP ${res.status} ${(await res.text()).slice(0, 200)}`); + const body = (await res.json()) as { text?: string; segments?: Segment[] }; + const segments = (body.segments ?? []).map((s) => ({ start: Number(s.start) || 0, end: Number(s.end) || 0, text: String(s.text ?? "") })); + const rel = `transcripts/${l.id}.md`; + await mkdir(join(course.dir, "transcripts"), { recursive: true }); + await writeFile(join(course.dir, rel), segmentsToMarkdown(l.title, segments, body.text ?? "")); + await mkdir(transcriptCacheDir(d, course.id), { recursive: true }); + await writeFile(join(transcriptCacheDir(d, course.id), `${l.id}.json`), JSON.stringify({ lessonId: l.id, text: body.text ?? "", segments })); + l.transcript = rel; + done++; + } catch (e) { + warnings.push(`${l.title}: ${e instanceof Error ? e.message : String(e)}`); + } + } + } finally { + if (done > 0) await saveManifest(course.dir, course.manifest); // keep what finished even if the job was cancelled + } + if (done > 0) await finish(d, course.id, warnings); + await jc.reportProgress(1, `Transcribed ${done} of ${lessons.length} lessons`); + return { courseId: course.id, transcribed: done, warnings }; + }); +} diff --git a/packages/feature-import/src/jobs/video-folder.ts b/packages/feature-import/src/jobs/video-folder.ts new file mode 100644 index 0000000..403f8af --- /dev/null +++ b/packages/feature-import/src/jobs/video-folder.ts @@ -0,0 +1,64 @@ +import { stat } from "node:fs/promises"; +import { basename, extname, join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context"; +import { probeDuration, putFile } from "../media"; +import { assertAllowedPath } from "../roots"; +import { naturalSort } from "../util/text"; +import { openSource } from "../util/source"; +import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common"; + +const VIDEO = /\.(mp4|m4v|webm|mov|mkv|avi)$/i; +const BROWSER_OK = /\.(mp4|m4v|webm|mov)$/i; + +export function lessonTitle(file: string): string { + const base = basename(file, extname(file)); + const t = base.replace(/^\s*\d+\s*[-_.)\s]+(?=\S)/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim(); + return t || base; +} + +export async function importVideoFolder(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "video-folder"); + const warnings: string[] = []; + await assertAllowedPath(d, path); + if (!(await stat(path).catch(() => null))?.isDirectory()) throw new Error(`not a folder: ${path}`); + const src = await openSource(path); + const files = naturalSort(src.entries.filter((e) => VIDEO.test(e) && !e.split("/").some((p) => p.startsWith(".")))); + if (files.length === 0) throw new Error(`no video files found in ${path}`); + const title = optStr(options, "title") ?? basename(path).replace(/[_]+/g, " "); + const id = await freshCourseId(d, optStr(options, "courseId"), title); + const mode = options.media === "link" ? "link" : "copy"; + const stage = await stageDir(d, "videos"); + + return withStage(stage, async () => { + const lessons: any[] = []; + const placed: string[] = []; + const used = new Set(); + for (const [i, f] of files.entries()) { + await jc.reportProgress(0.05 + 0.7 * (i / files.length), `Copying ${basename(f)}`); + let rel = `media/${f}`; + if (used.has(rel.toLowerCase())) rel = `media/${i + 1}-${basename(f)}`; + used.add(rel.toLowerCase()); + await putFile(join(path, f), join(stage, rel), mode); + placed.push(join(stage, rel)); + if (!BROWSER_OK.test(f)) warnings.push(`${f} may not play in browsers (${extname(f)}); consider converting it to MP4`); + const duration = await probeDuration(d.tools, join(stage, rel)); + lessons.push({ id: `l${i + 1}`, title: lessonTitle(f), type: "video", file: rel, ...(duration ? { duration } : {}) }); + } + await jc.reportProgress(0.8, "Making cover candidates"); + const covers = await frameCandidates(d, id, placed); + if (covers.length === 0) warnings.push("could not extract cover frames (is ffmpeg available?)"); + const cover = await adoptCover(d, id, stage, covers[0]); + await saveManifest(stage, { + schema: 1, id, title, visibility: "restricted", ...(cover ? { cover } : {}), + source: sourceBlock("video-folder", path), + modules: [{ id: "m1", title: "Videos", lessons }], + }); + await jc.reportProgress(0.92, "Publishing the course"); + const dir = await publishCourse(d, stage, id); + await finish(d, id, warnings); + await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings); + await jc.reportProgress(1, `Imported ${files.length} videos`); + return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings }; + }); +} diff --git a/packages/feature-import/src/media.ts b/packages/feature-import/src/media.ts new file mode 100644 index 0000000..0fab184 --- /dev/null +++ b/packages/feature-import/src/media.ts @@ -0,0 +1,40 @@ +import { link, mkdir, copyFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { run } from "./util/exec"; + +export interface Tools { ffmpeg: string; ffprobe: string } + +export async function probeDuration(t: Tools, file: string): Promise { + try { + const out = await run([t.ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file]); + const s = Number(out.trim()); + return Number.isFinite(s) && s > 0 ? Math.round(s) : undefined; + } catch { + return undefined; + } +} + +/** One JPEG frame at `fraction` of the clip, 640px wide. */ +export async function extractFrame(t: Tools, file: string, out: string, fraction = 0.2): Promise { + await mkdir(dirname(out), { recursive: true }); + const dur = (await probeDuration(t, file)) ?? 0; + const at = Math.max(0, dur * fraction); + await run([t.ffmpeg, "-v", "error", "-y", "-ss", String(at), "-i", file, "-frames:v", "1", "-vf", "scale=640:-2", "-q:v", "4", out]); +} + +/** + * Puts a large file into the course. "copy" uses `cp --reflink=auto` (a CoW clone where the filesystem supports it, + * a plain copy elsewhere); "link" makes a hard link and falls back to copying across filesystems. + * Either way the original is only read, and the course folder stays self-contained (the media route refuses symlinks that leave it). + */ +export async function putFile(src: string, dest: string, mode: "copy" | "link" = "copy"): Promise { + await mkdir(dirname(dest), { recursive: true }); + if (mode === "link") { + try { await link(src, dest); return; } catch { /* cross-device or unsupported: copy */ } + } + try { + await run(["cp", "--reflink=auto", "--", src, dest]); + } catch { + await copyFile(src, dest); + } +} diff --git a/packages/feature-import/src/packt.ts b/packages/feature-import/src/packt.ts new file mode 100644 index 0000000..07073d2 --- /dev/null +++ b/packages/feature-import/src/packt.ts @@ -0,0 +1,74 @@ +import { decodeEntities, stripTags } from "./util/text"; + +export interface PacktClip { number: number; title: string; video: string; } +export interface PacktSection { number: number; title: string; clips: PacktClip[] } +export interface PacktIndex { title: string; sections: PacktSection[]; warnings: string[] } + +/** + * Parses the Packt player's index.html structurally: + * - the section overview: `

Section N: …

` followed by a `
    ` of clip titles, in order, usually WITHOUT "Video N.M:" prefixes; + * - the per-section chapter blocks (`class="chapter" id="content-N"`) with one `
  • ` per clip: `

    N.M title

    ` and `data-video="…"`. + * Chapter blocks are authoritative for the video file; the overview list fills in any title or whole section that is missing. + */ +export function parsePacktIndex(html: string): PacktIndex { + const warnings: string[] = []; + const intro = /]*class="intro"[\s\S]*?]*>([\s\S]*?)<\/h1>/i.exec(html); + const titleTag = /]*>([\s\S]*?)<\/title>/i.exec(html); + const title = stripTags(intro?.[1] ?? titleTag?.[1] ?? "") || "Imported course"; + + const chapterStart = html.search(/]*class="chapter"/i); + const overview = chapterStart >= 0 ? html.slice(0, chapterStart) : html; + const chaptersHtml = chapterStart >= 0 ? html.slice(chapterStart) : ""; + + // overview: sections in document order + const secRe = /]*>\s*Section\s+(\d+)\s*[:.-]\s*([\s\S]*?)<\/h1>/gi; + const heads = [...overview.matchAll(secRe)]; + if (heads.length === 0) throw new Error("not a Packt video package: no \"Section N: …\" headings found in index.html"); + const overviewTitles = new Map(); + const sectionTitles = new Map(); + heads.forEach((h, i) => { + const n = Number(h[1]); + sectionTitles.set(n, stripTags(h[2]!)); + const block = overview.slice(h.index! + h[0].length, heads[i + 1]?.index ?? overview.length); + const ul = /]*>([\s\S]*?)<\/ul>/i.exec(block); + const items = ul ? [...ul[1]!.matchAll(/]*>([\s\S]*?)<\/li>/gi)].map((m) => stripTags(m[1]!)).filter(Boolean) : []; + const firstP = /]*>\s*Video\s+\d+\.\d+\s*:\s*([\s\S]*?)<\/p>/i.exec(block); + if (items.length === 0 && firstP) items.push(stripTags(firstP[1]!)); + overviewTitles.set(n, items); + }); + + // chapters: clips with their files + const chapterClips = new Map(); + const chapRe = /]*class="chapter"[^>]*id="content-(\d+)"[^>]*>/gi; + const chaps = [...chaptersHtml.matchAll(chapRe)]; + chaps.forEach((c, i) => { + const sec = Number(c[1]); + const block = chaptersHtml.slice(c.index! + c[0].length, chaps[i + 1]?.index ?? chaptersHtml.length); + const clipRe = /]*>\s*(?:Video\s+)?(\d+)\.(\d+)\s*[:.]?\s*([\s\S]*?)<\/h1>/gi; + const hs = [...block.matchAll(clipRe)]; + const clips: PacktClip[] = []; + hs.forEach((h, j) => { + const part = block.slice(h.index!, hs[j + 1]?.index ?? block.length); + const vid = /data-video="([^"]+)"/i.exec(part)?.[1] ?? /href="([^"]+\.(?:mp4|webm|m4v|mov))"/i.exec(part)?.[1]; + const num = Number(h[2]); + if (!vid) { warnings.push(`clip ${sec}.${num} has no video file in index.html`); return; } + clips.push({ number: num, title: stripTags(h[3]!), video: decodeEntities(vid) }); + }); + chapterClips.set(sec, clips); + }); + + const sections: PacktSection[] = []; + for (const n of [...sectionTitles.keys()].sort((a, b) => a - b)) { + let clips = chapterClips.get(n); + const titles = overviewTitles.get(n) ?? []; + if (!clips || clips.length === 0) { + warnings.push(`section ${n} has no chapter block; clips were taken from the overview list and mapped to videos/video${n}_M.mp4 in order`); + clips = titles.map((t, i) => ({ number: i + 1, title: t, video: `videos/video${n}_${i + 1}.mp4` })); + } else { + clips = clips.map((c, i) => ({ ...c, title: c.title || titles[i] || `Video ${n}.${c.number}` })); + if (titles.length && titles.length !== clips.length) warnings.push(`section ${n}: overview lists ${titles.length} clips but the chapter has ${clips.length}`); + } + sections.push({ number: n, title: sectionTitles.get(n)!, clips }); + } + return { title, sections, warnings }; +} diff --git a/packages/feature-import/src/pdf.ts b/packages/feature-import/src/pdf.ts new file mode 100644 index 0000000..c1b4366 --- /dev/null +++ b/packages/feature-import/src/pdf.ts @@ -0,0 +1,23 @@ +import { join } from "node:path"; +import { run } from "./util/exec"; + +export interface DeckRender { pages: number; title: string } + +const TOOL = join(import.meta.dir, "..", "py", "pdf_tool.py"); + +/** + * Renders page images (p-001.jpg ...) and text.json into `outDir` with PyMuPDF in a subprocess, + * so the server process never holds a PDF engine. `python` is STC_PYTHON (default python3). + */ +export async function renderPdf(python: string, pdf: string, outDir: string, opts: { onPage?: (page: number, total: number) => void; signal?: AbortSignal } = {}): Promise { + let last = ""; + await run([python, "-I", TOOL, "render", pdf, outDir], { + signal: opts.signal, + onLine(l) { + const m = /^page (\d+)\/(\d+)$/.exec(l); + if (m) opts.onPage?.(Number(m[1]), Number(m[2])); + else if (l.trim()) last = l; + }, + }); + return JSON.parse(last) as DeckRender; +} diff --git a/packages/feature-import/src/roots.ts b/packages/feature-import/src/roots.ts new file mode 100644 index 0000000..621c96f --- /dev/null +++ b/packages/feature-import/src/roots.ts @@ -0,0 +1,30 @@ +import { mkdirSync } from "node:fs"; +import { realpath } from "node:fs/promises"; +import { resolve, sep } from "node:path"; +import type { Deps } from "./context"; + +export const OUTSIDE = "path is outside the allowed import folders"; + +/** STC_IMPORT_ROOTS (":"-separated), default /import-inbox. */ +export function importRoots(d: Pick): string[] { + const raw = d.env.STC_IMPORT_ROOTS; + const list = raw ? raw.split(":").filter(Boolean) : [resolve(d.ctx.config.dataDir, "import-inbox")]; + return list.map((r) => resolve(r)); +} + +export function ensureRoots(d: Pick): void { + for (const r of importRoots(d)) { try { mkdirSync(r, { recursive: true }); } catch { /* reported when used */ } } +} + +/** Real path (symlinks followed) must lie inside the real path of one root. Throws OUTSIDE otherwise. */ +export async function assertAllowedPath(d: Pick, path: string): Promise { + let real: string; + try { real = await realpath(resolve(path)); } catch { throw new Error(OUTSIDE); } + for (const root of importRoots(d)) { + const rr = await realpath(root).catch(() => null); + if (rr && (real === rr || real.startsWith(rr + sep))) return real; + } + throw new Error(OUTSIDE); +} + +export const isGitUrl = (p: string) => /^(https?:\/\/|ssh:\/\/|git@)\S+$/.test(p); diff --git a/packages/feature-import/src/routes.ts b/packages/feature-import/src/routes.ts new file mode 100644 index 0000000..c44570f --- /dev/null +++ b/packages/feature-import/src/routes.ts @@ -0,0 +1,114 @@ +import { copyFile, mkdir, readdir } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; +import type { Feature, RouteHandler } from "@stc/contracts"; +import { serveMedia } from "@stc/core"; +import { assertAllowedPath, importRoots, OUTSIDE } from "./roots"; +import { coverCacheDir, coursesDir, exists, isPlainId, managedCourse, saveManifest, type Deps } from "./context"; + +export const KINDS = ["packt-video", "code-repo", "pdf-deck", "video-folder"] as const; +const OPTION_KEYS = ["title", "courseId", "attachTo", "media", "transcribe"] as const; +const COVER_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*\.jpg$/; +const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)\S+$/; + +const json = (body: unknown, status = 200) => Response.json(body, { status }); + +export function importRoutes(deps: () => Deps): Feature["routes"] { + /** Imports read arbitrary server paths, so every route needs the owner role. */ + const owner = (h: RouteHandler): RouteHandler => (req, rc) => { + if (!rc.user) return json({ error: "Sign in required" }, 401); + if (!rc.user.roles.includes("owner")) return json({ error: "Only the owner can import" }, 403); + return h(req, rc); + }; + + return { + "/api/import/info": { + GET: owner(() => { + const d = deps(); + const dir = coursesDir(d); + const served = d.ctx.config.courseDirs.some((c) => resolve(c) === dir); + return json({ transcribe: Boolean(d.env.STC_TRANSCRIBE_URL), coursesDir: dir, coursesDirServed: served, importRoots: importRoots(d) }); + }), + }, + "/api/import/jobs": { + GET: owner(async () => { + const jobs = (await deps().ctx.jobs.list()).filter((j) => j.kind.startsWith("import.")); + return json(jobs); + }), + POST: owner(async (req) => { + const d = deps(); + let body: any; + try { body = await req.json(); } catch { return json({ error: "Body must be JSON" }, 400); } + const kind = String(body?.kind ?? ""); + const options: Record = {}; + for (const k of OPTION_KEYS) if (body?.options?.[k] !== undefined && body.options[k] !== "") options[k] = body.options[k]; + if (kind === "transcribe") { + if (!d.env.STC_TRANSCRIBE_URL) return json({ error: "STC_TRANSCRIBE_URL is not set" }, 409); + if (typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400); + return json({ id: await d.ctx.jobs.enqueue("import.transcribe", { courseId: options.courseId }) }, 202); + } + if (!(KINDS as readonly string[]).includes(kind)) return json({ error: `kind must be one of ${KINDS.join(", ")}` }, 400); + const path = typeof body?.path === "string" ? body.path.trim() : ""; + if (!path) return json({ error: "path is required" }, 400); + const isGit = kind === "code-repo" && GIT_URL.test(path); + if (!isGit) { + if (!path.startsWith("/")) return json({ error: "Use an absolute path" }, 400); + try { await assertAllowedPath(d, path); } catch { return json({ error: OUTSIDE }, 400); } + } + if ((kind === "code-repo" || kind === "pdf-deck") && typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400); + if (kind === "pdf-deck" && typeof options.attachTo !== "string") return json({ error: "attachTo is required" }, 400); + return json({ id: await d.ctx.jobs.enqueue(`import.${kind}`, { kind, path, options }) }, 202); + }), + }, + "/api/import/courses": { + GET: owner(() => { + const d = deps(); + const base = coursesDir(d); + const out = d.ctx.courses.list() + .filter((c) => c.manifest && !relative(base, c.root).startsWith("..")) + .map((c) => ({ + id: c.id, title: c.manifest!.title, + modules: c.manifest!.modules.map((m) => ({ id: m.id, title: m.title, lessons: m.lessons.map((l) => ({ id: l.id, title: l.title })) })), + })); + return json(out); + }), + }, + "/api/import/covers/:courseId": { + GET: owner(async (_req, rc) => { + const d = deps(); + const id = rc.params.courseId!; + if (!isPlainId(id)) return json({ error: "Invalid course id" }, 400); + const c = d.ctx.courses.get(id); + const names = (await readdir(coverCacheDir(d, id)).catch(() => [] as string[])).filter((n) => COVER_NAME.test(n)).sort(); + return json({ current: c?.manifest?.cover ?? null, candidates: names }); + }), + POST: owner(async (req, rc) => { + const d = deps(); + let body: any; + try { body = await req.json(); } catch { return json({ error: "Body must be JSON" }, 400); } + const name = String(body?.name ?? ""); + if (!COVER_NAME.test(name)) return json({ error: "Invalid candidate name" }, 400); + if (!isPlainId(rc.params.courseId!)) return json({ error: "Invalid course id" }, 400); + const src = join(coverCacheDir(d, rc.params.courseId!), name); + if (!(await exists(src))) return json({ error: "No such candidate" }, 404); + let course; + try { course = await managedCourse(d, rc.params.courseId); } catch (e) { return json({ error: (e as Error).message }, 404); } + await mkdir(join(course.dir, "media"), { recursive: true }); + await copyFile(src, join(course.dir, "media", "cover.jpg")); + course.manifest.cover = "media/cover.jpg"; + await saveManifest(course.dir, course.manifest); + await d.ctx.courses.reload(course.id); + return json({ cover: "media/cover.jpg" }); + }), + }, + "/api/import/covers/:courseId/:name": { + GET: owner(async (req, rc) => { + const name = rc.params.name!; + if (!COVER_NAME.test(name)) return new Response("Not found", { status: 404 }); + if (!isPlainId(rc.params.courseId!)) return new Response("Not found", { status: 404 }); + const file = join(coverCacheDir(deps(), rc.params.courseId!), name); + if (!(await exists(file))) return new Response("Not found", { status: 404 }); + return serveMedia(req, file); + }), + }, + }; +} diff --git a/packages/feature-import/src/util/exec.ts b/packages/feature-import/src/util/exec.ts new file mode 100644 index 0000000..78b3c2f --- /dev/null +++ b/packages/feature-import/src/util/exec.ts @@ -0,0 +1,30 @@ +/** Runs a command (no shell). Resolves with stdout; rejects with the stderr tail on a non-zero exit. */ +export async function run(cmd: string[], opts: { signal?: AbortSignal; env?: Record; onLine?: (line: string) => void } = {}): Promise { + const proc = Bun.spawn(cmd, { + stdout: "pipe", stderr: "pipe", stdin: "ignore", + env: { ...process.env, LC_ALL: "C.UTF-8", ...opts.env } as Record, + signal: opts.signal, + }); + const out = (async () => { + let text = ""; + let pending = ""; + const dec = new TextDecoder(); + for await (const chunk of proc.stdout) { + const s = dec.decode(chunk, { stream: true }); + text += s; + if (opts.onLine) { + pending += s; + const lines = pending.split("\n"); + pending = lines.pop()!; + for (const l of lines) opts.onLine(l); + } + } + return text; + })(); + const [stdout, stderr, code] = await Promise.all([out, new Response(proc.stderr).text(), proc.exited]); + if (code !== 0) { + const tail = stderr.trim().split("\n").slice(-6).join("\n"); + throw new Error(`${cmd[0]} exited with ${code}${tail ? `: ${tail}` : ""}`); + } + return stdout; +} diff --git a/packages/feature-import/src/util/manifest.ts b/packages/feature-import/src/util/manifest.ts new file mode 100644 index 0000000..80b4ab3 --- /dev/null +++ b/packages/feature-import/src/util/manifest.ts @@ -0,0 +1,21 @@ +import { readFile, rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { ManifestSchema } from "@stc/course-format"; + +export type RawManifest = Record; + +export async function readRawManifest(courseDir: string): Promise { + return Bun.YAML.parse(await readFile(join(courseDir, "stcourse.yaml"), "utf8")) as RawManifest; +} + +/** Validates against the course-format schema, then writes stcourse.yaml via temp file + rename. */ +export async function writeManifest(courseDir: string, manifest: RawManifest): Promise { + const r = ManifestSchema.safeParse(manifest); + if (!r.success) { + throw new Error(`generated manifest is invalid: ${r.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`); + } + const target = join(courseDir, "stcourse.yaml"); + const tmp = `${target}.${process.pid}.tmp`; + await writeFile(tmp, Bun.YAML.stringify(manifest, null, 2) + "\n"); + await rename(tmp, target); +} diff --git a/packages/feature-import/src/util/source.ts b/packages/feature-import/src/util/source.ts new file mode 100644 index 0000000..40cb297 --- /dev/null +++ b/packages/feature-import/src/util/source.ts @@ -0,0 +1,67 @@ +import { cp, mkdir, readdir, readFile, rm, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { run } from "./exec"; + +/** A read-only view of an input: a zip file or an already extracted folder. Nothing in here writes to the input. */ +export interface Source { + kind: "zip" | "dir"; + entries: string[]; // files only, "/"-separated, relative to the source root + readText(entry: string): Promise; + /** Copies the given entries to `dest`, keeping their relative paths. */ + extract(entries: string[], dest: string, signal?: AbortSignal): Promise; +} + +const globEscape = (s: string) => s.replace(/[\\[\]*?]/g, (c) => `\\${c}`); + +async function walk(root: string, rel = "", out: string[] = []): Promise { + for (const e of await readdir(join(root, rel), { withFileTypes: true })) { + const r = rel ? `${rel}/${e.name}` : e.name; + if (e.isDirectory()) await walk(root, r, out); + else if (e.isFile()) out.push(r); + } + return out; +} + +async function removeSymlinks(dir: string): Promise { + for (const e of await readdir(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isSymbolicLink()) await rm(p, { force: true }); + else if (e.isDirectory()) await removeSymlinks(p); + } +} + +export async function openSource(path: string): Promise { + const st = await stat(path).catch(() => null); + if (!st) throw new Error(`source not found: ${path}`); + if (st.isDirectory()) { + return { + kind: "dir", + entries: await walk(path), + readText: (e) => readFile(join(path, e), "utf8"), + async extract(entries, dest) { + for (const e of entries) { + await mkdir(dirname(join(dest, e)), { recursive: true }); + await cp(join(path, e), join(dest, e)); + } + }, + }; + } + const listing = await run(["unzip", "-Z1", path]); + const entries = listing.split("\n").filter((l) => l && !l.endsWith("/")); + return { + kind: "zip", + entries, + readText: (e) => run(["unzip", "-p", path, globEscape(e)]), + async extract(list, dest, signal) { + await mkdir(dest, { recursive: true }); + // unzip treats arguments as patterns, so glob characters in names are escaped; batches keep argv small. + try { + for (let i = 0; i < list.length; i += 200) { + await run(["unzip", "-q", "-o", path, ...list.slice(i, i + 200).map(globEscape), "-d", dest], { signal }); + } + } finally { + await removeSymlinks(dest); // a zip can carry symlinks; none may survive extraction + } + }, + }; +} diff --git a/packages/feature-import/src/util/text.ts b/packages/feature-import/src/util/text.ts new file mode 100644 index 0000000..6fffbab --- /dev/null +++ b/packages/feature-import/src/util/text.ts @@ -0,0 +1,48 @@ +export function slugify(s: string): string { + const slug = s + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 48) + .replace(/-+$/g, ""); + return slug || "course"; +} + +export function uniqueId(base: string, taken: (id: string) => boolean): string { + if (!taken(base)) return base; + for (let i = 2; ; i++) if (!taken(`${base}-${i}`)) return `${base}-${i}`; +} + +const NAMED: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " }; +export function decodeEntities(s: string): string { + return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => { + if (e[0] === "#") { + const code = e[1]!.toLowerCase() === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); + return Number.isFinite(code) ? String.fromCodePoint(code) : m; + } + return NAMED[e.toLowerCase()] ?? m; + }); +} + +export function stripTags(s: string): string { + return decodeEntities(s.replace(/<[^>]*>/g, "")).replace(/\s+/g, " ").trim(); +} + +const collator = new Intl.Collator("en", { numeric: true, sensitivity: "base" }); +export function naturalSort(list: string[]): string[] { + return [...list].sort((a, b) => collator.compare(a, b) || (a < b ? -1 : a > b ? 1 : 0)); +} + +/** Lowercase alphanumeric words, for loose name matching. */ +export function words(s: string): string[] { + return s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 1); +} +/** Share of the words of `a` that also occur in `b`. */ +export function containment(a: string, b: string): number { + const wa = words(a); + if (wa.length === 0) return 0; + const wb = new Set(words(b)); + return wa.filter((w) => wb.has(w)).length / wa.length; +} diff --git a/packages/feature-import/src/web.ts b/packages/feature-import/src/web.ts new file mode 100644 index 0000000..369fa85 --- /dev/null +++ b/packages/feature-import/src/web.ts @@ -0,0 +1,7 @@ +import type { WebFeature } from "@stc/contracts"; + +/** Browser-safe entry (no server imports): apps/web imports this as "@stc/feature-import/web". */ +export const importWeb: WebFeature = { + name: "import", + routes: [{ path: "/admin/import", element: () => import("./web/ImportPage"), nav: { label: "Import", area: "admin" } }], +}; diff --git a/packages/feature-import/src/web/ImportPage.tsx b/packages/feature-import/src/web/ImportPage.tsx new file mode 100644 index 0000000..dd6a51d --- /dev/null +++ b/packages/feature-import/src/web/ImportPage.tsx @@ -0,0 +1,184 @@ +import { useCallback, useEffect, useState, type FormEvent } from "react"; +import { Pill, ProgressBar, SegmentedToggle } from "@stc/ui"; +import "./import.css"; + +type Kind = "packt-video" | "code-repo" | "pdf-deck" | "video-folder"; +interface Job { id: string; kind: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; payload: any; result: any; error: string | null; createdAt: string } +interface CourseInfo { id: string; title: string; modules: { id: string; title: string; lessons: { id: string; title: string }[] }[] } +interface Info { importRoots: string[]; transcribe: boolean; coursesDir: string; coursesDirServed: boolean } + +const KINDS: { value: Kind; label: string; hint: string; placeholder: string }[] = [ + { value: "packt-video", label: "Packt video zip", hint: "A video package whose index.html lists the sections and clips.", placeholder: "/data/downloads/video_zip.zip" }, + { value: "code-repo", label: "Code repository", hint: "A zip, a folder or a git URL. Folders such as S08 - Name attach to the matching section.", placeholder: "/data/downloads/course-main.zip or https://github.com/…" }, + { value: "pdf-deck", label: "Slide PDF", hint: "Becomes a deck on a lesson or section. Its text is extracted for search.", placeholder: "/data/downloads/slides.pdf" }, + { value: "video-folder", label: "Folder of videos", hint: "One lesson per video file, in natural file-name order.", placeholder: "/mnt/videos/my-course" }, +]; +const LABEL: Record = { "import.packt-video": "Packt video", "import.code-repo": "Code repository", "import.pdf-deck": "Slide PDF", "import.video-folder": "Video folder", "import.transcribe": "Transcription" }; + +async function api(path: string, init?: RequestInit): Promise { + const r = await fetch(path, { ...init, headers: { "content-type": "application/json", ...init?.headers } }); + const body = await r.json().catch(() => ({})); + if (!r.ok) throw Object.assign(new Error(body.error ?? `HTTP ${r.status}`), { status: r.status }); + return body as T; +} + +function CoverPicker({ courseId }: { courseId: string }) { + const [data, setData] = useState<{ current: string | null; candidates: string[] } | null>(null); + const [version, setVersion] = useState(0); + useEffect(() => { api(`/api/import/covers/${courseId}`).then(setData).catch(() => setData(null)); }, [courseId, version]); + if (!data || data.candidates.length === 0) return null; + return ( +
    + Cover candidates +
    + {data.candidates.map((name) => ( +
    + {`Cover + { await api(`/api/import/covers/${courseId}`, { method: "POST", body: JSON.stringify({ name }) }); setVersion((v) => v + 1); }}>Use as cover +
    + ))} +
    +
    + ); +} + +function JobRow({ job }: { job: Job }) { + const src = job.payload?.path ? String(job.payload.path).split("/").filter(Boolean).pop() : job.payload?.courseId; + const result = job.result; + const warnings: string[] = result?.warnings ?? []; + return ( +
  • +
    + {LABEL[job.kind] ?? job.kind} + {src} + {job.status} +
    +
    + + {job.message} +
    + {job.error ?
    {job.error}
    : null} + {job.status === "done" && result?.skipped ?
    Skipped: {result.reason}
    : null} + {job.status === "done" && result?.courseId && !result.skipped ? ( +
    Course {result.courseId}, restricted to the owner until you change its visibility.
    + ) : null} + {warnings.length ? ( +
    {warnings.length} warning{warnings.length > 1 ? "s" : ""}
      {warnings.map((w, i) =>
    • {w}
      • )}
    + ) : null} + {job.status === "done" && (job.kind === "import.packt-video" || job.kind === "import.video-folder") && result?.courseId ? : null} +
    • + ); +} + +export default function ImportPage() { + const [info, setInfo] = useState(null); + const [courses, setCourses] = useState([]); + const [jobs, setJobs] = useState([]); + const [denied, setDenied] = useState(null); + const [kind, setKind] = useState("packt-video"); + const [path, setPath] = useState(""); + const [title, setTitle] = useState(""); + const [courseId, setCourseId] = useState(""); + const [attachTo, setAttachTo] = useState(""); + const [transcribe, setTranscribe] = useState(false); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + + const refresh = useCallback(async () => { + try { + setJobs((await api("/api/import/jobs")).sort((a, b) => b.createdAt.localeCompare(a.createdAt))); + setCourses(await api("/api/import/courses")); + } catch (e: any) { + if (e.status === 401 || e.status === 403) setDenied(e.message); + } + }, []); + + useEffect(() => { + api("/api/import/info").then(setInfo).catch((e) => { if (e.status === 401 || e.status === 403) setDenied(e.message); }); + void refresh(); + const t = setInterval(refresh, 2000); + return () => clearInterval(t); + }, [refresh]); + + const spec = KINDS.find((k) => k.value === kind)!; + const needsCourse = kind === "code-repo" || kind === "pdf-deck"; + const course = courses.find((c) => c.id === courseId); + + async function submit(e: FormEvent) { + e.preventDefault(); + setError(null); + setBusy(true); + try { + await api("/api/import/jobs", { + method: "POST", + body: JSON.stringify({ kind, path, options: { title, courseId: needsCourse ? courseId : undefined, attachTo: kind === "pdf-deck" ? attachTo : undefined, transcribe: transcribe || undefined } }), + }); + setPath(""); setTitle(""); + await refresh(); + } catch (err: any) { + setError(err.message); + } finally { + setBusy(false); + } + } + + if (denied) { + return

    Import

    {denied === "Sign in required" ? "Sign in as the owner to import courses." : "Only the owner can import courses."}

    ; + } + + return ( +
    +

    Import

    +

    Turn material you own into a course. Originals are only read; the new course is restricted until you change its visibility.

    + {info && !info.coursesDirServed ? ( +

    The server is not scanning {info.coursesDir}. Add it to STC_COURSE_DIRS to see imported courses.

    + ) : null} + +
    + ({ value: k.value, label: k.label }))} value={kind} onChange={(v) => setKind(v as Kind)} /> + + {!needsCourse ? ( + + ) : ( +
    + + {kind === "pdf-deck" ? ( + + ) : null} +
    + )} + {info?.transcribe && (kind === "packt-video" || kind === "video-folder") ? ( + + ) : null} +
    + Start import + {error ? {error} : null} +
    + + +

    Jobs

    + {jobs.length === 0 ?

    No imports yet.

    :
      {jobs.map((j) => )}
    } +
    + ); +} diff --git a/packages/feature-import/src/web/import.css b/packages/feature-import/src/web/import.css new file mode 100644 index 0000000..6dfc535 --- /dev/null +++ b/packages/feature-import/src/web/import.css @@ -0,0 +1,37 @@ +.imp { max-width: 880px; margin: 0 auto; padding-bottom: 80px; } +.imp-h { font: 600 24px/1.2 var(--sans); color: var(--head); margin: 0 0 4px; } +.imp-sub { margin: 0 0 28px; color: var(--muted); font-size: 13.5px; } +.imp-sec { font: 600 14px var(--sans); color: var(--muted); margin: 34px 0 10px; } +.imp-note { margin: 0 0 18px; padding: 10px 14px; border: 1px solid var(--line2); border-radius: 8px; background: var(--side); font-size: 13px; color: var(--muted); } +.imp-note code, .imp-hint code { font-size: 12px; color: var(--text); } +.imp-form { display: grid; gap: 16px; padding: 20px; border: 1px solid var(--line); border-radius: 10px; background: var(--side); } +.imp-field { display: grid; gap: 6px; font-size: 12.5px; color: var(--muted); } +.imp-field input[type="text"], .imp-field select { height: 36px; padding: 0 12px; border: 1px solid var(--line2); border-radius: 8px; background: var(--ground); color: var(--text); font-size: 13.5px; width: 100%; } +.imp-field input.mono { font-family: var(--mono); font-size: 12.5px; } +.imp-field input::placeholder { color: var(--faint); } +.imp-hint { font-size: 12px; color: var(--faint); } +.imp-row { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } +.imp-check { display: flex; align-items: center; gap: 8px; font-size: 13px; color: var(--text); } +.imp-check input { accent-color: var(--accent); } +.imp-actions { display: flex; align-items: center; gap: 14px; } +.imp-err { color: var(--bad); font-size: 13px; } +.imp-jobs { list-style: none; margin: 0; padding: 0; border-top: 1px solid var(--line); } +.imp-job { display: grid; gap: 8px; padding: 14px 2px; border-bottom: 1px solid var(--line); font-size: 13.5px; } +.imp-job-top { display: flex; align-items: baseline; gap: 12px; } +.imp-job-kind { font-weight: 600; color: var(--head); } +.imp-job-src { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--faint); font-size: 12.5px; } +.imp-status { font-size: 12px; font-weight: 500; color: var(--muted); } +.imp-status[data-s="done"] { color: var(--ok); } +.imp-status[data-s="failed"] { color: var(--bad); } +.imp-status[data-s="running"] { color: var(--accent); } +.imp-job-bar { display: flex; align-items: center; gap: 12px; } +.imp-job-msg { font-size: 12.5px; color: var(--faint); } +.imp-job-bar .imp-job-msg { flex: none; width: 200px; text-align: right; } +.imp-job details { font-size: 12.5px; color: var(--muted); } +.imp-job summary { cursor: pointer; } +.imp-job ul.warn { margin: 6px 0 0; padding-left: 18px; } +.imp-covers { display: flex; flex-wrap: wrap; gap: 10px; align-items: flex-end; } +.imp-cand { display: grid; gap: 6px; justify-items: start; } +.imp-cand img { width: 160px; aspect-ratio: 16 / 9; object-fit: cover; border-radius: 6px; border: 1px solid var(--line2); background: var(--raised); } +.imp-cand[aria-current="true"] img { border-color: var(--accent); } +.imp-empty { color: var(--faint); font-size: 13px; padding: 14px 2px; } diff --git a/packages/feature-import/test/code-repo.test.ts b/packages/feature-import/test/code-repo.test.ts new file mode 100644 index 0000000..b0112bd --- /dev/null +++ b/packages/feature-import/test/code-repo.test.ts @@ -0,0 +1,57 @@ +import { expect, test } from "bun:test"; +import { mkdir, readdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { folderName, folderNumber, importCodeRepo } from "../src/jobs/code-repo"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { makeEnv } from "./harness"; +import { makePacktZip, makeRepoZip, tmp } from "./fixtures"; + +test("folder names split into number and name", () => { + expect(folderNumber("S08 - Containerizing Applications")).toBe(8); + expect(folderName("S08 - Containerizing Applications")).toBe("Containerizing Applications"); + expect(folderNumber("Section 3: Images")).toBe(3); + expect(folderNumber("Bonus Extras")).toBeNull(); +}); + +async function target() { + const env = await makeEnv(); + const r = await importPacktVideo(env.deps, { path: await makePacktZip(await tmp()), options: { title: "Docker Test" } }, env.jc); + return { env, id: r.courseId }; +} + +test("zip repo: folders attach to modules by number, PDFs become deck and attachment", async () => { + const { env, id } = await target(); + const zip = await makeRepoZip(await tmp()); + const res = await importCodeRepo(env.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, env.jc); + const c = env.deps.ctx.courses.get(id)!; + expect(c.status).toBe("ok"); + expect(c.manifest!.modules[0]!.code).toEqual({ dir: "code/S01 - Intro to Docker" }); + expect(c.manifest!.modules[1]!.code).toEqual({ dir: "code/S02 - Images (Sample)" }); + expect(env.deps.ctx.courses.resolvePath(id, "code/S02 - Images (Sample)/app/main.py")).toBeTruthy(); + expect(env.deps.ctx.courses.resolvePath(id, "code/S02 - Images (Sample)/Ünïcode notes.txt")).toBeTruthy(); + expect(c.manifest!.decks).toEqual([{ id: "d-slides-intro", file: "media/decks/Slides_Intro.pdf", title: "Slides Intro", attachTo: "m1" }]); + expect(c.manifest!.attachments).toEqual([{ file: "media/files/Docker_CheatSheet.pdf", title: "Docker CheatSheet", module: "m1" }]); + expect(await readdir(join(env.dataDir, "cache/decks", id, "d-slides-intro"))).toContain("p-003.jpg"); + expect(await readdir(join(c.root, "code/S01 - Intro to Docker"))).toEqual([]); // PDFs are not duplicated into code + expect(res.warnings).toEqual(['folder "Bonus Extras" matches no module and was skipped']); +}); + +test("an unnumbered folder matches a module by name; symlinks are not copied; the input is untouched", async () => { + const { env, id } = await target(); + const top = join(await tmp(), "repo"); + await mkdir(join(top, "Images Sample"), { recursive: true }); + await writeFile(join(top, "Images Sample", "a.txt"), "a"); + await symlink("/etc/passwd", join(top, "Images Sample", "link")); + const res = await importCodeRepo(env.deps, { path: top, options: { courseId: id } }, env.jc); + expect(res.warnings).toEqual([]); + const c = env.deps.ctx.courses.get(id)!; + expect(c.manifest!.modules[1]!.code).toEqual({ dir: "code/Images Sample" }); + expect(await readdir(join(c.root, "code/Images Sample"))).toEqual(["a.txt"]); + expect(await readdir(join(top, "Images Sample"))).toContain("link"); +}); + +test("rejects unmanaged courses and bad git URLs without running git", async () => { + const { env, id } = await target(); + await expect(importCodeRepo(env.deps, { path: "/x", options: { courseId: "sample" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importCodeRepo(env.deps, { path: "--upload-pack=touch /tmp/pwn", options: { courseId: id } }, env.jc)).rejects.toThrow(); +}); diff --git a/packages/feature-import/test/fixtures.ts b/packages/feature-import/test/fixtures.ts new file mode 100644 index 0000000..6e78abf --- /dev/null +++ b/packages/feature-import/test/fixtures.ts @@ -0,0 +1,125 @@ +// Tiny synthetic inputs for the importer tests. Nothing here is real course material. +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { run } from "../src/util/exec"; + +// PyMuPDF is needed for PDF tests: set STC_PYTHON, or this machine's venv is used when present. +const VENV_PY = "/home/user/.venvs/pdf/bin/python"; +export const PYTHON = process.env.STC_PYTHON ?? (existsSync(VENV_PY) ? VENV_PY : "python3"); + +export async function tmp(prefix = "stc-imp-"): Promise { + return mkdtemp(join(tmpdir(), prefix)); +} + +export async function makeVideo(path: string, seconds = 2): Promise { + await mkdir(dirname(path), { recursive: true }); + await run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", `testsrc=duration=${seconds}:size=160x90:rate=10`, "-f", "lavfi", "-i", `sine=frequency=440:duration=${seconds}`, + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest", path]); +} + +/** Minimal valid PDF with one line of text per page. */ +export function pdfBytes(pages: string[]): Uint8Array { + const objs: string[] = []; + const kids = pages.map((_, i) => `${4 + i * 2} 0 R`).join(" "); + objs.push("<< /Type /Catalog /Pages 2 0 R >>"); + objs.push(`<< /Type /Pages /Kids [${kids}] /Count ${pages.length} >>`); + objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + pages.forEach((text, i) => { + const content = `BT /F1 24 Tf 40 150 Td (${text.replace(/[()\\]/g, "")}) Tj ET`; + objs.push(`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${5 + i * 2} 0 R >>`); + objs.push(`<< /Length ${content.length} >>\nstream\n${content}\nendstream`); + }); + let out = "%PDF-1.4\n"; + const offsets: number[] = []; + objs.forEach((o, i) => { offsets.push(out.length); out += `${i + 1} 0 obj\n${o}\nendobj\n`; }); + const xref = out.length; + out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n` + offsets.map((o) => `${String(o).padStart(10, "0")} 00000 n \n`).join(""); + out += `trailer\n<< /Size ${objs.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return new TextEncoder().encode(out); +} + +export async function makePdf(path: string, pages = ["First slide alpha", "Second slide beta"]): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, pdfBytes(pages)); +} + +export async function zipDir(srcDir: string, zipPath: string): Promise { + await run([PYTHON, "-I", "-c", + "import sys,os,zipfile\nsrc,out=sys.argv[1:3]\nz=zipfile.ZipFile(out,'w',zipfile.ZIP_STORED)\nfor r,d,f in os.walk(src):\n for n in f:\n p=os.path.join(r,n); z.write(p, os.path.relpath(p,src))\nz.close()", + srcDir, zipPath]); +} + +const li = (t: string) => `
  • ${t}
    • `; +const clipHtml = (s: number, m: number, title: string) => ` +
  • + +

    ${s}.${m} ${title}

    + In this video, ${title}, you'll learn it. +
    Watch Video
    + Download +
    • `; + +export const PACKT_SECTIONS = [ + { title: "Intro to Docker", clips: ["Course Introduction", "Docker & Friends"] }, + { title: "Images (Sample)", clips: ["Topic Introduction - Images"] }, +]; + +/** Mimics the real package layout: summary lists without "Video N.M:" prefixes, then chapter blocks. */ +export function packtIndexHtml(opts: { dropChapter?: number } = {}): string { + const summary = PACKT_SECTIONS.map((s, i) => ` +
  • + +

    Section ${i + 1}: ${s.title}

    +

    Video ${i + 1}.1: ${s.clips[0]}

    +
      ${s.clips.map(li).join("\n")}
    +
    Browse Videos
    +
    • `).join(""); + const chapters = PACKT_SECTIONS.map((s, i) => i + 1 === opts.dropChapter ? "" : ` + `).join(""); + return `Fake +

    Fake Docker Course

    +
      ${summary}
    ${chapters}
    `; +} + +export async function makePacktTree(root: string): Promise { + await mkdir(join(root, "package", "videos"), { recursive: true }); + await writeFile(join(root, "package", "index.html"), packtIndexHtml()); + await writeFile(join(root, "Video Package_User Guide.pdf"), "not used"); + await makeVideo(join(root, "package/videos/video1_1.mp4"), 2); + for (const n of ["video1_2", "video2_1"]) await Bun.write(join(root, "package/videos", `${n}.mp4`), Bun.file(join(root, "package/videos/video1_1.mp4"))); +} + +export async function makePacktZip(dir: string): Promise { + const tree = join(dir, "packt-tree"); + await makePacktTree(tree); + const zip = join(dir, "video_zip.zip"); + await zipDir(tree, zip); + return zip; +} + +/** Companion repo with folders that have spaces; S01 has a deck and a cheat sheet, S02 has code. */ +export async function makeRepoTree(root: string): Promise { + const top = join(root, "Course-Repo-main"); + await makePdf(join(top, "S01 - Intro to Docker", "Slides_Intro.pdf"), ["Repo deck one", "Repo deck two", "Repo deck three"]); + await makePdf(join(top, "S01 - Intro to Docker", "Docker_CheatSheet.pdf"), ["cheats"]); + await mkdir(join(top, "S02 - Images (Sample)", "app"), { recursive: true }); + await writeFile(join(top, "S02 - Images (Sample)", "app", "main.py"), "print('hi')\n"); + await writeFile(join(top, "S02 - Images (Sample)", "Ünïcode notes.txt"), "notes\n"); + await mkdir(join(top, "Bonus Extras"), { recursive: true }); + await writeFile(join(top, "Bonus Extras", "x.txt"), "x"); + await writeFile(join(top, "README.md"), "# repo\n"); + return top; +} + +export async function makeRepoZip(dir: string): Promise { + const tree = join(dir, "repo-tree"); + await makeRepoTree(tree); + const zip = join(dir, "repo.zip"); + await zipDir(tree, zip); + return zip; +} diff --git a/packages/feature-import/test/harness.ts b/packages/feature-import/test/harness.ts new file mode 100644 index 0000000..ff22cc2 --- /dev/null +++ b/packages/feature-import/test/harness.ts @@ -0,0 +1,22 @@ +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JobContext, ServerContext } from "@stc/contracts"; +import { createCourseRepository } from "@stc/course-format"; +import type { Deps } from "../src/context"; +import { PYTHON, tmp } from "./fixtures"; + +export interface TestEnv { deps: Deps; dataDir: string; queued: { kind: string; payload: unknown }[]; jc: JobContext; progress: [number, string | undefined][] } + +export async function makeEnv(env: Record = {}): Promise { + const dataDir = await tmp("stc-data-"); + const courses = await createCourseRepository([join(dataDir, "courses")]); + const queued: TestEnv["queued"] = []; + const progress: TestEnv["progress"] = []; + const ctx = { + config: { port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "http://x" }, + courses, + jobs: { enqueue: async (kind: string, payload: unknown) => { queued.push({ kind, payload }); return "j1"; }, get: async () => null, list: async () => [] }, + } as unknown as ServerContext; + const jc: JobContext = { signal: new AbortController().signal, reportProgress: async (f, m) => { progress.push([f, m]); } }; + return { deps: { ctx, env: { STC_IMPORT_ROOTS: tmpdir(), ...env }, tools: { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }, python: PYTHON }, dataDir, queued, jc, progress }; +} diff --git a/packages/feature-import/test/hostile-zip.test.ts b/packages/feature-import/test/hostile-zip.test.ts new file mode 100644 index 0000000..3fabdeb --- /dev/null +++ b/packages/feature-import/test/hostile-zip.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { openSource } from "../src/util/source"; +import { PYTHON, tmp } from "./fixtures"; +import { run } from "../src/util/exec"; + +/** Builds a zip by hand with traversal, absolute and symlink entries. */ +async function hostileZip(path: string) { + await run([PYTHON, "-I", "-c", ` +import sys, zipfile, stat +z = zipfile.ZipFile(sys.argv[1], 'w') +z.writestr('../../evil-up.txt', 'x') +z.writestr('/tmp/stc-evil-abs.txt', 'x') +z.writestr('ok/fine.txt', 'fine') +i = zipfile.ZipInfo('ok/link'); i.create_system = 3; i.external_attr = (stat.S_IFLNK | 0o777) << 16 +z.writestr(i, '/etc') +i2 = zipfile.ZipInfo('ok/link/pwned.txt'); z.writestr(i2, 'x') +z.close()`, path]); +} + +test("hostile zip entries never write outside the extraction dir", async () => { + const base = await tmp("stc-zip-"); + await hostileZip(join(base, "h.zip")); + const src = await openSource(join(base, "h.zip")); + const dest = join(base, "a", "b", "out"); + await src.extract(src.entries, dest).catch(() => {}); // unzip may warn; what matters is where files land + expect(existsSync(join(base, "evil-up.txt"))).toBe(false); + expect(existsSync(join(base, "a", "evil-up.txt"))).toBe(false); + expect(existsSync(join(base, "..", "evil-up.txt"))).toBe(false); + expect(existsSync("/tmp/stc-evil-abs.txt")).toBe(false); + expect(existsSync("/etc/pwned.txt")).toBe(false); + expect(await Bun.file(join(dest, "ok/fine.txt")).text()).toBe("fine"); + const left: string[] = []; + const walk = async (d: string) => { for (const e of await readdir(d, { withFileTypes: true })) { if (e.isSymbolicLink()) left.push(e.name); else if (e.isDirectory()) await walk(join(d, e.name)); } }; + await walk(dest); + expect(left).toEqual([]); // the symlink entry is dropped +}); diff --git a/packages/feature-import/test/media.test.ts b/packages/feature-import/test/media.test.ts new file mode 100644 index 0000000..7704dc3 --- /dev/null +++ b/packages/feature-import/test/media.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { join } from "node:path"; +import { readFile, stat } from "node:fs/promises"; +import { extractFrame, probeDuration, putFile } from "../src/media"; +import { renderPdf } from "../src/pdf"; +import { makePdf, makeVideo, PYTHON, tmp } from "./fixtures"; + +const tools = { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }; + +test("duration and frame extraction on a tiny clip", async () => { + const d = await tmp(); + await makeVideo(join(d, "a.mp4"), 2); + expect(await probeDuration(tools, join(d, "a.mp4"))).toBe(2); + await extractFrame(tools, join(d, "a.mp4"), join(d, "out", "f.jpg")); + expect((await stat(join(d, "out", "f.jpg"))).size).toBeGreaterThan(100); +}); + +test("putFile copies and links without touching the original", async () => { + const d = await tmp(); + await makeVideo(join(d, "sp ace", "ö.mp4"), 1); + const before = await readFile(join(d, "sp ace", "ö.mp4")); + await putFile(join(d, "sp ace", "ö.mp4"), join(d, "c", "x.mp4"), "copy"); + await putFile(join(d, "sp ace", "ö.mp4"), join(d, "l", "x.mp4"), "link"); + expect(Buffer.compare(await readFile(join(d, "c", "x.mp4")), before)).toBe(0); + expect((await stat(join(d, "l", "x.mp4"))).ino).toBe((await stat(join(d, "sp ace", "ö.mp4"))).ino); +}); + +test("renderPdf writes page images and text", async () => { + const d = await tmp(); + await makePdf(join(d, "deck.pdf"), ["Hello alpha", "Second beta"]); + const seen: number[] = []; + const r = await renderPdf(PYTHON, join(d, "deck.pdf"), join(d, "out"), { onPage: (p) => seen.push(p) }); + expect(r.pages).toBe(2); + expect(seen).toEqual([1, 2]); + expect((await stat(join(d, "out", "p-002.jpg"))).size).toBeGreaterThan(100); + const text = JSON.parse(await readFile(join(d, "out", "text.json"), "utf8")); + expect(text[0].text).toContain("Hello alpha"); +}); diff --git a/packages/feature-import/test/packt-video.test.ts b/packages/feature-import/test/packt-video.test.ts new file mode 100644 index 0000000..6ca5ab5 --- /dev/null +++ b/packages/feature-import/test/packt-video.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "bun:test"; +import { readdir, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { makeEnv } from "./harness"; +import { makePacktZip, tmp } from "./fixtures"; + +test("imports a Packt zip into a restricted course with sections, titles and media", async () => { + const env = await makeEnv(); + const zip = await makePacktZip(await tmp()); + const before = await readFile(zip); + const r = await importPacktVideo(env.deps, { kind: "packt-video", path: zip, options: { title: "Docker Test" } }, env.jc); + expect(r.courseId).toBe("docker-test"); + expect(r.warnings).toEqual([]); + const c = env.deps.ctx.courses.get("docker-test")!; + expect(c.status).toBe("ok"); + expect(c.diagnostics).toEqual([]); + const m = c.manifest!; + expect(m.visibility).toBe("restricted"); + expect(m.source?.kind).toBe("packt-video"); + expect(m.source?.origin).toBe("video_zip.zip"); + expect(m.source?.importedAt).toBeTruthy(); + expect(m.modules.map((x) => x.title)).toEqual(["Intro to Docker", "Images (Sample)"]); + expect(m.modules.map((x) => x.lessons.map((l) => l.title))).toEqual([["Course Introduction", "Docker & Friends"], ["Topic Introduction - Images"]]); + expect(m.modules[0]!.lessons[0]).toMatchObject({ id: "m1-l1", type: "video", file: "media/video1_1.mp4", duration: 2 }); + expect(env.deps.ctx.courses.resolvePath("docker-test", "media/video2_1.mp4")).toBeTruthy(); + expect(m.cover).toBe("media/cover.jpg"); + expect((await readdir(join(env.dataDir, "cache/covers/docker-test"))).sort()).toEqual(["frame-1.jpg", "frame-2.jpg", "frame-3.jpg"]); + expect((await readdir(c.root)).sort()).toEqual(["media", "stcourse.yaml"]); // no extraction leftovers + expect(await readdir(join(env.dataDir, "tmp"))).toEqual([]); // scratch removed + expect(Buffer.compare(await readFile(zip), before)).toBe(0); // original untouched + expect(env.progress.at(-1)![0]).toBe(1); +}); + +test("second import of the same title gets a fresh id; transcribe requested without a URL warns", async () => { + const env = await makeEnv(); + const zip = await makePacktZip(await tmp()); + await importPacktVideo(env.deps, { path: zip, options: { title: "Same" } }, env.jc); + const r = await importPacktVideo(env.deps, { path: zip, options: { title: "Same", transcribe: true } }, env.jc); + expect(r.courseId).toBe("same-2"); + expect(r.warnings.join()).toContain("STC_TRANSCRIBE_URL is not set"); + expect(env.queued).toEqual([]); +}); + +test("a zip that is not a Packt package fails and leaves nothing behind", async () => { + const env = await makeEnv(); + const d = await tmp(); + await Bun.write(join(d, "t", "readme.txt"), "x"); + const { zipDir } = await import("./fixtures"); + await zipDir(join(d, "t"), join(d, "bad.zip")); + await expect(importPacktVideo(env.deps, { path: join(d, "bad.zip") }, env.jc)).rejects.toThrow(/index\.html/); + expect(await stat(join(env.dataDir, "courses")).catch(() => null)).toBeNull(); +}); + +test("transcribe option queues the transcription job when a URL is configured", async () => { + const env = await makeEnv({ STC_TRANSCRIBE_URL: "http://127.0.0.1:1/v1/audio/transcriptions" }); + const zip = await makePacktZip(await tmp()); + const r = await importPacktVideo(env.deps, { path: zip, options: { transcribe: true } }, env.jc); + expect(env.queued).toEqual([{ kind: "import.transcribe", payload: { courseId: r.courseId } }]); +}); diff --git a/packages/feature-import/test/packt.test.ts b/packages/feature-import/test/packt.test.ts new file mode 100644 index 0000000..dbdf306 --- /dev/null +++ b/packages/feature-import/test/packt.test.ts @@ -0,0 +1,28 @@ +import { expect, test } from "bun:test"; +import { parsePacktIndex } from "../src/packt"; +import { packtIndexHtml } from "./fixtures"; + +test("sections, clip titles and video files come from the structure", () => { + const p = parsePacktIndex(packtIndexHtml()); + expect(p.title).toBe("Fake Docker Course"); + expect(p.sections.map((s) => s.title)).toEqual(["Intro to Docker", "Images (Sample)"]); + expect(p.sections[0]!.clips).toEqual([ + { number: 1, title: "Course Introduction", video: "videos/video1_1.mp4" }, + { number: 2, title: "Docker & Friends", video: "videos/video1_2.mp4" }, + ]); + expect(p.sections[1]!.clips[0]!.video).toBe("videos/video2_1.mp4"); + expect(p.warnings).toEqual([]); +}); + +test("a section without a chapter block falls back to the overview list", () => { + const p = parsePacktIndex(packtIndexHtml({ dropChapter: 1 })); + expect(p.sections[0]!.clips.map((c) => [c.title, c.video])).toEqual([ + ["Course Introduction", "videos/video1_1.mp4"], + ["Docker & Friends", "videos/video1_2.mp4"], + ]); + expect(p.warnings.join()).toContain("section 1 has no chapter block"); +}); + +test("not a Packt page is a clear error", () => { + expect(() => parsePacktIndex("

    Hello

    ")).toThrow(/no "Section N/); +}); diff --git a/packages/feature-import/test/roots.test.ts b/packages/feature-import/test/roots.test.ts new file mode 100644 index 0000000..84ffbe5 --- /dev/null +++ b/packages/feature-import/test/roots.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from "bun:test"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { importCodeRepo } from "../src/jobs/code-repo"; +import { importPdfDeck } from "../src/jobs/pdf-deck"; +import { importVideoFolder } from "../src/jobs/video-folder"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { makeEnv } from "./harness"; +import { makePdf, tmp } from "./fixtures"; + +test("every job re-checks its path against the roots", async () => { + const inbox = await tmp("stc-inbox-"); + const outside = await tmp("stc-outside-"); + await makePdf(join(outside, "a.pdf")); + await symlink(outside, join(inbox, "link")); + const env = await makeEnv({ STC_IMPORT_ROOTS: inbox }); + const msg = /outside the allowed import folders/; + await expect(importVideoFolder(env.deps, { path: outside }, env.jc)).rejects.toThrow(msg); + await expect(importVideoFolder(env.deps, { path: join(inbox, "link") }, env.jc)).rejects.toThrow(msg); + await expect(importPacktVideo(env.deps, { path: join(outside, "x.zip") }, env.jc)).rejects.toThrow(msg); + await expect(importPdfDeck(env.deps, { path: join(outside, "a.pdf"), options: {} }, env.jc)).rejects.toThrow(msg); + await expect(importCodeRepo(env.deps, { path: outside, options: { courseId: "x" } }, env.jc)).rejects.toThrow(/created|outside/); + await mkdir(join(inbox, "empty")); + await writeFile(join(inbox, "empty", "n.txt"), ""); + await expect(importVideoFolder(env.deps, { path: join(inbox, "empty") }, env.jc)).rejects.toThrow(/no video files/); // allowed, then fails on content +}); diff --git a/packages/feature-import/test/routes.test.ts b/packages/feature-import/test/routes.test.ts new file mode 100644 index 0000000..3d2f359 --- /dev/null +++ b/packages/feature-import/test/routes.test.ts @@ -0,0 +1,115 @@ +import { expect } from "bun:test"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AuthUser, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue } from "@stc/core"; +import { createCourseRepository } from "@stc/course-format"; +import { forEachDb } from "../../db/test/helpers"; +import { importFeature } from "../src"; +import { mkdir, stat, symlink } from "node:fs/promises"; +import { makeVideo, tmp } from "./fixtures"; + +const owner: AuthUser = { id: "o", email: "o@x", name: "O", roles: ["owner"], groups: [] }; +const learner: AuthUser = { id: "l", email: "l@x", name: "L", roles: ["learner"], groups: [] }; + +async function setup(db: any, env: Record = {}) { + const dataDir = await tmp("stc-data-"); + const courses = await createCourseRepository([join(dataDir, "courses")]); + const ctx = { + config: { port: 0, databaseUrl: "", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "" }, + db, courses, + auth: { userFromRequest: async (req: Request) => ({ owner, learner }[req.headers.get("x-user") ?? ""] ?? null) }, + } as unknown as ServerContext; + const feature = importFeature(ctx, { env: { STC_PYTHON: process.env.STC_PYTHON, STC_IMPORT_ROOTS: tmpdir(), ...env } }); + const queue = createJobQueue(db, feature.jobs!, { pollMs: 10 }); + ctx.jobs = queue; + const app = createApp(ctx, [feature]); + const call = (path: string, init: RequestInit & { as?: string; json?: unknown } = {}) => + app.fetch(new Request(`http://x${path}`, { + method: init.method, headers: { ...(init.as ? { "x-user": init.as } : {}), ...(init.json !== undefined ? { "content-type": "application/json" } : {}) }, + body: init.json !== undefined ? JSON.stringify(init.json) : undefined, + })); + return { ctx, queue, call, dataDir }; +} +const until = async (f: () => Promise) => { for (let i = 0; i < 600; i++) { if (await f()) return; await Bun.sleep(20); } throw new Error("timeout"); }; + +forEachDb("only the owner reaches the import API", async (db) => { + const { call } = await setup(db); + expect((await call("/api/import/info")).status).toBe(401); + expect((await call("/api/import/info", { as: "learner" })).status).toBe(403); + expect((await call("/api/import/jobs", { as: "learner", method: "POST", json: {} })).status).toBe(403); + const r = await call("/api/import/info", { as: "owner" }); + expect(await r.json()).toMatchObject({ transcribe: false, coursesDirServed: true }); +}); + +forEachDb("job creation validates input", async (db) => { + const { call } = await setup(db); + const post = async (json: unknown) => call("/api/import/jobs", { as: "owner", method: "POST", json }); + expect((await post({ kind: "nope", path: "/tmp" })).status).toBe(400); + expect((await post({ kind: "video-folder" })).status).toBe(400); + expect((await post({ kind: "video-folder", path: "/definitely/not/here" })).status).toBe(400); + expect((await post({ kind: "video-folder", path: "relative" })).status).toBe(400); + expect((await post({ kind: "pdf-deck", path: "/tmp", options: { courseId: "x" } })).status).toBe(400); + expect((await post({ kind: "code-repo", path: "/tmp" })).status).toBe(400); + expect((await post({ kind: "transcribe", options: { courseId: "x" } })).status).toBe(409); +}); + +forEachDb("a folder import runs through the real queue, then cover candidates can be chosen", async (db) => { + const { call, queue, dataDir } = await setup(db); + queue.start(); + try { + const src = join(await tmp(), "Queue Videos"); + await makeVideo(join(src, "1 - a.mp4"), 1); + await makeVideo(join(src, "2 - b.mp4"), 1); + const r = await call("/api/import/jobs", { as: "owner", method: "POST", json: { kind: "video-folder", path: src, options: { title: "Queued", bogus: 1 } } }); + expect(r.status).toBe(202); + const { id } = (await r.json()) as { id: string }; + await until(async () => ["done", "failed"].includes((await queue.get(id))!.status)); + const job = (await queue.get(id))!; + expect(job.error).toBeNull(); + expect(job.status).toBe("done"); + expect(job.result).toMatchObject({ courseId: "queued", warnings: [] }); + expect(job.payload).toEqual({ kind: "video-folder", path: src, options: { title: "Queued" } }); + + const list = (await (await call("/api/import/jobs", { as: "owner" })).json()) as any[]; + expect(list.map((j) => j.id)).toContain(id); + const courses = (await (await call("/api/import/courses", { as: "owner" })).json()) as any[]; + expect(courses).toEqual([{ id: "queued", title: "Queued", modules: [{ id: "m1", title: "Videos", lessons: [{ id: "l1", title: "a" }, { id: "l2", title: "b" }] }] }]); + + const covers = (await (await call("/api/import/covers/queued", { as: "owner" })).json()) as any; + expect(covers.current).toBe("media/cover.jpg"); + expect(covers.candidates).toEqual(["frame-1.jpg", "frame-2.jpg"]); + const img = await call("/api/import/covers/queued/frame-2.jpg", { as: "owner" }); + expect(img.headers.get("content-type")).toBe("image/jpeg"); + expect((await call("/api/import/covers/queued/frame-2.jpg")).status).toBe(401); + expect((await call("/api/import/covers/queued/..%2F..%2Fx.jpg", { as: "owner" })).status).toBe(404); + const set = await call("/api/import/covers/queued", { as: "owner", method: "POST", json: { name: "frame-2.jpg" } }); + expect(set.status).toBe(200); + expect((await call("/api/import/covers/queued", { as: "owner", method: "POST", json: { name: "../evil.jpg" } })).status).toBe(400); + expect(await Bun.file(join(dataDir, "courses/queued/media/cover.jpg")).exists()).toBe(true); + } finally { await queue.stop(); } +}); + +forEachDb("paths must stay inside the import roots", async (db) => { + const inbox = await tmp("stc-inbox-"); + const outside = await tmp("stc-outside-"); + await mkdir(join(inbox, "ok")); + await symlink(outside, join(inbox, "escape")); + const { call, ctx } = await setup(db, { STC_IMPORT_ROOTS: inbox }); + const post = (path: string) => call("/api/import/jobs", { as: "owner", method: "POST", json: { kind: "video-folder", path } }); + const bad = await post(outside); + expect(bad.status).toBe(400); + expect(await bad.json()).toEqual({ error: "path is outside the allowed import folders" }); + expect((await post(join(inbox, "..", "x"))).status).toBe(400); + expect((await post(join(inbox, "escape"))).status).toBe(400); // symlink out of the root + expect((await post("/etc")).status).toBe(400); + expect((await post(join(inbox, "ok"))).status).toBe(202); + const info = (await (await call("/api/import/info", { as: "owner" })).json()) as any; + expect(info.importRoots).toEqual([inbox]); + void ctx; +}); + +forEachDb("the default root is created under dataDir", async (db) => { + const { dataDir } = await setup(db, { STC_IMPORT_ROOTS: undefined }); + expect((await stat(join(dataDir, "import-inbox"))).isDirectory()).toBe(true); +}); diff --git a/packages/feature-import/test/transcribe.test.ts b/packages/feature-import/test/transcribe.test.ts new file mode 100644 index 0000000..1273e1d --- /dev/null +++ b/packages/feature-import/test/transcribe.test.ts @@ -0,0 +1,57 @@ +import { afterAll, expect, test } from "bun:test"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { segmentsToMarkdown, transcribeCourse } from "../src/jobs/transcribe"; +import { makeEnv } from "./harness"; +import { makePacktZip, tmp } from "./fixtures"; + +let calls = 0; +let auth: string | null = null; +const server = Bun.serve({ + port: 0, + async fetch(req) { + calls++; + auth = req.headers.get("authorization"); + const form = await req.formData(); + const f = form.get("file") as File; + if (!f || f.size < 100 || form.get("model") !== "whisper-1") return new Response("bad", { status: 400 }); + return Response.json({ text: "hello world", segments: [{ start: 0, end: 2, text: " hello" }, { start: 35, end: 40, text: " world" }] }); + }, +}); +afterAll(() => server.stop(true)); + +test("segments are grouped into timestamped paragraphs", () => { + expect(segmentsToMarkdown("T", [{ start: 0, end: 1, text: "a" }, { start: 31, end: 32, text: "b" }, { start: 33, end: 34, text: "c" }], "")).toBe("# T\n\n[0:00] a\n\n[0:31] b c\n"); +}); + +test("skipped when STC_TRANSCRIBE_URL is unset", async () => { + const env = await makeEnv(); + const r = await transcribeCourse(env.deps, { courseId: "x" }, env.jc); + expect(r).toMatchObject({ skipped: true, transcribed: 0 }); +}); + +test("transcribes video lessons through a Whisper-compatible endpoint", async () => { + const env = await makeEnv({ STC_TRANSCRIBE_URL: `http://127.0.0.1:${server.port}/v1/audio/transcriptions`, STC_TRANSCRIBE_KEY: "k" }); + const imp = await importPacktVideo(env.deps, { path: await makePacktZip(await tmp()), options: {} }, env.jc); + const r = await transcribeCourse(env.deps, { courseId: imp.courseId }, env.jc); + expect(r.warnings).toEqual([]); + expect(r.transcribed).toBe(3); + expect(calls).toBe(3); + expect(auth).toBe("Bearer k"); + const c = env.deps.ctx.courses.get(imp.courseId)!; + expect(c.status).toBe("ok"); + expect(c.manifest!.modules[0]!.lessons[0]!.transcript).toBe("transcripts/m1-l1.md"); + expect(await env.deps.ctx.courses.readLessonMarkdown(imp.courseId, "m1-l1")).toContain("[0:35] world"); + expect(JSON.parse(await readFile(join(env.dataDir, "cache/transcripts", imp.courseId, "m1-l1.json"), "utf8")).text).toBe("hello world"); + // a second run has nothing left to do + expect((await transcribeCourse(env.deps, { courseId: imp.courseId }, env.jc)).transcribed).toBe(0); +}); + +test("an endpoint failure becomes a warning, not a failed job", async () => { + const env = await makeEnv({ STC_TRANSCRIBE_URL: `http://127.0.0.1:1/x` }); + const imp = await importPacktVideo(env.deps, { path: await makePacktZip(await tmp()), options: {} }, env.jc); + const r = await transcribeCourse(env.deps, { courseId: imp.courseId }, env.jc); + expect(r.transcribed).toBe(0); + expect(r.warnings).toHaveLength(3); +}); diff --git a/packages/feature-import/test/util.test.ts b/packages/feature-import/test/util.test.ts new file mode 100644 index 0000000..7f04d61 --- /dev/null +++ b/packages/feature-import/test/util.test.ts @@ -0,0 +1,41 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { decodeEntities, naturalSort, slugify, uniqueId } from "../src/util/text"; +import { run } from "../src/util/exec"; +import { readRawManifest, writeManifest } from "../src/util/manifest"; + +test("slugify folds accents and falls back", () => { + expect(slugify("Mastering Docker Essentials - Hands-on DevOps!")).toBe("mastering-docker-essentials-hands-on-devops"); + expect(slugify("Café Ünïcode")).toBe("cafe-unicode"); + expect(slugify("日本語")).toBe("course"); +}); + +test("uniqueId appends a counter", () => { + expect(uniqueId("a", (x) => x === "a" || x === "a-2")).toBe("a-3"); + expect(uniqueId("a", () => false)).toBe("a"); +}); + +test("decodeEntities handles named and numeric entities", () => { + expect(decodeEntities("you'll & <b> "x" A")).toBe(`you'll & "x" A`); +}); + +test("naturalSort orders numbers numerically", () => { + expect(naturalSort(["10 b.mp4", "2 a.mp4", "1 z.mp4"])).toEqual(["1 z.mp4", "2 a.mp4", "10 b.mp4"]); +}); + +test("run reports stderr on failure", async () => { + expect(await run(["echo", "hi"])).toContain("hi"); + await expect(run(["sh", "-c", "echo oops >&2; exit 3"])).rejects.toThrow(/oops/); +}); + +test("writeManifest validates and round-trips", async () => { + const dir = await mkdtemp(join(tmpdir(), "stc-m-")); + await mkdir(join(dir, "c")); + const m = { schema: 1, id: "x", title: "T: with colon", visibility: "restricted", modules: [{ id: "m1", title: "S08 - Foo", lessons: [] }] }; + await writeManifest(join(dir, "c"), m); + expect((await readRawManifest(join(dir, "c")) as any).modules[0].title).toBe("S08 - Foo"); + await expect(writeManifest(join(dir, "c"), { ...m, id: "BAD ID" })).rejects.toThrow(/id/); + await writeFile(join(dir, "c", "marker"), ""); +}); diff --git a/packages/feature-import/test/video-folder-pdf.test.ts b/packages/feature-import/test/video-folder-pdf.test.ts new file mode 100644 index 0000000..a319c82 --- /dev/null +++ b/packages/feature-import/test/video-folder-pdf.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from "bun:test"; +import { readdir, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { importPdfDeck } from "../src/jobs/pdf-deck"; +import { importVideoFolder, lessonTitle } from "../src/jobs/video-folder"; +import { makeEnv } from "./harness"; +import { makePdf, makeVideo, tmp } from "./fixtures"; + +test("lessonTitle strips ordering prefixes", () => { + expect(lessonTitle("01 - Intro to Nix.mp4")).toBe("Intro to Nix"); + expect(lessonTitle("2_setup_env.mkv")).toBe("setup env"); + expect(lessonTitle("12.mp4")).toBe("12"); +}); + +async function folderCourse() { + const env = await makeEnv(); + const dir = await tmp(); + const src = join(dir, "My Homelab Videos"); + await makeVideo(join(src, "10 - Last.mp4"), 1); + await makeVideo(join(src, "2 - Second Clip.mp4"), 1); + await makeVideo(join(src, "1 - Ünïcode first.mp4"), 1); + const before = (await readdir(src)).sort(); + const r = await importVideoFolder(env.deps, { kind: "video-folder", path: src }, env.jc); + return { env, src, before, r }; +} + +test("a folder of videos becomes one lesson per file in natural order", async () => { + const { env, src, before, r } = await folderCourse(); + expect(r.courseId).toBe("my-homelab-videos"); + const m = env.deps.ctx.courses.get(r.courseId)!; + expect(m.status).toBe("ok"); + expect(m.manifest!.modules[0]!.lessons.map((l) => l.title)).toEqual(["Ünïcode first", "Second Clip", "Last"]); + expect(m.manifest!.visibility).toBe("restricted"); + expect(m.manifest!.source?.kind).toBe("video-folder"); + expect(env.deps.ctx.courses.resolvePath(r.courseId, "media/1 - Ünïcode first.mp4")).toBeTruthy(); + expect((await readdir(src)).sort()).toEqual(before); +}); + +test("a deck attaches to a lesson, builds page images and text, and leaves the PDF original alone", async () => { + const { env, r } = await folderCourse(); + const d = await tmp(); + await makePdf(join(d, "Slides Deck.pdf"), ["Alpha slide", "Beta slide", "Gamma slide"]); + const before = await readFile(join(d, "Slides Deck.pdf")); + const res = await importPdfDeck(env.deps, { kind: "pdf-deck", path: join(d, "Slides Deck.pdf"), options: { courseId: r.courseId, attachTo: "l2", title: "My deck" } }, env.jc); + expect(res.warnings).toEqual([]); + const c = env.deps.ctx.courses.get(r.courseId)!; + expect(c.status).toBe("ok"); + expect(c.manifest!.decks).toEqual([{ id: "d-slides-deck", file: "media/decks/Slides Deck.pdf", title: "My deck", attachTo: "l2" }]); + const cache = join(env.dataDir, "cache/decks", r.courseId, "d-slides-deck"); + expect((await readdir(cache)).sort()).toEqual(["p-001.jpg", "p-002.jpg", "p-003.jpg", "text.json"]); + expect(JSON.parse(await readFile(join(cache, "text.json"), "utf8"))[2].text).toContain("Gamma slide"); + expect((await stat(join(env.dataDir, "cache/covers", r.courseId, "deck-d-slides-deck.jpg"))).size).toBeGreaterThan(100); + expect(Buffer.compare(await readFile(join(d, "Slides Deck.pdf")), before)).toBe(0); +}); + +test("deck attach validates the target and refuses courses the importer did not create", async () => { + const { env, r } = await folderCourse(); + const d = await tmp(); + await makePdf(join(d, "a.pdf")); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "nope" } }, env.jc)).rejects.toThrow(/not a lesson or module/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId } }, env.jc)).rejects.toThrow(/attachTo is required/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "sample", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "../x", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importPdfDeck(env.deps, { path: join(d, "missing.pdf"), options: {} }, env.jc)).rejects.toThrow(/not a PDF|outside the allowed/); +}); + +test("without PyMuPDF the deck is still attached, with a warning", async () => { + const { env, r } = await folderCourse(); + env.deps.python = "/nonexistent/python"; + const d = await tmp(); + await makePdf(join(d, "a.pdf")); + const res = await importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "m1" } }, env.jc); + expect(res.warnings.join()).toContain("page images and text could not be built"); + expect(env.deps.ctx.courses.get(r.courseId)!.manifest!.decks).toHaveLength(1); +});