feat(import): Packt parser and packt-video job
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
34f27d05c9
commit
07e2223c8a
@@ -0,0 +1,102 @@
|
||||
import { mkdir, readdir, rename, stat } from "node:fs/promises";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import type { ServerContext } from "@stc/contracts";
|
||||
import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest";
|
||||
import { slugify, uniqueId } from "./util/text";
|
||||
import { extractFrame, type Tools } from "./media";
|
||||
|
||||
export interface ImportEnv { [k: string]: string | undefined }
|
||||
export interface ImportOptions { env?: ImportEnv }
|
||||
|
||||
export interface Deps {
|
||||
ctx: ServerContext;
|
||||
env: ImportEnv;
|
||||
tools: Tools;
|
||||
python: string;
|
||||
}
|
||||
|
||||
export const coursesDir = (d: Deps) => resolve(d.ctx.config.dataDir, "courses");
|
||||
export const cacheDir = (d: Deps) => resolve(d.ctx.config.dataDir, "cache");
|
||||
export const deckCacheDir = (d: Deps, courseId: string, deckId: string) => join(cacheDir(d), "decks", courseId, deckId);
|
||||
export const coverCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "covers", courseId);
|
||||
export const transcriptCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "transcripts", courseId);
|
||||
|
||||
export const isPlainId = (s: string) => /^[a-z0-9][a-z0-9-]*$/.test(s);
|
||||
|
||||
/** A fresh course id from a title, unique among managed folders and every course the server already knows. */
|
||||
export async function freshCourseId(d: Deps, wanted: string | undefined, title: string): Promise<string> {
|
||||
if (wanted !== undefined) {
|
||||
if (!isPlainId(wanted)) throw new Error(`invalid courseId "${wanted}" (use lowercase letters, digits and dashes)`);
|
||||
if (await exists(join(coursesDir(d), wanted)) || d.ctx.courses.get(wanted)) throw new Error(`a course with id "${wanted}" already exists`);
|
||||
return wanted;
|
||||
}
|
||||
const taken = new Set<string>(d.ctx.courses.list().map((c) => c.id));
|
||||
for (const e of await readdir(coursesDir(d)).catch(() => [] as string[])) taken.add(e);
|
||||
return uniqueId(slugify(title), (x) => taken.has(x));
|
||||
}
|
||||
|
||||
export async function exists(p: string): Promise<boolean> {
|
||||
return stat(p).then(() => true, () => false);
|
||||
}
|
||||
|
||||
/** Scratch folder on the same filesystem as the courses, so finished work is moved in with one rename. */
|
||||
export async function stageDir(d: Deps, label: string): Promise<string> {
|
||||
const dir = resolve(d.ctx.config.dataDir, "tmp", `${label}-${Bun.randomUUIDv7()}`);
|
||||
await mkdir(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
export async function publishCourse(d: Deps, stage: string, id: string): Promise<string> {
|
||||
await mkdir(coursesDir(d), { recursive: true });
|
||||
const dest = join(coursesDir(d), id);
|
||||
await rename(stage, dest);
|
||||
return dest;
|
||||
}
|
||||
|
||||
export interface ManagedCourse { id: string; dir: string; manifest: RawManifest }
|
||||
|
||||
/** An existing course the importer may add to. Only folders directly under <dataDir>/courses qualify: overlays and authored repos are never edited. */
|
||||
export async function managedCourse(d: Deps, courseId: string | undefined): Promise<ManagedCourse> {
|
||||
if (!courseId) throw new Error("options.courseId is required: choose the course to add to");
|
||||
const dir = join(coursesDir(d), courseId);
|
||||
const rel = relative(coursesDir(d), dir);
|
||||
if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) {
|
||||
throw new Error(`course "${courseId}" is not an imported course under ${coursesDir(d)}; the importer only edits courses it created`);
|
||||
}
|
||||
const manifest = await readRawManifest(dir);
|
||||
if (manifest.id !== courseId) throw new Error(`course folder "${courseId}" has manifest id "${manifest.id}"`);
|
||||
return { id: courseId, dir, manifest };
|
||||
}
|
||||
|
||||
/** Reloads the repository and turns load problems into warnings. */
|
||||
export async function finish(d: Deps, id: string, warnings: string[]): Promise<void> {
|
||||
await d.ctx.courses.reload(id);
|
||||
const c = d.ctx.courses.get(id);
|
||||
if (!c) {
|
||||
warnings.push(`course "${id}" is not visible to the server: add ${coursesDir(d)} to STC_COURSE_DIRS`);
|
||||
return;
|
||||
}
|
||||
for (const diag of c.diagnostics) warnings.push(`${diag.level}: ${diag.message}${diag.path ? ` (${diag.path})` : ""}`);
|
||||
}
|
||||
|
||||
export async function saveManifest(dir: string, manifest: RawManifest): Promise<void> {
|
||||
await writeManifest(dir, manifest);
|
||||
}
|
||||
|
||||
export function sourceBlock(kind: string, origin: string) {
|
||||
return { kind, origin, importedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
/** Writes up to `count` frame candidates from evenly spaced clips; returns the candidate file names. Failures are skipped. */
|
||||
export async function frameCandidates(d: Deps, courseId: string, clips: string[], count = 3): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const n = Math.min(count, clips.length);
|
||||
for (let i = 0; i < n; i++) {
|
||||
const name = `frame-${i + 1}.jpg`;
|
||||
try {
|
||||
await extractFrame(d.tools, clips[Math.floor((i * clips.length) / n)]!, join(coverCacheDir(d, courseId), name), 0.2);
|
||||
out.push(name);
|
||||
} catch { /* a bad clip must not fail the import */ }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import type { ImportResult, ImportSource, JobContext } from "@stc/contracts";
|
||||
import { copyFile, mkdir, rm } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { coverCacheDir, exists, type Deps } from "../context";
|
||||
|
||||
export type Opts = Record<string, unknown>;
|
||||
|
||||
export function parsePayload(payload: unknown, kind: ImportSource["kind"]): { path: string; options: Opts } {
|
||||
const p = payload as Partial<ImportSource> | null;
|
||||
if (!p || typeof p.path !== "string" || p.path === "") throw new Error("payload.path is required");
|
||||
if (p.kind !== undefined && p.kind !== kind) throw new Error(`payload.kind "${p.kind}" does not match job kind "${kind}"`);
|
||||
return { path: p.path, options: (p.options ?? {}) as Opts };
|
||||
}
|
||||
|
||||
export const optStr = (o: Opts, k: string): string | undefined => (typeof o[k] === "string" && (o[k] as string).trim() !== "" ? (o[k] as string).trim() : undefined);
|
||||
export const optBool = (o: Opts, k: string): boolean => o[k] === true || o[k] === "true";
|
||||
|
||||
/** Runs a job body with a scratch folder that is always removed, and a result shape shared by all import jobs. */
|
||||
export async function withStage<T>(stage: string, fn: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Copies the first cover candidate into the course (relative path returned) so the library has a cover straight away. */
|
||||
export async function adoptCover(d: Deps, courseId: string, courseDir: string, candidate: string | undefined): Promise<string | undefined> {
|
||||
if (!candidate) return undefined;
|
||||
const src = join(coverCacheDir(d, courseId), candidate);
|
||||
if (!(await exists(src))) return undefined;
|
||||
const rel = "media/cover.jpg";
|
||||
await mkdir(dirname(join(courseDir, rel)), { recursive: true });
|
||||
await copyFile(src, join(courseDir, rel));
|
||||
return rel;
|
||||
}
|
||||
|
||||
export async function maybeQueueTranscription(d: Deps, courseId: string, wanted: boolean, warnings: string[]): Promise<void> {
|
||||
if (!wanted) return;
|
||||
if (!d.env.STC_TRANSCRIBE_URL) {
|
||||
warnings.push("transcription requested but STC_TRANSCRIBE_URL is not set; skipped");
|
||||
return;
|
||||
}
|
||||
await d.ctx.jobs.enqueue("import.transcribe", { courseId });
|
||||
}
|
||||
|
||||
export type Reporter = JobContext["reportProgress"];
|
||||
export type { ImportResult };
|
||||
@@ -0,0 +1,86 @@
|
||||
import { mkdir, rename } from "node:fs/promises";
|
||||
import { basename, dirname, join, posix } from "node:path";
|
||||
import type { ImportResult, JobContext } from "@stc/contracts";
|
||||
import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context";
|
||||
import { probeDuration } from "../media";
|
||||
import { parsePacktIndex } from "../packt";
|
||||
import { openSource } from "../util/source";
|
||||
import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common";
|
||||
|
||||
export async function importPacktVideo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
|
||||
const { path, options } = parsePayload(payload, "packt-video");
|
||||
const warnings: string[] = [];
|
||||
await jc.reportProgress(0.02, "Reading the package");
|
||||
const src = await openSource(path);
|
||||
|
||||
// The player page is the shallowest index.html that lists sections.
|
||||
const candidates = src.entries.filter((e) => /(^|\/)index\.html$/i.test(e)).sort((a, b) => a.split("/").length - b.split("/").length);
|
||||
let indexEntry: string | undefined;
|
||||
let index: ReturnType<typeof parsePacktIndex> | undefined;
|
||||
for (const c of candidates) {
|
||||
try { index = parsePacktIndex(await src.readText(c)); indexEntry = c; break; } catch { /* try the next */ }
|
||||
}
|
||||
if (!index || indexEntry === undefined) throw new Error("no Packt player index.html (with \"Section N: …\" headings) found in the package");
|
||||
warnings.push(...index.warnings);
|
||||
const base = dirname(indexEntry) === "." ? "" : dirname(indexEntry);
|
||||
const at = (rel: string) => posix.normalize(base ? `${base}/${rel}` : rel);
|
||||
|
||||
const title = optStr(options, "title") ?? index.title;
|
||||
const id = await freshCourseId(d, optStr(options, "courseId"), title);
|
||||
const stage = await stageDir(d, "packt");
|
||||
|
||||
return withStage(stage, async () => {
|
||||
const known = new Set(src.entries);
|
||||
const modules: any[] = [];
|
||||
const used = new Set<string>();
|
||||
const clipFiles: string[] = [];
|
||||
const total = index.sections.reduce((n, s) => n + s.clips.length, 0) || 1;
|
||||
let done = 0;
|
||||
for (const sec of index.sections) {
|
||||
const mid = `m${sec.number}`;
|
||||
const entries: { clip: (typeof sec.clips)[number]; entry: string }[] = [];
|
||||
for (const clip of sec.clips) {
|
||||
const entry = at(clip.video);
|
||||
if (!known.has(entry)) { warnings.push(`video for ${sec.number}.${clip.number} is missing from the package: ${clip.video}`); continue; }
|
||||
entries.push({ clip, entry });
|
||||
used.add(entry);
|
||||
}
|
||||
await jc.reportProgress(0.05 + 0.65 * (done / total), `Copying section ${sec.number}: ${sec.title}`);
|
||||
const tmp = join(stage, "x");
|
||||
await src.extract(entries.map((e) => e.entry), tmp, jc.signal);
|
||||
const lessons: any[] = [];
|
||||
for (const { clip, entry } of entries) {
|
||||
const rel = `media/${basename(entry)}`;
|
||||
await mkdir(join(stage, "media"), { recursive: true });
|
||||
await rename(join(tmp, entry), join(stage, rel));
|
||||
clipFiles.push(join(stage, rel));
|
||||
const duration = await probeDuration(d.tools, join(stage, rel));
|
||||
lessons.push({ id: `${mid}-l${clip.number}`, title: clip.title, type: "video", file: rel, ...(duration ? { duration } : {}) });
|
||||
done++;
|
||||
}
|
||||
modules.push({ id: mid, title: sec.title.replace(/^Section\s+\d+\s*[:.-]\s*/i, ""), lessons });
|
||||
}
|
||||
const unused = src.entries.filter((e) => /\.(mp4|webm|m4v|mov)$/i.test(e) && !used.has(e));
|
||||
if (unused.length) warnings.push(`${unused.length} video file(s) in the package are not listed in index.html and were left out`);
|
||||
|
||||
await jc.reportProgress(0.75, "Making cover candidates");
|
||||
const covers = await frameCandidates(d, id, clipFiles);
|
||||
if (covers.length === 0 && clipFiles.length) warnings.push("could not extract cover frames (is ffmpeg available?)");
|
||||
const cover = await adoptCover(d, id, stage, covers[0]);
|
||||
|
||||
const manifest = {
|
||||
schema: 1, id, title,
|
||||
visibility: "restricted",
|
||||
...(cover ? { cover } : {}),
|
||||
source: sourceBlock("packt-video", basename(path)),
|
||||
modules,
|
||||
};
|
||||
await saveManifest(stage, manifest);
|
||||
await jc.reportProgress(0.9, "Publishing the course");
|
||||
const dir = await publishCourse(d, stage, id);
|
||||
await finish(d, id, warnings);
|
||||
await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings);
|
||||
await jc.reportProgress(1, `Imported ${total} clips`);
|
||||
return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings };
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { decodeEntities, stripTags } from "./util/text";
|
||||
|
||||
export interface PacktClip { number: number; title: string; video: string; }
|
||||
export interface PacktSection { number: number; title: string; clips: PacktClip[] }
|
||||
export interface PacktIndex { title: string; sections: PacktSection[]; warnings: string[] }
|
||||
|
||||
/**
|
||||
* Parses the Packt player's index.html structurally:
|
||||
* - the section overview: `<h1>Section N: …</h1>` followed by a `<ul>` of clip titles, in order, usually WITHOUT "Video N.M:" prefixes;
|
||||
* - the per-section chapter blocks (`class="chapter" id="content-N"`) with one `<li>` per clip: `<h1>N.M title</h1>` and `data-video="…"`.
|
||||
* Chapter blocks are authoritative for the video file; the overview list fills in any title or whole section that is missing.
|
||||
*/
|
||||
export function parsePacktIndex(html: string): PacktIndex {
|
||||
const warnings: string[] = [];
|
||||
const intro = /<div[^>]*class="intro"[\s\S]*?<h1[^>]*>([\s\S]*?)<\/h1>/i.exec(html);
|
||||
const titleTag = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
|
||||
const title = stripTags(intro?.[1] ?? titleTag?.[1] ?? "") || "Imported course";
|
||||
|
||||
const chapterStart = html.search(/<div[^>]*class="chapter"/i);
|
||||
const overview = chapterStart >= 0 ? html.slice(0, chapterStart) : html;
|
||||
const chaptersHtml = chapterStart >= 0 ? html.slice(chapterStart) : "";
|
||||
|
||||
// overview: sections in document order
|
||||
const secRe = /<h1[^>]*>\s*Section\s+(\d+)\s*[:.-]\s*([\s\S]*?)<\/h1>/gi;
|
||||
const heads = [...overview.matchAll(secRe)];
|
||||
if (heads.length === 0) throw new Error("not a Packt video package: no \"Section N: …\" headings found in index.html");
|
||||
const overviewTitles = new Map<number, string[]>();
|
||||
const sectionTitles = new Map<number, string>();
|
||||
heads.forEach((h, i) => {
|
||||
const n = Number(h[1]);
|
||||
sectionTitles.set(n, stripTags(h[2]!));
|
||||
const block = overview.slice(h.index! + h[0].length, heads[i + 1]?.index ?? overview.length);
|
||||
const ul = /<ul[^>]*>([\s\S]*?)<\/ul>/i.exec(block);
|
||||
const items = ul ? [...ul[1]!.matchAll(/<li[^>]*>([\s\S]*?)<\/li>/gi)].map((m) => stripTags(m[1]!)).filter(Boolean) : [];
|
||||
const firstP = /<p[^>]*>\s*Video\s+\d+\.\d+\s*:\s*([\s\S]*?)<\/p>/i.exec(block);
|
||||
if (items.length === 0 && firstP) items.push(stripTags(firstP[1]!));
|
||||
overviewTitles.set(n, items);
|
||||
});
|
||||
|
||||
// chapters: clips with their files
|
||||
const chapterClips = new Map<number, PacktClip[]>();
|
||||
const chapRe = /<div[^>]*class="chapter"[^>]*id="content-(\d+)"[^>]*>/gi;
|
||||
const chaps = [...chaptersHtml.matchAll(chapRe)];
|
||||
chaps.forEach((c, i) => {
|
||||
const sec = Number(c[1]);
|
||||
const block = chaptersHtml.slice(c.index! + c[0].length, chaps[i + 1]?.index ?? chaptersHtml.length);
|
||||
const clipRe = /<h1[^>]*>\s*(?:Video\s+)?(\d+)\.(\d+)\s*[:.]?\s*([\s\S]*?)<\/h1>/gi;
|
||||
const hs = [...block.matchAll(clipRe)];
|
||||
const clips: PacktClip[] = [];
|
||||
hs.forEach((h, j) => {
|
||||
const part = block.slice(h.index!, hs[j + 1]?.index ?? block.length);
|
||||
const vid = /data-video="([^"]+)"/i.exec(part)?.[1] ?? /href="([^"]+\.(?:mp4|webm|m4v|mov))"/i.exec(part)?.[1];
|
||||
const num = Number(h[2]);
|
||||
if (!vid) { warnings.push(`clip ${sec}.${num} has no video file in index.html`); return; }
|
||||
clips.push({ number: num, title: stripTags(h[3]!), video: decodeEntities(vid) });
|
||||
});
|
||||
chapterClips.set(sec, clips);
|
||||
});
|
||||
|
||||
const sections: PacktSection[] = [];
|
||||
for (const n of [...sectionTitles.keys()].sort((a, b) => a - b)) {
|
||||
let clips = chapterClips.get(n);
|
||||
const titles = overviewTitles.get(n) ?? [];
|
||||
if (!clips || clips.length === 0) {
|
||||
warnings.push(`section ${n} has no chapter block; clips were taken from the overview list and mapped to videos/video${n}_M.mp4 in order`);
|
||||
clips = titles.map((t, i) => ({ number: i + 1, title: t, video: `videos/video${n}_${i + 1}.mp4` }));
|
||||
} else {
|
||||
clips = clips.map((c, i) => ({ ...c, title: c.title || titles[i] || `Video ${n}.${c.number}` }));
|
||||
if (titles.length && titles.length !== clips.length) warnings.push(`section ${n}: overview lists ${titles.length} clips but the chapter has ${clips.length}`);
|
||||
}
|
||||
sections.push({ number: n, title: sectionTitles.get(n)!, clips });
|
||||
}
|
||||
return { title, sections, warnings };
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { cp, mkdir, readdir, readFile, stat } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { run } from "./exec";
|
||||
|
||||
/** A read-only view of an input: a zip file or an already extracted folder. Nothing in here writes to the input. */
|
||||
export interface Source {
|
||||
kind: "zip" | "dir";
|
||||
entries: string[]; // files only, "/"-separated, relative to the source root
|
||||
readText(entry: string): Promise<string>;
|
||||
/** Copies the given entries to `dest`, keeping their relative paths. */
|
||||
extract(entries: string[], dest: string, signal?: AbortSignal): Promise<void>;
|
||||
}
|
||||
|
||||
const globEscape = (s: string) => s.replace(/[\\[\]*?]/g, (c) => `\\${c}`);
|
||||
|
||||
async function walk(root: string, rel = "", out: string[] = []): Promise<string[]> {
|
||||
for (const e of await readdir(join(root, rel), { withFileTypes: true })) {
|
||||
const r = rel ? `${rel}/${e.name}` : e.name;
|
||||
if (e.isDirectory()) await walk(root, r, out);
|
||||
else if (e.isFile()) out.push(r);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function openSource(path: string): Promise<Source> {
|
||||
const st = await stat(path).catch(() => null);
|
||||
if (!st) throw new Error(`source not found: ${path}`);
|
||||
if (st.isDirectory()) {
|
||||
return {
|
||||
kind: "dir",
|
||||
entries: await walk(path),
|
||||
readText: (e) => readFile(join(path, e), "utf8"),
|
||||
async extract(entries, dest) {
|
||||
for (const e of entries) {
|
||||
await mkdir(dirname(join(dest, e)), { recursive: true });
|
||||
await cp(join(path, e), join(dest, e));
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
const listing = await run(["unzip", "-Z1", path]);
|
||||
const entries = listing.split("\n").filter((l) => l && !l.endsWith("/"));
|
||||
return {
|
||||
kind: "zip",
|
||||
entries,
|
||||
readText: (e) => run(["unzip", "-p", path, globEscape(e)]),
|
||||
async extract(list, dest, signal) {
|
||||
await mkdir(dest, { recursive: true });
|
||||
// unzip treats arguments as patterns, so glob characters in names are escaped; batches keep argv small.
|
||||
for (let i = 0; i < list.length; i += 200) {
|
||||
await run(["unzip", "-q", "-o", path, ...list.slice(i, i + 200).map(globEscape), "-d", dest], { signal });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { join } from "node:path";
|
||||
import type { JobContext, ServerContext } from "@stc/contracts";
|
||||
import { createCourseRepository } from "@stc/course-format";
|
||||
import type { Deps } from "../src/context";
|
||||
import { PYTHON, tmp } from "./fixtures";
|
||||
|
||||
export interface TestEnv { deps: Deps; dataDir: string; queued: { kind: string; payload: unknown }[]; jc: JobContext; progress: [number, string | undefined][] }
|
||||
|
||||
export async function makeEnv(env: Record<string, string | undefined> = {}): Promise<TestEnv> {
|
||||
const dataDir = await tmp("stc-data-");
|
||||
const courses = await createCourseRepository([join(dataDir, "courses")]);
|
||||
const queued: TestEnv["queued"] = [];
|
||||
const progress: TestEnv["progress"] = [];
|
||||
const ctx = {
|
||||
config: { port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "http://x" },
|
||||
courses,
|
||||
jobs: { enqueue: async (kind: string, payload: unknown) => { queued.push({ kind, payload }); return "j1"; }, get: async () => null, list: async () => [] },
|
||||
} as unknown as ServerContext;
|
||||
const jc: JobContext = { signal: new AbortController().signal, reportProgress: async (f, m) => { progress.push([f, m]); } };
|
||||
return { deps: { ctx, env, tools: { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }, python: PYTHON }, dataDir, queued, jc, progress };
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { readdir, readFile, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { importPacktVideo } from "../src/jobs/packt-video";
|
||||
import { makeEnv } from "./harness";
|
||||
import { makePacktZip, tmp } from "./fixtures";
|
||||
|
||||
test("imports a Packt zip into a restricted course with sections, titles and media", async () => {
|
||||
const env = await makeEnv();
|
||||
const zip = await makePacktZip(await tmp());
|
||||
const before = await readFile(zip);
|
||||
const r = await importPacktVideo(env.deps, { kind: "packt-video", path: zip, options: { title: "Docker Test" } }, env.jc);
|
||||
expect(r.courseId).toBe("docker-test");
|
||||
expect(r.warnings).toEqual([]);
|
||||
const c = env.deps.ctx.courses.get("docker-test")!;
|
||||
expect(c.status).toBe("ok");
|
||||
expect(c.diagnostics).toEqual([]);
|
||||
const m = c.manifest!;
|
||||
expect(m.visibility).toBe("restricted");
|
||||
expect(m.source?.kind).toBe("packt-video");
|
||||
expect(m.source?.origin).toBe("video_zip.zip");
|
||||
expect(m.source?.importedAt).toBeTruthy();
|
||||
expect(m.modules.map((x) => x.title)).toEqual(["Intro to Docker", "Images (Sample)"]);
|
||||
expect(m.modules.map((x) => x.lessons.map((l) => l.title))).toEqual([["Course Introduction", "Docker & Friends"], ["Topic Introduction - Images"]]);
|
||||
expect(m.modules[0]!.lessons[0]).toMatchObject({ id: "m1-l1", type: "video", file: "media/video1_1.mp4", duration: 2 });
|
||||
expect(env.deps.ctx.courses.resolvePath("docker-test", "media/video2_1.mp4")).toBeTruthy();
|
||||
expect(m.cover).toBe("media/cover.jpg");
|
||||
expect((await readdir(join(env.dataDir, "cache/covers/docker-test"))).sort()).toEqual(["frame-1.jpg", "frame-2.jpg", "frame-3.jpg"]);
|
||||
expect(await readdir(join(env.dataDir, "tmp"))).toEqual([]); // scratch removed
|
||||
expect(Buffer.compare(await readFile(zip), before)).toBe(0); // original untouched
|
||||
expect(env.progress.at(-1)![0]).toBe(1);
|
||||
});
|
||||
|
||||
test("second import of the same title gets a fresh id; transcribe requested without a URL warns", async () => {
|
||||
const env = await makeEnv();
|
||||
const zip = await makePacktZip(await tmp());
|
||||
await importPacktVideo(env.deps, { path: zip, options: { title: "Same" } }, env.jc);
|
||||
const r = await importPacktVideo(env.deps, { path: zip, options: { title: "Same", transcribe: true } }, env.jc);
|
||||
expect(r.courseId).toBe("same-2");
|
||||
expect(r.warnings.join()).toContain("STC_TRANSCRIBE_URL is not set");
|
||||
expect(env.queued).toEqual([]);
|
||||
});
|
||||
|
||||
test("a zip that is not a Packt package fails and leaves nothing behind", async () => {
|
||||
const env = await makeEnv();
|
||||
const d = await tmp();
|
||||
await Bun.write(join(d, "t", "readme.txt"), "x");
|
||||
const { zipDir } = await import("./fixtures");
|
||||
await zipDir(join(d, "t"), join(d, "bad.zip"));
|
||||
await expect(importPacktVideo(env.deps, { path: join(d, "bad.zip") }, env.jc)).rejects.toThrow(/index\.html/);
|
||||
expect(await stat(join(env.dataDir, "courses")).catch(() => null)).toBeNull();
|
||||
});
|
||||
|
||||
test("transcribe option queues the transcription job when a URL is configured", async () => {
|
||||
const env = await makeEnv({ STC_TRANSCRIBE_URL: "http://127.0.0.1:1/v1/audio/transcriptions" });
|
||||
const zip = await makePacktZip(await tmp());
|
||||
const r = await importPacktVideo(env.deps, { path: zip, options: { transcribe: true } }, env.jc);
|
||||
expect(env.queued).toEqual([{ kind: "import.transcribe", payload: { courseId: r.courseId } }]);
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { parsePacktIndex } from "../src/packt";
|
||||
import { packtIndexHtml } from "./fixtures";
|
||||
|
||||
test("sections, clip titles and video files come from the structure", () => {
|
||||
const p = parsePacktIndex(packtIndexHtml());
|
||||
expect(p.title).toBe("Fake Docker Course");
|
||||
expect(p.sections.map((s) => s.title)).toEqual(["Intro to Docker", "Images (Sample)"]);
|
||||
expect(p.sections[0]!.clips).toEqual([
|
||||
{ number: 1, title: "Course Introduction", video: "videos/video1_1.mp4" },
|
||||
{ number: 2, title: "Docker & Friends", video: "videos/video1_2.mp4" },
|
||||
]);
|
||||
expect(p.sections[1]!.clips[0]!.video).toBe("videos/video2_1.mp4");
|
||||
expect(p.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
test("a section without a chapter block falls back to the overview list", () => {
|
||||
const p = parsePacktIndex(packtIndexHtml({ dropChapter: 1 }));
|
||||
expect(p.sections[0]!.clips.map((c) => [c.title, c.video])).toEqual([
|
||||
["Course Introduction", "videos/video1_1.mp4"],
|
||||
["Docker & Friends", "videos/video1_2.mp4"],
|
||||
]);
|
||||
expect(p.warnings.join()).toContain("section 1 has no chapter block");
|
||||
});
|
||||
|
||||
test("not a Packt page is a clear error", () => {
|
||||
expect(() => parsePacktIndex("<html><h1>Hello</h1></html>")).toThrow(/no "Section N/);
|
||||
});
|
||||
Reference in New Issue
Block a user