feat(import): admin UI at /admin/import, dev server, extraction cleanup

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/feature-import/dev/serve.ts

@@ -0,0 +1,23 @@
+// Dev server for the import UI while real sign-in (learner package) is not merged: everyone is the owner.
+// Usage: STC_PORT=3462 STC_DATA_DIR=/tmp/stc-import-data STC_PYTHON=… bun packages/feature-import/dev/serve.ts
+import { resolve } from "node:path";
+import type { AuthService, ServerContext } from "@stc/contracts";
+import { createApp, createJobQueue, loadConfig } from "@stc/core";
+import { createCourseRepository } from "@stc/course-format";
+import { connect, migrate } from "@stc/db";
+import indexHtml from "../../../apps/web/index.html";
+import { importFeature } from "../src";
+
+const config = loadConfig();
+config.courseDirs.push(resolve(config.dataDir, "courses"));
+const db = await connect(config.databaseUrl);
+await migrate(db);
+const auth: AuthService = { userFromRequest: async () => ({ id: "dev", email: "dev@localhost", name: "Dev owner", roles: ["owner"], groups: [] }) };
+const ctx = { config, db, courses: await createCourseRepository(config.courseDirs), auth } as ServerContext;
+const feature = importFeature(ctx);
+const jobs = createJobQueue(db, feature.jobs!);
+ctx.jobs = jobs;
+const app = createApp(ctx, [feature]);
+jobs.start();
+const server = Bun.serve({ port: config.port, routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, fetch: (req) => app.fetch(req) });
+console.log(`import dev server on ${server.url}`);

packages/feature-import/src/jobs/packt-video.ts

@@ -1,4 +1,4 @@
-import { mkdir, rename } from "node:fs/promises";
+import { mkdir, rename, rm } from "node:fs/promises";
 import { basename, dirname, join, posix } from "node:path";
 import type { ImportResult, JobContext } from "@stc/contracts";
 import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context";
@@ -46,7 +46,7 @@ export async function importPacktVideo(d: Deps, payload: unknown, jc: JobContext
         used.add(entry);
       }
       await jc.reportProgress(0.05 + 0.65 * (done / total), `Copying section ${sec.number}: ${sec.title}`);
-      const tmp = join(stage, "x");
+      const tmp = join(stage, ".extract");
       await src.extract(entries.map((e) => e.entry), tmp, jc.signal);
       const lessons: any[] = [];
       for (const { clip, entry } of entries) {
@@ -58,6 +58,7 @@ export async function importPacktVideo(d: Deps, payload: unknown, jc: JobContext
         lessons.push({ id: `${mid}-l${clip.number}`, title: clip.title, type: "video", file: rel, ...(duration ? { duration } : {}) });
         done++;
       }
+      await rm(tmp, { recursive: true, force: true });
       modules.push({ id: mid, title: sec.title.replace(/^Section\s+\d+\s*[:.-]\s*/i, ""), lessons });
     }
     const unused = src.entries.filter((e) => /\.(mp4|webm|m4v|mov)$/i.test(e) && !used.has(e));

packages/feature-import/src/web/ImportPage.tsx

@@ -1 +1,183 @@
-export default function ImportPage() { return <p>Import</p>; }
+import { useCallback, useEffect, useState, type FormEvent } from "react";
+import { Pill, ProgressBar, SegmentedToggle } from "@stc/ui";
+import "./import.css";
+
+type Kind = "packt-video" | "code-repo" | "pdf-deck" | "video-folder";
+interface Job { id: string; kind: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; payload: any; result: any; error: string | null; createdAt: string }
+interface CourseInfo { id: string; title: string; modules: { id: string; title: string; lessons: { id: string; title: string }[] }[] }
+interface Info { transcribe: boolean; coursesDir: string; coursesDirServed: boolean }
+
+const KINDS: { value: Kind; label: string; hint: string; placeholder: string }[] = [
+  { value: "packt-video", label: "Packt video zip", hint: "A video package whose index.html lists the sections and clips.", placeholder: "/data/downloads/video_zip.zip" },
+  { value: "code-repo", label: "Code repository", hint: "A zip, a folder or a git URL. Folders such as S08 - Name attach to the matching section.", placeholder: "/data/downloads/course-main.zip or https://github.com/…" },
+  { value: "pdf-deck", label: "Slide PDF", hint: "Becomes a deck on a lesson or section. Its text is extracted for search.", placeholder: "/data/downloads/slides.pdf" },
+  { value: "video-folder", label: "Folder of videos", hint: "One lesson per video file, in natural file-name order.", placeholder: "/mnt/videos/my-course" },
+];
+const LABEL: Record<string, string> = { "import.packt-video": "Packt video", "import.code-repo": "Code repository", "import.pdf-deck": "Slide PDF", "import.video-folder": "Video folder", "import.transcribe": "Transcription" };
+
+async function api<T>(path: string, init?: RequestInit): Promise<T> {
+  const r = await fetch(path, { ...init, headers: { "content-type": "application/json", ...init?.headers } });
+  const body = await r.json().catch(() => ({}));
+  if (!r.ok) throw Object.assign(new Error(body.error ?? `HTTP ${r.status}`), { status: r.status });
+  return body as T;
+}
+
+function CoverPicker({ courseId }: { courseId: string }) {
+  const [data, setData] = useState<{ current: string | null; candidates: string[] } | null>(null);
+  const [version, setVersion] = useState(0);
+  useEffect(() => { api<typeof data>(`/api/import/covers/${courseId}`).then(setData).catch(() => setData(null)); }, [courseId, version]);
+  if (!data || data.candidates.length === 0) return null;
+  return (
+    <details>
+      <summary>Cover candidates</summary>
+      <div className="imp-covers">
+        {data.candidates.map((name) => (
+          <div className="imp-cand" key={name}>
+            <img src={`/api/import/covers/${courseId}/${name}`} alt={`Cover candidate ${name}`} loading="lazy" />
+            <Pill onClick={async () => { await api(`/api/import/covers/${courseId}`, { method: "POST", body: JSON.stringify({ name }) }); setVersion((v) => v + 1); }}>Use as cover</Pill>
+          </div>
+        ))}
+      </div>
+    </details>
+  );
+}
+
+function JobRow({ job }: { job: Job }) {
+  const src = job.payload?.path ? String(job.payload.path).split("/").filter(Boolean).pop() : job.payload?.courseId;
+  const result = job.result;
+  const warnings: string[] = result?.warnings ?? [];
+  return (
+    <li className="imp-job">
+      <div className="imp-job-top">
+        <span className="imp-job-kind">{LABEL[job.kind] ?? job.kind}</span>
+        <span className="imp-job-src" title={job.payload?.path}>{src}</span>
+        <span className="imp-status" data-s={job.status}>{job.status}</span>
+      </div>
+      <div className="imp-job-bar">
+        <ProgressBar value={job.status === "done" ? 1 : job.progress} label={`${LABEL[job.kind] ?? job.kind} progress`} />
+        <span className="imp-job-msg">{job.message}</span>
+      </div>
+      {job.error ? <div className="imp-err" role="alert">{job.error}</div> : null}
+      {job.status === "done" && result?.skipped ? <div className="imp-job-msg">Skipped: {result.reason}</div> : null}
+      {job.status === "done" && result?.courseId && !result.skipped ? (
+        <div className="imp-job-msg">Course <a href={`/c/${result.courseId}`}>{result.courseId}</a>, restricted to the owner until you change its visibility.</div>
+      ) : null}
+      {warnings.length ? (
+        <details><summary>{warnings.length} warning{warnings.length > 1 ? "s" : ""}</summary><ul className="warn">{warnings.map((w, i) => <li key={i}>{w}</li>)}</ul></details>
+      ) : null}
+      {job.status === "done" && job.kind !== "import.transcribe" && result?.courseId ? <CoverPicker courseId={result.courseId} /> : null}
+    </li>
+  );
+}
+
+export default function ImportPage() {
+  const [info, setInfo] = useState<Info | null>(null);
+  const [courses, setCourses] = useState<CourseInfo[]>([]);
+  const [jobs, setJobs] = useState<Job[]>([]);
+  const [denied, setDenied] = useState<string | null>(null);
+  const [kind, setKind] = useState<Kind>("packt-video");
+  const [path, setPath] = useState("");
+  const [title, setTitle] = useState("");
+  const [courseId, setCourseId] = useState("");
+  const [attachTo, setAttachTo] = useState("");
+  const [transcribe, setTranscribe] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [busy, setBusy] = useState(false);
+
+  const refresh = useCallback(async () => {
+    try {
+      setJobs((await api<Job[]>("/api/import/jobs")).sort((a, b) => b.createdAt.localeCompare(a.createdAt)));
+      setCourses(await api<CourseInfo[]>("/api/import/courses"));
+    } catch (e: any) {
+      if (e.status === 401 || e.status === 403) setDenied(e.message);
+    }
+  }, []);
+
+  useEffect(() => {
+    api<Info>("/api/import/info").then(setInfo).catch((e) => { if (e.status === 401 || e.status === 403) setDenied(e.message); });
+    void refresh();
+    const t = setInterval(refresh, 2000);
+    return () => clearInterval(t);
+  }, [refresh]);
+
+  const spec = KINDS.find((k) => k.value === kind)!;
+  const needsCourse = kind === "code-repo" || kind === "pdf-deck";
+  const course = courses.find((c) => c.id === courseId);
+
+  async function submit(e: FormEvent) {
+    e.preventDefault();
+    setError(null);
+    setBusy(true);
+    try {
+      await api("/api/import/jobs", {
+        method: "POST",
+        body: JSON.stringify({ kind, path, options: { title, courseId: needsCourse ? courseId : undefined, attachTo: kind === "pdf-deck" ? attachTo : undefined, transcribe: transcribe || undefined } }),
+      });
+      setPath(""); setTitle("");
+      await refresh();
+    } catch (err: any) {
+      setError(err.message);
+    } finally {
+      setBusy(false);
+    }
+  }
+
+  if (denied) {
+    return <div className="imp"><h1 className="imp-h">Import</h1><p className="imp-sub" role="alert">{denied === "Sign in required" ? "Sign in as the owner to import courses." : "Only the owner can import courses."}</p></div>;
+  }
+
+  return (
+    <div className="imp">
+      <h1 className="imp-h">Import</h1>
+      <p className="imp-sub">Turn material you own into a course. Originals are only read; the new course is restricted until you change its visibility.</p>
+      {info && !info.coursesDirServed ? (
+        <p className="imp-note" role="status">The server is not scanning <code>{info.coursesDir}</code>. Add it to <code>STC_COURSE_DIRS</code> to see imported courses.</p>
+      ) : null}
+
+      <form className="imp-form" onSubmit={submit}>
+        <SegmentedToggle label="Source type" options={KINDS.map((k) => ({ value: k.value, label: k.label }))} value={kind} onChange={(v) => setKind(v as Kind)} />
+        <label className="imp-field">Path on the server
+          <input type="text" className="mono" value={path} onChange={(e) => setPath(e.target.value)} placeholder={spec.placeholder} required />
+          <span className="imp-hint">{spec.hint}</span>
+        </label>
+        {!needsCourse ? (
+          <label className="imp-field">Course title (optional)
+            <input type="text" value={title} onChange={(e) => setTitle(e.target.value)} placeholder="Taken from the source when empty" />
+          </label>
+        ) : (
+          <div className="imp-row">
+            <label className="imp-field">Add to course
+              <select value={courseId} onChange={(e) => { setCourseId(e.target.value); setAttachTo(""); }} required>
+                <option value="">Choose an imported course</option>
+                {courses.map((c) => <option key={c.id} value={c.id}>{c.title}</option>)}
+              </select>
+            </label>
+            {kind === "pdf-deck" ? (
+              <label className="imp-field">Attach deck to
+                <select value={attachTo} onChange={(e) => setAttachTo(e.target.value)} required disabled={!course}>
+                  <option value="">Choose a section or lesson</option>
+                  {course?.modules.map((m) => (
+                    <optgroup key={m.id} label={m.title}>
+                      <option value={m.id}>Whole section: {m.title}</option>
+                      {m.lessons.map((l) => <option key={l.id} value={l.id}>{l.title}</option>)}
+                    </optgroup>
+                  ))}
+                </select>
+              </label>
+            ) : null}
+          </div>
+        )}
+        {info?.transcribe && (kind === "packt-video" || kind === "video-folder") ? (
+          <label className="imp-check"><input type="checkbox" checked={transcribe} onChange={(e) => setTranscribe(e.target.checked)} />Transcribe the videos afterwards</label>
+        ) : null}
+        <div className="imp-actions">
+          <Pill variant="primary" type="submit" disabled={busy || !path}>Start import</Pill>
+          {error ? <span className="imp-err" role="alert">{error}</span> : null}
+        </div>
+      </form>
+
+      <h2 className="imp-sec">Jobs</h2>
+      {jobs.length === 0 ? <p className="imp-empty">No imports yet.</p> : <ul className="imp-jobs">{jobs.map((j) => <JobRow key={j.id} job={j} />)}</ul>}
+    </div>
+  );
+}

packages/feature-import/src/web/import.css

@@ -0,0 +1,36 @@
+.imp { max-width: 880px; margin: 0 auto; padding-bottom: 80px; }
+.imp-h { font: 600 24px/1.2 var(--sans); color: var(--head); margin: 0 0 4px; }
+.imp-sub { margin: 0 0 28px; color: var(--muted); font-size: 13.5px; }
+.imp-sec { font: 600 14px var(--sans); color: var(--muted); margin: 34px 0 10px; }
+.imp-note { margin: 0 0 18px; padding: 10px 14px; border: 1px solid var(--line2); border-radius: 8px; background: var(--side); font-size: 13px; color: var(--muted); }
+.imp-note code, .imp-hint code { font-size: 12px; color: var(--text); }
+.imp-form { display: grid; gap: 16px; padding: 20px; border: 1px solid var(--line); border-radius: 10px; background: var(--side); }
+.imp-field { display: grid; gap: 6px; font-size: 12.5px; color: var(--muted); }
+.imp-field input[type="text"], .imp-field select { height: 36px; padding: 0 12px; border: 1px solid var(--line2); border-radius: 8px; background: var(--ground); color: var(--text); font-size: 13.5px; width: 100%; }
+.imp-field input.mono { font-family: var(--mono); font-size: 12.5px; }
+.imp-field input::placeholder { color: var(--faint); }
+.imp-hint { font-size: 12px; color: var(--faint); }
+.imp-row { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
+.imp-check { display: flex; align-items: center; gap: 8px; font-size: 13px; color: var(--text); }
+.imp-check input { accent-color: var(--accent); }
+.imp-actions { display: flex; align-items: center; gap: 14px; }
+.imp-err { color: var(--bad); font-size: 13px; }
+.imp-jobs { list-style: none; margin: 0; padding: 0; border-top: 1px solid var(--line); }
+.imp-job { display: grid; gap: 8px; padding: 14px 2px; border-bottom: 1px solid var(--line); font-size: 13.5px; }
+.imp-job-top { display: flex; align-items: baseline; gap: 12px; }
+.imp-job-kind { font-weight: 600; color: var(--head); }
+.imp-job-src { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--faint); font-size: 12.5px; }
+.imp-status { font-size: 12px; font-weight: 500; color: var(--muted); }
+.imp-status[data-s="done"] { color: var(--ok); }
+.imp-status[data-s="failed"] { color: var(--bad); }
+.imp-status[data-s="running"] { color: var(--accent); }
+.imp-job-bar { display: flex; align-items: center; gap: 12px; }
+.imp-job-msg { font-size: 12.5px; color: var(--faint); }
+.imp-job details { font-size: 12.5px; color: var(--muted); }
+.imp-job summary { cursor: pointer; }
+.imp-job ul.warn { margin: 6px 0 0; padding-left: 18px; }
+.imp-covers { display: flex; flex-wrap: wrap; gap: 10px; align-items: flex-end; }
+.imp-cand { display: grid; gap: 6px; justify-items: start; }
+.imp-cand img { width: 160px; aspect-ratio: 16 / 9; object-fit: cover; border-radius: 6px; border: 1px solid var(--line2); background: var(--raised); }
+.imp-cand[aria-current="true"] img { border-color: var(--accent); }
+.imp-empty { color: var(--faint); font-size: 13px; padding: 14px 2px; }

packages/feature-import/test/packt-video.test.ts

@@ -26,6 +26,7 @@ test("imports a Packt zip into a restricted course with sections, titles and med
   expect(env.deps.ctx.courses.resolvePath("docker-test", "media/video2_1.mp4")).toBeTruthy();
   expect(m.cover).toBe("media/cover.jpg");
   expect((await readdir(join(env.dataDir, "cache/covers/docker-test"))).sort()).toEqual(["frame-1.jpg", "frame-2.jpg", "frame-3.jpg"]);
+  expect((await readdir(c.root)).sort()).toEqual(["media", "stcourse.yaml"]); // no extraction leftovers
   expect(await readdir(join(env.dataDir, "tmp"))).toEqual([]); // scratch removed
   expect(Buffer.compare(await readFile(zip), before)).toBe(0); // original untouched
   expect(env.progress.at(-1)![0]).toBe(1);