diff --git a/packages/feature-learner/src/server/code.ts b/packages/feature-learner/src/server/code.ts new file mode 100644 index 0000000..5d70353 --- /dev/null +++ b/packages/feature-learner/src/server/code.ts @@ -0,0 +1,81 @@ +import { lstatSync, readdirSync, readFileSync, realpathSync } from "node:fs"; +import { join, sep } from "node:path"; +import type { Feature, RequestContext, ServerContext } from "@stc/contracts"; +import { contentDisposition } from "@stc/core"; +import type { CodeListing } from "../shared/types"; +import { courseFor } from "./access"; +import { json, notFound, param } from "./http"; +import { zip } from "./zip"; + +const SKIP = new Set([".git", "node_modules", "__pycache__", ".DS_Store", ".venv", "target", "dist", ".idea", ".vscode"]); +export const MAX_FILES = 2000; +export const MAX_ZIP_BYTES = 200 * 1024 * 1024; + +/** + * Regular files under `dirAbs` (relative paths, sorted). Symlinks are followed only when their target is a + * regular file inside `rootReal`; symlinked directories are skipped, so there are no cycles and no escapes. + */ +export function listFiles(rootReal: string, dirAbs: string, max = MAX_FILES): { files: { path: string; size: number; abs: string }[]; truncated: boolean } { + const files: { path: string; size: number; abs: string }[] = []; + let truncated = false; + const walk = (abs: string, rel: string) => { + let entries; + try { entries = readdirSync(abs, { withFileTypes: true }); } catch { return; } + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const e of entries) { + if (SKIP.has(e.name)) continue; + if (files.length >= max) { truncated = true; return; } + const childAbs = join(abs, e.name), childRel = rel ? `${rel}/${e.name}` : e.name; + if (e.isDirectory()) { walk(childAbs, childRel); continue; } + let real = childAbs; + if (e.isSymbolicLink()) { + try { real = realpathSync(childAbs); } catch { continue; } + if (!(real === rootReal || real.startsWith(rootReal + sep))) continue; + } + try { + const st = lstatSync(real); + if (st.isFile()) files.push({ path: childRel, size: st.size, abs: real }); + } catch { /* vanished */ } + } + }; + walk(dirAbs, ""); + return { files, truncated }; +} + +export function codeRoutes(ctx: ServerContext): Feature["routes"] { + async function resolve(rc: RequestContext) { + const found = await courseFor(ctx, rc.user, param(rc, "courseId")); + if (!found) return null; + const mod = found.manifest.modules.find((m) => m.id === param(rc, "moduleId")); + if (!mod?.code) return null; + const dirAbs = ctx.courses.resolvePath(found.manifest.id, mod.code.dir); + if (!dirAbs) return null; + return { found, mod, code: mod.code, dirAbs, rootReal: realpathSync(found.course.root) }; + } + + return { + "/api/courses/:courseId/code/:moduleId": { + GET: async (_req, rc) => { + const r = await resolve(rc); + if (!r) return notFound(); + const { files, truncated } = listFiles(r.rootReal, r.dirAbs); + const listing: CodeListing = { moduleId: r.mod.id, dir: r.code.dir, command: r.code.command ?? null, files: files.map(({ path, size }) => ({ path, size })), truncated }; + return json(listing); + }, + }, + "/api/courses/:courseId/code/:moduleId/zip": { + GET: async (_req, rc) => { + const r = await resolve(rc); + if (!r) return notFound(); + const { files } = listFiles(r.rootReal, r.dirAbs, 20_000); + const total = files.reduce((s, f) => s + f.size, 0); + if (total > MAX_ZIP_BYTES) return json({ error: "this folder is too large to download as one zip" }, 413); + const folder = r.code.dir.split("/").filter(Boolean).pop() ?? r.mod.id; + const data = zip(files.map((f) => ({ name: `${folder}/${f.path}`, data: readFileSync(f.abs), mtime: lstatSync(f.abs).mtime }))); + return new Response(data, { + headers: { "Content-Type": "application/zip", "Content-Length": String(data.length), "Content-Disposition": contentDisposition(`${folder}.zip`) }, + }); + }, + }, + }; +} diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index 0a857c0..a13c159 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -6,6 +6,7 @@ import { adminRoutes } from "./admin"; import { libraryRoutes } from "./catalog"; import { lessonRoutes } from "./lesson"; import { userDataRoutes } from "./userdata"; +import { codeRoutes } from "./code"; /** * The learner feature: auth, library, reader, notes, progress and exercises. @@ -28,6 +29,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic ...libraryRoutes(ctx), ...lessonRoutes(ctx), ...userDataRoutes(ctx), + ...codeRoutes(ctx), }; // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). for (const methods of Object.values(routes)) { diff --git a/packages/feature-learner/src/server/zip.ts b/packages/feature-learner/src/server/zip.ts new file mode 100644 index 0000000..42f997e --- /dev/null +++ b/packages/feature-learner/src/server/zip.ts @@ -0,0 +1,70 @@ +/** Minimal ZIP writer (PKWARE APPNOTE 6.3): deflate or store per entry, CRC-32, UTF-8 names (flag bit 11). No ZIP64. */ +export interface ZipEntry { name: string; data: Uint8Array; mtime?: Date } + +function dosTime(d: Date): { time: number; date: number } { + const y = Math.max(1980, d.getFullYear()); + return { + time: (d.getHours() << 11) | (d.getMinutes() << 5) | Math.floor(d.getSeconds() / 2), + date: ((y - 1980) << 9) | ((d.getMonth() + 1) << 5) | d.getDate(), + }; +} + +export function zip(entries: ZipEntry[]): Uint8Array { + const enc = new TextEncoder(); + const locals: Uint8Array[] = []; + const centrals: Uint8Array[] = []; + let offset = 0; + for (const e of entries) { + const name = enc.encode(e.name); + const crc = Bun.hash.crc32(e.data) >>> 0; + const deflated = Bun.deflateSync(e.data); + const useDeflate = deflated.length < e.data.length; + const body = useDeflate ? deflated : e.data; + const { time, date } = dosTime(e.mtime ?? new Date()); + const local = new Uint8Array(30 + name.length); + const lv = new DataView(local.buffer); + lv.setUint32(0, 0x04034b50, true); + lv.setUint16(4, 20, true); + lv.setUint16(6, 0x0800, true); // UTF-8 names + lv.setUint16(8, useDeflate ? 8 : 0, true); + lv.setUint16(10, time, true); + lv.setUint16(12, date, true); + lv.setUint32(14, crc, true); + lv.setUint32(18, body.length, true); + lv.setUint32(22, e.data.length, true); + lv.setUint16(26, name.length, true); + lv.setUint16(28, 0, true); + local.set(name, 30); + const central = new Uint8Array(46 + name.length); + const cv = new DataView(central.buffer); + cv.setUint32(0, 0x02014b50, true); + cv.setUint16(4, 0x0314, true); // made by: unix, 2.0 + cv.setUint16(6, 20, true); + cv.setUint16(8, 0x0800, true); + cv.setUint16(10, useDeflate ? 8 : 0, true); + cv.setUint16(12, time, true); + cv.setUint16(14, date, true); + cv.setUint32(16, crc, true); + cv.setUint32(20, body.length, true); + cv.setUint32(24, e.data.length, true); + cv.setUint16(28, name.length, true); + cv.setUint32(38, (0o100644 << 16) >>> 0, true); // external attrs: regular file rw-r--r-- + cv.setUint32(42, offset, true); + central.set(name, 46); + locals.push(local, body); + centrals.push(central); + offset += local.length + body.length; + } + const cdSize = centrals.reduce((s, c) => s + c.length, 0); + const end = new Uint8Array(22); + const ev = new DataView(end.buffer); + ev.setUint32(0, 0x06054b50, true); + ev.setUint16(8, entries.length, true); + ev.setUint16(10, entries.length, true); + ev.setUint32(12, cdSize, true); + ev.setUint32(16, offset, true); + const out = new Uint8Array(offset + cdSize + 22); + let p = 0; + for (const part of [...locals, ...centrals, end]) { out.set(part, p); p += part.length; } + return out; +} diff --git a/packages/feature-learner/test/code.test.ts b/packages/feature-learner/test/code.test.ts new file mode 100644 index 0000000..858a06f --- /dev/null +++ b/packages/feature-learner/test/code.test.ts @@ -0,0 +1,58 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, symlinkSync, writeFileSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { zip } from "../src/server/zip"; +import { forEachDb } from "./helpers"; +import { FIXTURES, makeApp } from "./app"; + +async function unzipList(bytes: Uint8Array): Promise { + const dir = mkdtempSync(join(tmpdir(), "stc-zip-")); + await Bun.write(join(dir, "a.zip"), bytes); + const out = await Bun.$`unzip -l ${join(dir, "a.zip")}`.text(); + await Bun.$`unzip -o -q ${join(dir, "a.zip")} -d ${join(dir, "x")}`; + return out + "\n" + (await Bun.$`cd ${join(dir, "x")} && find . -type f -exec cat {} +`.text()); +} + +test("zip round-trips through unzip with non-ASCII names", async () => { + const big = new TextEncoder().encode("abc".repeat(1000)); + const listing = await unzipList(zip([ + { name: "S08 - Containerizing Applications/Æble notes.txt", data: new TextEncoder().encode("hej") }, + { name: "S08 - Containerizing Applications/big.txt", data: big }, + ])); + expect(listing).toContain("S08 - Containerizing Applications/Æble notes.txt"); + expect(listing).toContain("hej"); + expect(listing).toContain("abcabc"); +}); + +forEachDb("code listing and zip for a module, skipping escaping symlinks", async (db) => { + const root = mkdtempSync(join(tmpdir(), "stc-code-")); + const outside = mkdtempSync(join(tmpdir(), "stc-outside-")); + writeFileSync(join(outside, "secret.txt"), "TOP SECRET"); + mkdirSync(join(root, "c/S01 - Ünïcode/sub"), { recursive: true }); + mkdirSync(join(root, "c/S01 - Ünïcode/node_modules"), { recursive: true }); + writeFileSync(join(root, "c/S01 - Ünïcode/run.sh"), "echo hi"); + writeFileSync(join(root, "c/S01 - Ünïcode/sub/app.py"), "print(1)"); + writeFileSync(join(root, "c/S01 - Ünïcode/node_modules/x.js"), "x"); + symlinkSync(join(outside, "secret.txt"), join(root, "c/S01 - Ünïcode/leak.txt")); + symlinkSync(join(root, "c/S01 - Ünïcode/run.sh"), join(root, "c/S01 - Ünïcode/alias.sh")); + writeFileSync(join(root, "stcourse.yaml"), `schema: 1\nid: codey\ntitle: Codey\nvisibility: public\nmodules:\n - id: m1\n title: M\n code: { dir: "c/S01 - Ünïcode", command: "sh run.sh" }\n lessons: [{ id: l1, title: L, type: exercise }]\n`); + const t = await makeApp(db, [FIXTURES, root]); + + const list = await (await t.call("GET", "/api/courses/codey/code/m1")).json(); + expect(list.files.map((f: { path: string }) => f.path)).toEqual(["alias.sh", "run.sh", "sub/app.py"]); + expect(list.command).toBe("sh run.sh"); + + const r = await t.call("GET", "/api/courses/codey/code/m1/zip"); + expect(r.status).toBe(200); + expect(r.headers.get("content-type")).toBe("application/zip"); + expect(r.headers.get("content-disposition")).toContain("filename*=UTF-8''S01%20-%20%C3%9Cn%C3%AFcode.zip"); + const listing = await unzipList(new Uint8Array(await r.arrayBuffer())); + expect(listing).toContain("S01 - Ünïcode/sub/app.py"); + expect(listing).not.toContain("TOP SECRET"); + expect(listing).not.toContain("node_modules"); + + expect((await t.call("GET", "/api/courses/codey/code/nope")).status).toBe(404); + expect((await t.call("GET", "/api/courses/sample/code/m1")).status).toBe(200); + expect((await t.call("GET", "/api/courses/sample/code/m2")).status).toBe(404); +}, 30_000);