feat(learner): section code listing and zip download
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
478ebeb255
commit
34826762a8
@@ -0,0 +1,81 @@
|
||||
import { lstatSync, readdirSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { join, sep } from "node:path";
|
||||
import type { Feature, RequestContext, ServerContext } from "@stc/contracts";
|
||||
import { contentDisposition } from "@stc/core";
|
||||
import type { CodeListing } from "../shared/types";
|
||||
import { courseFor } from "./access";
|
||||
import { json, notFound, param } from "./http";
|
||||
import { zip } from "./zip";
|
||||
|
||||
const SKIP = new Set([".git", "node_modules", "__pycache__", ".DS_Store", ".venv", "target", "dist", ".idea", ".vscode"]);
|
||||
export const MAX_FILES = 2000;
|
||||
export const MAX_ZIP_BYTES = 200 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Regular files under `dirAbs` (relative paths, sorted). Symlinks are followed only when their target is a
|
||||
* regular file inside `rootReal`; symlinked directories are skipped, so there are no cycles and no escapes.
|
||||
*/
|
||||
export function listFiles(rootReal: string, dirAbs: string, max = MAX_FILES): { files: { path: string; size: number; abs: string }[]; truncated: boolean } {
|
||||
const files: { path: string; size: number; abs: string }[] = [];
|
||||
let truncated = false;
|
||||
const walk = (abs: string, rel: string) => {
|
||||
let entries;
|
||||
try { entries = readdirSync(abs, { withFileTypes: true }); } catch { return; }
|
||||
entries.sort((a, b) => a.name.localeCompare(b.name));
|
||||
for (const e of entries) {
|
||||
if (SKIP.has(e.name)) continue;
|
||||
if (files.length >= max) { truncated = true; return; }
|
||||
const childAbs = join(abs, e.name), childRel = rel ? `${rel}/${e.name}` : e.name;
|
||||
if (e.isDirectory()) { walk(childAbs, childRel); continue; }
|
||||
let real = childAbs;
|
||||
if (e.isSymbolicLink()) {
|
||||
try { real = realpathSync(childAbs); } catch { continue; }
|
||||
if (!(real === rootReal || real.startsWith(rootReal + sep))) continue;
|
||||
}
|
||||
try {
|
||||
const st = lstatSync(real);
|
||||
if (st.isFile()) files.push({ path: childRel, size: st.size, abs: real });
|
||||
} catch { /* vanished */ }
|
||||
}
|
||||
};
|
||||
walk(dirAbs, "");
|
||||
return { files, truncated };
|
||||
}
|
||||
|
||||
export function codeRoutes(ctx: ServerContext): Feature["routes"] {
|
||||
async function resolve(rc: RequestContext) {
|
||||
const found = await courseFor(ctx, rc.user, param(rc, "courseId"));
|
||||
if (!found) return null;
|
||||
const mod = found.manifest.modules.find((m) => m.id === param(rc, "moduleId"));
|
||||
if (!mod?.code) return null;
|
||||
const dirAbs = ctx.courses.resolvePath(found.manifest.id, mod.code.dir);
|
||||
if (!dirAbs) return null;
|
||||
return { found, mod, code: mod.code, dirAbs, rootReal: realpathSync(found.course.root) };
|
||||
}
|
||||
|
||||
return {
|
||||
"/api/courses/:courseId/code/:moduleId": {
|
||||
GET: async (_req, rc) => {
|
||||
const r = await resolve(rc);
|
||||
if (!r) return notFound();
|
||||
const { files, truncated } = listFiles(r.rootReal, r.dirAbs);
|
||||
const listing: CodeListing = { moduleId: r.mod.id, dir: r.code.dir, command: r.code.command ?? null, files: files.map(({ path, size }) => ({ path, size })), truncated };
|
||||
return json(listing);
|
||||
},
|
||||
},
|
||||
"/api/courses/:courseId/code/:moduleId/zip": {
|
||||
GET: async (_req, rc) => {
|
||||
const r = await resolve(rc);
|
||||
if (!r) return notFound();
|
||||
const { files } = listFiles(r.rootReal, r.dirAbs, 20_000);
|
||||
const total = files.reduce((s, f) => s + f.size, 0);
|
||||
if (total > MAX_ZIP_BYTES) return json({ error: "this folder is too large to download as one zip" }, 413);
|
||||
const folder = r.code.dir.split("/").filter(Boolean).pop() ?? r.mod.id;
|
||||
const data = zip(files.map((f) => ({ name: `${folder}/${f.path}`, data: readFileSync(f.abs), mtime: lstatSync(f.abs).mtime })));
|
||||
return new Response(data, {
|
||||
headers: { "Content-Type": "application/zip", "Content-Length": String(data.length), "Content-Disposition": contentDisposition(`${folder}.zip`) },
|
||||
});
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { adminRoutes } from "./admin";
|
||||
import { libraryRoutes } from "./catalog";
|
||||
import { lessonRoutes } from "./lesson";
|
||||
import { userDataRoutes } from "./userdata";
|
||||
import { codeRoutes } from "./code";
|
||||
|
||||
/**
|
||||
* The learner feature: auth, library, reader, notes, progress and exercises.
|
||||
@@ -28,6 +29,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic
|
||||
...libraryRoutes(ctx),
|
||||
...lessonRoutes(ctx),
|
||||
...userDataRoutes(ctx),
|
||||
...codeRoutes(ctx),
|
||||
};
|
||||
// Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()).
|
||||
for (const methods of Object.values(routes)) {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/** Minimal ZIP writer (PKWARE APPNOTE 6.3): deflate or store per entry, CRC-32, UTF-8 names (flag bit 11). No ZIP64. */
|
||||
export interface ZipEntry { name: string; data: Uint8Array; mtime?: Date }
|
||||
|
||||
function dosTime(d: Date): { time: number; date: number } {
|
||||
const y = Math.max(1980, d.getFullYear());
|
||||
return {
|
||||
time: (d.getHours() << 11) | (d.getMinutes() << 5) | Math.floor(d.getSeconds() / 2),
|
||||
date: ((y - 1980) << 9) | ((d.getMonth() + 1) << 5) | d.getDate(),
|
||||
};
|
||||
}
|
||||
|
||||
export function zip(entries: ZipEntry[]): Uint8Array {
|
||||
const enc = new TextEncoder();
|
||||
const locals: Uint8Array[] = [];
|
||||
const centrals: Uint8Array[] = [];
|
||||
let offset = 0;
|
||||
for (const e of entries) {
|
||||
const name = enc.encode(e.name);
|
||||
const crc = Bun.hash.crc32(e.data) >>> 0;
|
||||
const deflated = Bun.deflateSync(e.data);
|
||||
const useDeflate = deflated.length < e.data.length;
|
||||
const body = useDeflate ? deflated : e.data;
|
||||
const { time, date } = dosTime(e.mtime ?? new Date());
|
||||
const local = new Uint8Array(30 + name.length);
|
||||
const lv = new DataView(local.buffer);
|
||||
lv.setUint32(0, 0x04034b50, true);
|
||||
lv.setUint16(4, 20, true);
|
||||
lv.setUint16(6, 0x0800, true); // UTF-8 names
|
||||
lv.setUint16(8, useDeflate ? 8 : 0, true);
|
||||
lv.setUint16(10, time, true);
|
||||
lv.setUint16(12, date, true);
|
||||
lv.setUint32(14, crc, true);
|
||||
lv.setUint32(18, body.length, true);
|
||||
lv.setUint32(22, e.data.length, true);
|
||||
lv.setUint16(26, name.length, true);
|
||||
lv.setUint16(28, 0, true);
|
||||
local.set(name, 30);
|
||||
const central = new Uint8Array(46 + name.length);
|
||||
const cv = new DataView(central.buffer);
|
||||
cv.setUint32(0, 0x02014b50, true);
|
||||
cv.setUint16(4, 0x0314, true); // made by: unix, 2.0
|
||||
cv.setUint16(6, 20, true);
|
||||
cv.setUint16(8, 0x0800, true);
|
||||
cv.setUint16(10, useDeflate ? 8 : 0, true);
|
||||
cv.setUint16(12, time, true);
|
||||
cv.setUint16(14, date, true);
|
||||
cv.setUint32(16, crc, true);
|
||||
cv.setUint32(20, body.length, true);
|
||||
cv.setUint32(24, e.data.length, true);
|
||||
cv.setUint16(28, name.length, true);
|
||||
cv.setUint32(38, (0o100644 << 16) >>> 0, true); // external attrs: regular file rw-r--r--
|
||||
cv.setUint32(42, offset, true);
|
||||
central.set(name, 46);
|
||||
locals.push(local, body);
|
||||
centrals.push(central);
|
||||
offset += local.length + body.length;
|
||||
}
|
||||
const cdSize = centrals.reduce((s, c) => s + c.length, 0);
|
||||
const end = new Uint8Array(22);
|
||||
const ev = new DataView(end.buffer);
|
||||
ev.setUint32(0, 0x06054b50, true);
|
||||
ev.setUint16(8, entries.length, true);
|
||||
ev.setUint16(10, entries.length, true);
|
||||
ev.setUint32(12, cdSize, true);
|
||||
ev.setUint32(16, offset, true);
|
||||
const out = new Uint8Array(offset + cdSize + 22);
|
||||
let p = 0;
|
||||
for (const part of [...locals, ...centrals, end]) { out.set(part, p); p += part.length; }
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { mkdtempSync, symlinkSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { zip } from "../src/server/zip";
|
||||
import { forEachDb } from "./helpers";
|
||||
import { FIXTURES, makeApp } from "./app";
|
||||
|
||||
async function unzipList(bytes: Uint8Array): Promise<string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "stc-zip-"));
|
||||
await Bun.write(join(dir, "a.zip"), bytes);
|
||||
const out = await Bun.$`unzip -l ${join(dir, "a.zip")}`.text();
|
||||
await Bun.$`unzip -o -q ${join(dir, "a.zip")} -d ${join(dir, "x")}`;
|
||||
return out + "\n" + (await Bun.$`cd ${join(dir, "x")} && find . -type f -exec cat {} +`.text());
|
||||
}
|
||||
|
||||
test("zip round-trips through unzip with non-ASCII names", async () => {
|
||||
const big = new TextEncoder().encode("abc".repeat(1000));
|
||||
const listing = await unzipList(zip([
|
||||
{ name: "S08 - Containerizing Applications/Æble notes.txt", data: new TextEncoder().encode("hej") },
|
||||
{ name: "S08 - Containerizing Applications/big.txt", data: big },
|
||||
]));
|
||||
expect(listing).toContain("S08 - Containerizing Applications/Æble notes.txt");
|
||||
expect(listing).toContain("hej");
|
||||
expect(listing).toContain("abcabc");
|
||||
});
|
||||
|
||||
forEachDb("code listing and zip for a module, skipping escaping symlinks", async (db) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "stc-code-"));
|
||||
const outside = mkdtempSync(join(tmpdir(), "stc-outside-"));
|
||||
writeFileSync(join(outside, "secret.txt"), "TOP SECRET");
|
||||
mkdirSync(join(root, "c/S01 - Ünïcode/sub"), { recursive: true });
|
||||
mkdirSync(join(root, "c/S01 - Ünïcode/node_modules"), { recursive: true });
|
||||
writeFileSync(join(root, "c/S01 - Ünïcode/run.sh"), "echo hi");
|
||||
writeFileSync(join(root, "c/S01 - Ünïcode/sub/app.py"), "print(1)");
|
||||
writeFileSync(join(root, "c/S01 - Ünïcode/node_modules/x.js"), "x");
|
||||
symlinkSync(join(outside, "secret.txt"), join(root, "c/S01 - Ünïcode/leak.txt"));
|
||||
symlinkSync(join(root, "c/S01 - Ünïcode/run.sh"), join(root, "c/S01 - Ünïcode/alias.sh"));
|
||||
writeFileSync(join(root, "stcourse.yaml"), `schema: 1\nid: codey\ntitle: Codey\nvisibility: public\nmodules:\n - id: m1\n title: M\n code: { dir: "c/S01 - Ünïcode", command: "sh run.sh" }\n lessons: [{ id: l1, title: L, type: exercise }]\n`);
|
||||
const t = await makeApp(db, [FIXTURES, root]);
|
||||
|
||||
const list = await (await t.call("GET", "/api/courses/codey/code/m1")).json();
|
||||
expect(list.files.map((f: { path: string }) => f.path)).toEqual(["alias.sh", "run.sh", "sub/app.py"]);
|
||||
expect(list.command).toBe("sh run.sh");
|
||||
|
||||
const r = await t.call("GET", "/api/courses/codey/code/m1/zip");
|
||||
expect(r.status).toBe(200);
|
||||
expect(r.headers.get("content-type")).toBe("application/zip");
|
||||
expect(r.headers.get("content-disposition")).toContain("filename*=UTF-8''S01%20-%20%C3%9Cn%C3%AFcode.zip");
|
||||
const listing = await unzipList(new Uint8Array(await r.arrayBuffer()));
|
||||
expect(listing).toContain("S01 - Ünïcode/sub/app.py");
|
||||
expect(listing).not.toContain("TOP SECRET");
|
||||
expect(listing).not.toContain("node_modules");
|
||||
|
||||
expect((await t.call("GET", "/api/courses/codey/code/nope")).status).toBe(404);
|
||||
expect((await t.call("GET", "/api/courses/sample/code/m1")).status).toBe(200);
|
||||
expect((await t.call("GET", "/api/courses/sample/code/m2")).status).toBe(404);
|
||||
}, 30_000);
|
||||
Reference in New Issue
Block a user