fix: final-review wave (CSRF guard, fail-closed owner bootstrap, streamed zips, e2e visibility suite)
- Critical 1 (CSRF): core wrap refuses every POST/PUT/PATCH/DELETE whose Origin (Referer as fallback) is not publicUrl's origin or STC_ALLOWED_ORIGINS, and any body that is not application/json (multipart only on declared upload routes). Better Auth routes go through the same guard; only declared identity-provider callbacks (SAML ACS/SLO, OAuth callbacks) are exempt. Better Auth trustedOrigins use the same list. Session cookie stays Lax (Strict breaks OAuth/OIDC/SAML returns); documented. - Important 4 / learner minor 4: the first-owner claim and the role row are one transaction guarded by "no owner yet"; only a unique violation means learner; any other error fails the sign-up with 500 and removes the half-created user. Serialised per DB handle (Bun's SQLite connection). - Important 2 / learner minor 12: code zips stream entry by entry (large files stored in 256 KB chunks with a data descriptor and incremental CRC-32), so memory stays at one chunk; the cap is now 1 GiB (ZIP32). - Important 3 / learner minor 10: the e2e runs the production wiring (productFeatures order, AI drafts through the real sink) against a restricted course and asserts 404/401 for learner and anonymous on course, lesson, media, cover, code, zip, notes, progress, attempts, AI chat, library, search and AI status; a real Range request (206, 416); forged writes (foreign Origin, text/plain, no Origin) are 403 and the MCP test command never runs. - Minors 7-9: compose passes every documented variable; the health check uses STC_PORT; productFeatures checks the studio-before-AI order. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
69aaedcb43
commit
8045c80d93
@@ -1,4 +1,5 @@
|
||||
import { betterAuth, type BetterAuthOptions } from "better-auth";
|
||||
import { APIError } from "better-auth/api";
|
||||
import { genericOAuth } from "better-auth/plugins/generic-oauth";
|
||||
import { sso } from "@better-auth/sso";
|
||||
import type { AuthService, AuthUser, Role, ServerContext } from "@stc/contracts";
|
||||
@@ -31,20 +32,67 @@ export async function groupsOf(db: Db, userId: string): Promise<string[]> {
|
||||
return rows.map((r: { group_id: string }) => r.group_id);
|
||||
}
|
||||
|
||||
/** A unique or primary-key violation, on SQLite or Postgres: another sign-up claimed the owner key first. */
|
||||
export function isUniqueViolation(e: unknown): boolean {
|
||||
const err = e as { code?: string; errno?: string | number; message?: string } | null;
|
||||
if (!err) return false;
|
||||
if (err.code === "23505") return true; // Postgres unique_violation
|
||||
const text = `${err.code ?? ""} ${err.errno ?? ""} ${err.message ?? ""}`;
|
||||
return /SQLITE_CONSTRAINT_(PRIMARYKEY|UNIQUE)|UNIQUE constraint failed|duplicate key value/i.test(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* First user ever to be created becomes the owner. The claim is an INSERT into `learner_meta`
|
||||
* keyed `owner`, so with concurrent sign-ups exactly one wins (primary-key conflict for the rest).
|
||||
* First user ever to be created becomes the owner. Atomic and fail-closed:
|
||||
* - one transaction claims the `owner` key in `learner_meta` only if no owner exists (the primary key settles
|
||||
* races) and inserts the user's role row; either both land or neither does;
|
||||
* - only a unique violation (a concurrent sign-up won the claim) means "learner": the role is then inserted in
|
||||
* its own transaction;
|
||||
* - any other error is rethrown. The caller fails the sign-up; nobody is silently made a learner while the
|
||||
* owner key stays free.
|
||||
*/
|
||||
export async function assignInitialRole(db: Db, userId: string): Promise<Role> {
|
||||
let role: Role = "learner";
|
||||
export function assignInitialRole(db: Db, userId: string): Promise<Role> {
|
||||
// Serialised per database handle: Bun's SQLite handle is one connection and refuses a second open transaction.
|
||||
const prev = roleLocks.get(db) ?? Promise.resolve();
|
||||
const run = prev.catch(() => {}).then(() => claimRole(db, userId));
|
||||
roleLocks.set(db, run.catch(() => {}));
|
||||
return run;
|
||||
}
|
||||
|
||||
const roleLocks = new WeakMap<Db, Promise<unknown>>();
|
||||
|
||||
async function claimRole(db: Db, userId: string): Promise<Role> {
|
||||
try {
|
||||
await db.sql`INSERT INTO learner_meta (key, value) VALUES ('owner', ${userId})`;
|
||||
role = "owner";
|
||||
} catch {
|
||||
role = "learner";
|
||||
return await db.sql.begin(async (tx) => {
|
||||
const claimed = await tx`
|
||||
INSERT INTO learner_meta (key, value)
|
||||
SELECT 'owner', ${userId} WHERE NOT EXISTS (SELECT 1 FROM learner_meta WHERE key = 'owner')
|
||||
RETURNING key`;
|
||||
const role: Role = claimed.length === 1 ? "owner" : "learner";
|
||||
await tx`INSERT INTO user_roles (user_id, role) VALUES (${userId}, ${role})`;
|
||||
return role;
|
||||
});
|
||||
} catch (e) {
|
||||
if (!isUniqueViolation(e)) throw e;
|
||||
await db.sql.begin(async (tx) => { await tx`INSERT INTO user_roles (user_id, role) VALUES (${userId}, 'learner')`; });
|
||||
return "learner";
|
||||
}
|
||||
await db.sql`INSERT INTO user_roles (user_id, role) VALUES (${userId}, ${role})`;
|
||||
return role;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a user whose sign-up could not be completed (role assignment failed), so no account without a role
|
||||
* is left behind and the sign-up can simply be retried.
|
||||
*/
|
||||
async function discardUser(db: Db, userId: string): Promise<void> {
|
||||
// Each step on its own: the failure that brought us here may affect one of these tables.
|
||||
const steps = [
|
||||
() => db.sql`DELETE FROM auth_user WHERE id = ${userId}`,
|
||||
() => db.sql`DELETE FROM auth_account WHERE user_id = ${userId}`,
|
||||
() => db.sql`DELETE FROM auth_session WHERE user_id = ${userId}`,
|
||||
() => db.sql`DELETE FROM user_roles WHERE user_id = ${userId}`,
|
||||
];
|
||||
const errors: unknown[] = [];
|
||||
for (const s of steps) { try { await s(); } catch (e) { errors.push(e); } }
|
||||
if (errors.length === steps.length) throw errors[0];
|
||||
}
|
||||
|
||||
export interface LearnerAuth {
|
||||
@@ -110,11 +158,24 @@ export function createAuth(ctx: ServerContext, ready: Promise<unknown>, env: Env
|
||||
emailAndPassword: { enabled: true, minPasswordLength: 8, autoSignIn: true },
|
||||
socialProviders,
|
||||
plugins,
|
||||
trustedOrigins: [config.publicUrl, ...(env.STC_TRUSTED_ORIGINS ?? "").split(",").map((s) => s.trim()).filter(Boolean)],
|
||||
// The same list core's cross-site guard enforces (publicUrl + STC_ALLOWED_ORIGINS / STC_TRUSTED_ORIGINS).
|
||||
trustedOrigins: config.allowedOrigins?.length ? config.allowedOrigins : [config.publicUrl],
|
||||
telemetry: { enabled: false },
|
||||
session: { cookieCache: { enabled: true, maxAge: 60 } },
|
||||
databaseHooks: {
|
||||
user: { create: { after: async (user) => { await assignInitialRole(db, user.id); } } },
|
||||
user: {
|
||||
create: {
|
||||
after: async (user) => {
|
||||
try {
|
||||
await assignInitialRole(db, user.id);
|
||||
} catch (e) {
|
||||
console.error("[learner] role assignment failed; the sign-up is refused", e);
|
||||
await discardUser(db, user.id).catch((d) => console.error("[learner] could not remove the half-created user", d));
|
||||
throw new APIError("INTERNAL_SERVER_ERROR", { message: "Could not set up the account. Try again." });
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
session: {
|
||||
create: {
|
||||
after: async (session) => {
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
import { lstatSync, readdirSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { lstatSync, readdirSync, realpathSync } from "node:fs";
|
||||
import { join, sep } from "node:path";
|
||||
import type { Feature, RequestContext, ServerContext } from "@stc/contracts";
|
||||
import { contentDisposition } from "@stc/core";
|
||||
import type { CodeListing } from "../shared/types";
|
||||
import { courseFor } from "./access";
|
||||
import { json, notFound, param } from "./http";
|
||||
import { zip } from "./zip";
|
||||
import { toStream, zipStream } from "./zip";
|
||||
|
||||
const SKIP = new Set([".git", "node_modules", "__pycache__", ".DS_Store", ".venv", "target", "dist", ".idea", ".vscode"]);
|
||||
export const MAX_FILES = 2000;
|
||||
export const MAX_ZIP_BYTES = 200 * 1024 * 1024;
|
||||
/** The zip is streamed (bounded memory); this caps the work per download and keeps it under ZIP32's 4 GiB. */
|
||||
export const MAX_ZIP_BYTES = 1024 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Regular files under `dirAbs` (relative paths, sorted). Symlinks are followed only when their target is a
|
||||
@@ -69,11 +70,12 @@ export function codeRoutes(ctx: ServerContext): Feature["routes"] {
|
||||
if (!r) return notFound();
|
||||
const { files } = listFiles(r.rootReal, r.dirAbs, 20_000);
|
||||
const total = files.reduce((s, f) => s + f.size, 0);
|
||||
if (total > MAX_ZIP_BYTES) return json({ error: "this folder is too large to download as one zip" }, 413);
|
||||
if (total > MAX_ZIP_BYTES) return json({ error: `this folder is larger than ${MAX_ZIP_BYTES / 1024 / 1024} MB; download its files one by one` }, 413);
|
||||
const folder = r.code.dir.split("/").filter(Boolean).pop() ?? r.mod.id;
|
||||
const data = zip(files.map((f) => ({ name: `${folder}/${f.path}`, data: new Uint8Array(readFileSync(f.abs)), mtime: lstatSync(f.abs).mtime })));
|
||||
return new Response(data, {
|
||||
headers: { "Content-Type": "application/zip", "Content-Length": String(data.length), "Content-Disposition": contentDisposition(`${folder}.zip`) },
|
||||
const entries = files.map((f) => ({ name: `${folder}/${f.path}`, path: f.abs, size: f.size, mtime: lstatSync(f.abs).mtime }));
|
||||
// Streamed one entry (or one chunk of a large file) at a time; the length is not known up front.
|
||||
return new Response(toStream(zipStream(entries)), {
|
||||
headers: { "Content-Type": "application/zip", "Content-Disposition": contentDisposition(`${folder}.zip`), "Cache-Control": "private, no-store" },
|
||||
});
|
||||
},
|
||||
},
|
||||
|
||||
@@ -37,5 +37,9 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic
|
||||
...assetRoutes(),
|
||||
...runnerFrameRoutes(),
|
||||
};
|
||||
return { name: "learner", routes, auth: auth.service };
|
||||
return {
|
||||
name: "learner", routes, auth: auth.service,
|
||||
// Posted by identity providers (SAML assertions, OAuth/OIDC redirects); Better Auth checks signatures and state.
|
||||
externalCallbacks: ["/api/auth/sso/saml2/sp/acs", "/api/auth/sso/saml2/sp/slo", "/api/auth/sso/callback", "/api/auth/callback/", "/api/auth/oauth2/callback/"],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -68,3 +68,150 @@ export function zip(entries: ZipEntry[]): Uint8Array<ArrayBuffer> {
|
||||
for (const part of [...locals, ...centrals, end]) { out.set(part, p); p += part.length; }
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- streaming writer ----------
|
||||
|
||||
const CRC_TABLE = (() => {
|
||||
const t = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
t[n] = c >>> 0;
|
||||
}
|
||||
return t;
|
||||
})();
|
||||
|
||||
/** Incremental CRC-32 (the same polynomial as Bun.hash.crc32). Start with 0, pass the result back in. */
|
||||
export function crc32Update(crc: number, data: Uint8Array): number {
|
||||
let c = ~crc >>> 0;
|
||||
for (let i = 0; i < data.length; i++) c = CRC_TABLE[(c ^ data[i]!) & 0xff]! ^ (c >>> 8);
|
||||
return ~c >>> 0;
|
||||
}
|
||||
|
||||
export interface ZipFileEntry { name: string; path: string; size: number; mtime?: Date }
|
||||
|
||||
/** Files up to this size are read whole and deflated; larger ones are stored and streamed in chunks. */
|
||||
export const SMALL_FILE = 4 * 1024 * 1024;
|
||||
export const CHUNK = 256 * 1024;
|
||||
const ZIP32_LIMIT = 0xffffffff;
|
||||
|
||||
/**
|
||||
* A ZIP written as it is read: one entry at a time, never more than one small file or one chunk of a large file in
|
||||
* memory, whatever the folder's size. Large entries use a data descriptor (flag bit 3) so their CRC and sizes come
|
||||
* after the data. No ZIP64: the caller keeps the total under 4 GiB.
|
||||
*/
|
||||
export async function* zipStream(entries: ZipFileEntry[], readChunks: (path: string) => AsyncIterable<Uint8Array> = fileChunks): AsyncGenerator<Uint8Array> {
|
||||
const enc = new TextEncoder();
|
||||
const centrals: Uint8Array[] = [];
|
||||
let offset = 0;
|
||||
for (const e of entries) {
|
||||
const name = enc.encode(e.name);
|
||||
const { time, date } = dosTime(e.mtime ?? new Date());
|
||||
let crc = 0, csize = 0, usize = 0, method = 0, flags = 0x0800;
|
||||
const header = (known: boolean) => {
|
||||
const local = new Uint8Array(30 + name.length);
|
||||
const lv = new DataView(local.buffer);
|
||||
lv.setUint32(0, 0x04034b50, true);
|
||||
lv.setUint16(4, 20, true);
|
||||
lv.setUint16(6, flags, true);
|
||||
lv.setUint16(8, method, true);
|
||||
lv.setUint16(10, time, true);
|
||||
lv.setUint16(12, date, true);
|
||||
lv.setUint32(14, known ? crc : 0, true);
|
||||
lv.setUint32(18, known ? csize : 0, true);
|
||||
lv.setUint32(22, known ? usize : 0, true);
|
||||
lv.setUint16(26, name.length, true);
|
||||
local.set(name, 30);
|
||||
return local;
|
||||
};
|
||||
const start = offset;
|
||||
if (e.size <= SMALL_FILE) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for await (const c of readChunks(e.path)) parts.push(c);
|
||||
const data = new Uint8Array(parts.reduce((s, p) => s + p.length, 0));
|
||||
let p = 0;
|
||||
for (const part of parts) { data.set(part, p); p += part.length; }
|
||||
crc = Bun.hash.crc32(data) >>> 0;
|
||||
usize = data.length;
|
||||
const deflated = Bun.deflateSync(data);
|
||||
const body = deflated.length < data.length ? deflated : data;
|
||||
method = body === deflated ? 8 : 0;
|
||||
csize = body.length;
|
||||
const local = header(true);
|
||||
yield local;
|
||||
yield body;
|
||||
offset += local.length + body.length;
|
||||
} else {
|
||||
flags |= 0x0008; // sizes and CRC follow the data
|
||||
const local = header(false);
|
||||
yield local;
|
||||
offset += local.length;
|
||||
for await (const c of readChunks(e.path)) {
|
||||
crc = crc32Update(crc, c);
|
||||
usize += c.length;
|
||||
yield c;
|
||||
}
|
||||
csize = usize;
|
||||
offset += usize;
|
||||
const dd = new Uint8Array(16);
|
||||
const dv = new DataView(dd.buffer);
|
||||
dv.setUint32(0, 0x08074b50, true);
|
||||
dv.setUint32(4, crc, true);
|
||||
dv.setUint32(8, csize, true);
|
||||
dv.setUint32(12, usize, true);
|
||||
yield dd;
|
||||
offset += dd.length;
|
||||
}
|
||||
if (offset > ZIP32_LIMIT) throw new Error("zip larger than 4 GiB (no ZIP64)");
|
||||
const central = new Uint8Array(46 + name.length);
|
||||
const cv = new DataView(central.buffer);
|
||||
cv.setUint32(0, 0x02014b50, true);
|
||||
cv.setUint16(4, 0x0314, true);
|
||||
cv.setUint16(6, 20, true);
|
||||
cv.setUint16(8, flags, true);
|
||||
cv.setUint16(10, method, true);
|
||||
cv.setUint16(12, time, true);
|
||||
cv.setUint16(14, date, true);
|
||||
cv.setUint32(16, crc, true);
|
||||
cv.setUint32(20, csize, true);
|
||||
cv.setUint32(24, usize, true);
|
||||
cv.setUint16(28, name.length, true);
|
||||
cv.setUint32(38, (0o100644 << 16) >>> 0, true);
|
||||
cv.setUint32(42, start, true);
|
||||
central.set(name, 46);
|
||||
centrals.push(central);
|
||||
}
|
||||
let cdSize = 0;
|
||||
for (const c of centrals) { yield c; cdSize += c.length; }
|
||||
const end = new Uint8Array(22);
|
||||
const ev = new DataView(end.buffer);
|
||||
ev.setUint32(0, 0x06054b50, true);
|
||||
ev.setUint16(8, entries.length, true);
|
||||
ev.setUint16(10, entries.length, true);
|
||||
ev.setUint32(12, cdSize, true);
|
||||
ev.setUint32(16, offset, true);
|
||||
yield end;
|
||||
}
|
||||
|
||||
/** A file's bytes in chunks of at most CHUNK, read one at a time. */
|
||||
export async function* fileChunks(path: string): AsyncGenerator<Uint8Array> {
|
||||
const f = Bun.file(path);
|
||||
const size = f.size;
|
||||
for (let pos = 0; pos < size; pos += CHUNK) yield new Uint8Array(await f.slice(pos, Math.min(size, pos + CHUNK)).arrayBuffer());
|
||||
}
|
||||
|
||||
/** Wraps a chunk generator as a pull-based stream: the next chunk is produced only when the client takes one. */
|
||||
export function toStream(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
try {
|
||||
const { done, value } = await gen.next();
|
||||
if (done) controller.close();
|
||||
else controller.enqueue(value);
|
||||
} catch (e) {
|
||||
controller.error(e);
|
||||
}
|
||||
},
|
||||
async cancel() { await gen.return(undefined); },
|
||||
}, { highWaterMark: 1 });
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { assignInitialRole } from "../src/server/auth";
|
||||
import { expect, spyOn, test } from "bun:test";
|
||||
import { assignInitialRole, isUniqueViolation } from "../src/server/auth";
|
||||
import { syncClaimGroups, groupsFromClaims } from "../src/server/auth/groups";
|
||||
import { forEachDb } from "./helpers";
|
||||
import { cookieFrom, makeApp } from "./app";
|
||||
@@ -60,3 +60,43 @@ test("groupsFromClaims reads arrays and strings", () => {
|
||||
expect(groupsFromClaims({ extraFields: { groups: ["z"] } })).toEqual(["z"]);
|
||||
expect(groupsFromClaims(null)).toEqual([]);
|
||||
});
|
||||
|
||||
forEachDb("owner bootstrap fails closed: an injected failure refuses the sign-up, no silent learner, no free owner slot", async (db) => {
|
||||
const t = await makeApp(db);
|
||||
const count = async (q: string) => Number((await db.sql.unsafe(q))[0].n);
|
||||
const err = spyOn(console, "error").mockImplementation(() => {});
|
||||
try {
|
||||
// The role table is unreachable: role assignment fails with a non-unique error.
|
||||
await db.sql.unsafe("ALTER TABLE user_roles RENAME TO user_roles_off");
|
||||
const signUp = (email: string) => t.call("POST", "/api/auth/sign-up/email", { body: { email, name: "X", password: "correct horse battery" } });
|
||||
const first = await signUp("first@example.com");
|
||||
expect(first.status).toBe(500);
|
||||
// nothing is left: no account, no owner claim
|
||||
expect(await count("SELECT COUNT(*) AS n FROM auth_user")).toBe(0);
|
||||
expect(await count("SELECT COUNT(*) AS n FROM learner_meta WHERE key = 'owner'")).toBe(0);
|
||||
// while the failure lasts, the next sign-up does not become owner either: it fails the same way
|
||||
expect((await signUp("stranger@example.com")).status).toBe(500);
|
||||
expect(await count("SELECT COUNT(*) AS n FROM auth_user")).toBe(0);
|
||||
expect(await count("SELECT COUNT(*) AS n FROM learner_meta WHERE key = 'owner'")).toBe(0);
|
||||
// once the database recovers, the first completed account is the owner, the next a learner
|
||||
await db.sql.unsafe("ALTER TABLE user_roles_off RENAME TO user_roles");
|
||||
const own = await t.signUp("first@example.com");
|
||||
const lea = await t.signUp("stranger@example.com");
|
||||
expect((await (await t.call("GET", "/api/me", { cookie: own })).json()).user.roles).toEqual(["owner"]);
|
||||
expect((await (await t.call("GET", "/api/me", { cookie: lea })).json()).user.roles).toEqual(["learner"]);
|
||||
} finally {
|
||||
err.mockRestore();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
forEachDb("assignInitialRole rethrows anything but a unique violation, and rolls the owner claim back", async (db) => {
|
||||
await db.sql.unsafe("ALTER TABLE user_roles RENAME TO user_roles_off");
|
||||
await expect(assignInitialRole(db, "u1")).rejects.toThrow();
|
||||
expect((await db.sql`SELECT COUNT(*) AS n FROM learner_meta WHERE key = 'owner'`)[0].n.toString()).toBe("0");
|
||||
await db.sql.unsafe("ALTER TABLE user_roles_off RENAME TO user_roles");
|
||||
expect(await assignInitialRole(db, "u1")).toBe("owner");
|
||||
expect(await assignInitialRole(db, "u2")).toBe("learner");
|
||||
expect(isUniqueViolation({ code: "23505" })).toBe(true);
|
||||
expect(isUniqueViolation(new Error("UNIQUE constraint failed: learner_meta.key"))).toBe(true);
|
||||
expect(isUniqueViolation(new Error("no such table: user_roles"))).toBe(false);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { closeSync, mkdirSync, mkdtempSync, openSync, writeFileSync, writeSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { CHUNK, SMALL_FILE, crc32Update, toStream, zipStream } from "../src/server/zip";
|
||||
import { forEachDb } from "./helpers";
|
||||
import { FIXTURES, makeApp } from "./app";
|
||||
|
||||
const rnd = (n: number) => { const b = new Uint8Array(n); for (let i = 0; i < n; i += 65536) crypto.getRandomValues(b.subarray(i, Math.min(n, i + 65536))); return b; };
|
||||
|
||||
test("incremental CRC-32 matches Bun.hash.crc32", () => {
|
||||
const data = rnd(300_000);
|
||||
let c = 0;
|
||||
for (let i = 0; i < data.length; i += 7777) c = crc32Update(c, data.subarray(i, i + 7777));
|
||||
expect(c).toBe(Bun.hash.crc32(data) >>> 0);
|
||||
});
|
||||
|
||||
test("the stream holds at most one chunk of a large file; unzip verifies every CRC", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "stc-zs-"));
|
||||
mkdirSync(join(dir, "Ünï sub"));
|
||||
writeFileSync(join(dir, "big.bin"), rnd(SMALL_FILE * 3 + 12345)); // streamed, stored, data descriptor
|
||||
writeFileSync(join(dir, "Ünï sub/small.txt"), "hello ".repeat(10_000)); // deflated
|
||||
writeFileSync(join(dir, "empty.txt"), "");
|
||||
const entries = ["big.bin", "Ünï sub/small.txt", "empty.txt"].map((p) => ({ name: `pkg/${p}`, path: join(dir, p), size: Bun.file(join(dir, p)).size }));
|
||||
const out = join(dir, "out.zip");
|
||||
const fd = openSync(out, "w");
|
||||
let largest = 0;
|
||||
for await (const chunk of zipStream(entries)) { largest = Math.max(largest, chunk.length); writeSync(fd, chunk); }
|
||||
closeSync(fd);
|
||||
expect(largest).toBeLessThanOrEqual(Math.max(CHUNK, 60_000)); // never the whole 12 MB file
|
||||
const t = await Bun.$`unzip -t ${out}`.nothrow().text();
|
||||
expect(t).toContain("No errors detected");
|
||||
const x = join(dir, "x");
|
||||
await Bun.$`unzip -q ${out} -d ${x}`;
|
||||
expect(Bun.hash.crc32(new Uint8Array(await Bun.file(join(x, "pkg/big.bin")).arrayBuffer()))).toBe(Bun.hash.crc32(new Uint8Array(await Bun.file(join(dir, "big.bin")).arrayBuffer())));
|
||||
expect(await Bun.file(join(x, "pkg/Ünï sub/small.txt")).text()).toBe("hello ".repeat(10_000));
|
||||
});
|
||||
|
||||
test("toStream pulls one chunk at a time", async () => {
|
||||
let produced = 0;
|
||||
async function* gen() { for (let i = 0; i < 100; i++) { produced++; yield new Uint8Array(10); } }
|
||||
const reader = toStream(gen()).getReader();
|
||||
await reader.read();
|
||||
await Bun.sleep(20);
|
||||
expect(produced).toBeLessThanOrEqual(3); // not run ahead to 100
|
||||
await reader.cancel();
|
||||
});
|
||||
|
||||
forEachDb("a 300 MB code folder downloads as a valid streamed zip without buffering it", async (db) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "stc-bigcode-"));
|
||||
mkdirSync(join(root, "code"), { recursive: true });
|
||||
// sparse file: no disk or memory cost to create, 300 MB to read
|
||||
const f = openSync(join(root, "code/huge.bin"), "w");
|
||||
writeSync(f, new Uint8Array([1]), 0, 1, 300 * 1024 * 1024 - 1);
|
||||
closeSync(f);
|
||||
writeFileSync(join(root, "code/readme.md"), "# hi\n");
|
||||
writeFileSync(join(root, "stcourse.yaml"), `schema: 1\nid: bigcode\ntitle: Big\nvisibility: public\nmodules:\n - id: m1\n title: M\n code: { dir: code }\n lessons: [{ id: l1, title: L, type: text, file: code/readme.md }]\n`);
|
||||
const t = await makeApp(db, [FIXTURES, root]);
|
||||
Bun.gc(true);
|
||||
const before = process.memoryUsage().rss;
|
||||
let peak = before;
|
||||
const r = await t.call("GET", "/api/courses/bigcode/code/m1/zip");
|
||||
expect(r.status).toBe(200);
|
||||
expect(r.headers.get("content-length")).toBeNull(); // streamed
|
||||
const out = join(root, "out.zip");
|
||||
const w = Bun.file(out).writer();
|
||||
const reader = r.body!.getReader();
|
||||
let n = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
w.write(value);
|
||||
if (++n % 64 === 0) { await w.flush(); peak = Math.max(peak, process.memoryUsage().rss); }
|
||||
}
|
||||
await w.end();
|
||||
expect(Bun.file(out).size).toBeGreaterThan(300 * 1024 * 1024);
|
||||
expect(peak - before).toBeLessThan(150 * 1024 * 1024); // a buffered zip would need > 300 MB
|
||||
expect(await Bun.$`unzip -t ${out}`.nothrow().text()).toContain("No errors detected");
|
||||
}, 120_000);
|
||||
Reference in New Issue
Block a user