fix(server): inert proxy responses, API-call header, image-only uploads; origin check by host

From the final review: proxied archived HTML could run on our origin and read a
remembered token; /api/upload accepted any Wayback URL and GIF-prefixed bytes; the
origin check broke behind a TLS proxy; one unreadable snapshot could fail a run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas WintherandClaude Opus 5.5 committed 2026-10-15 01:49:52 +02:00
1 parent 5084f8c557
commit baa3476b73
16 files changed
+183 -26

No files matched your search

+2
View File
@@ -58,6 +58,8 @@ Put it behind a reverse proxy that terminates TLS and **appends** the client add
no file is deleted before its 30 days.
- Nothing is uploaded or submitted without an explicit user action; saves use `save_mode: "review"`.
- `/api/fetch` only fetches allowlisted hosts; `/api/upload` only takes a source URL, never bytes.
- Proxied bodies are inert (`application/octet-stream`, sandbox CSP, attachment), and `/api/*` fetch/upload need
the `x-mbseed: 1` header; `/api/upload` only rehosts from source image hosts or their Wayback copies.
## Known rough edges
See the open questions in spec §7 and [`TODO.md`](TODO.md).
+18
View File
@@ -16,6 +16,24 @@ the repo every session — measurably slower and more tokens).
**Consequences:** Rule changes happen in one file; pointer files must stay thin. Plans are working
artefacts: kept after merge they go stale and contradict the code, so the spec is the record.
## 2026-10-15 — Harden the server proxy and uploads (final review)
**Status:** accepted (amends "Implementation details: server API" below)
**Context:** The whole-branch review found that `/api/fetch` passed the upstream content type through. Anyone can
archive an HTML page on Wayback and link a user to our `/api/fetch?url=…` for it: the script would run on our origin
and could read a MangaBaka token saved with "remember on this device". `/api/upload` also accepted any Wayback URL
and any bytes starting with `GIF`, so our domain could be used as a 30-day file drop and the store filled.
**Decision:**
- Proxied bodies are always `application/octet-stream` with `nosniff`, `Content-Security-Policy: sandbox;
default-src 'none'` and `Content-Disposition: attachment`. Nothing in our clients reads the content type.
- `/api/fetch` and `/api/upload` require the header `x-mbseed: 1`, which our page and userscript send and a link or
form cannot.
- `/api/upload` rehosts only from a source's image hosts (`Source.imageHosts`) or Wayback's copy of one, and only
images 1–10 000 px a side.
- The origin check compares hosts, so `https://host` passes behind a TLS proxy that Bun sees as `http://host`.
**Alternatives:** A session cookie or the password gate (deferred, spec §4.11); serving the page and the proxy on
different origins (more deployment work).
**Consequences:** A new source must list its image hosts. Clients other than ours must send `x-mbseed: 1`.
## 2026-10-14 — Implementation details: server API, Webtoons fetch forms
**Status:** accepted
**Context:** Decisions made while implementing plan 001 that the spec leaves open.
+4
View File
@@ -18,6 +18,10 @@ Design: [specs/001-webtoons-covers.md](specs/001-webtoons-covers.md).
submitted with its hash on MangaBaka. If it doesn't, "already on MangaBaka" never matches.
- [ ] Userscript: fill MangaBaka's cover form for users without a token (spec §4.9) — needs the logged-in
form's markup and a check that it accepts programmatic input (spec §7.5). The panel offers copy-and-paste instead.
- [ ] Small fixes from the final review (deferred as minor): a malformed `Location` on a proxied redirect gives 500
instead of 502; cap the `/api/upload` JSON body size (`maxRequestBodySize`); after Preview, re-run `draw()`
so the button respects an unconfirmed link mismatch and the counts refresh after a partial upload;
label the "direct" host's URL as "Source URL", not "Rehosted".
- [ ] Mobile layout (the user deferred it, 2026-10-15): the page stacks under 1000px but was only checked at 760px.
## Verify with real services (once possible)
+11 -5
View File
@@ -103,11 +103,17 @@ export async function waybackHistory(source: Source, series: SourceSeries, opts:
break;
}
if (res.ok) {
const pageUrl = new URL(row.original);
for (const c of source.extractCovers(await res.text(), pageUrl)) {
const candidate: CoverCandidate = { ...c, current: false, seenAt: row.timestamp, lastSeenAt: row.timestamp };
if (c.originalUrl) candidate.archivedUrl = archivedImageUrl(row.timestamp, c.originalUrl);
found.push(candidate);
try {
const pageUrl = new URL(row.original);
for (const c of source.extractCovers(await res.text(), pageUrl)) {
const candidate: CoverCandidate = { ...c, current: false, seenAt: row.timestamp, lastSeenAt: row.timestamp };
if (c.originalUrl) candidate.archivedUrl = archivedImageUrl(row.timestamp, c.originalUrl);
found.push(candidate);
}
} catch (err) {
// One unreadable snapshot never costs the rest of the history (spec §4.3).
if (opts.signal?.aborted) break;
opts.warn(`Snapshot ${formatMonth(row.timestamp)} couldn't be read (${message(err)}).`);
}
} else {
await res.body?.cancel();
+2 -2
View File
@@ -1,4 +1,4 @@
import type { Transport } from "../transport";
import { API_CALL_HEADER, type Transport } from "../transport";
export type HostId = "server" | "litterbox" | "catbox" | "direct";
export type LitterboxTime = "1h" | "12h" | "24h" | "72h";
@@ -55,7 +55,7 @@ export async function viaServer(
if (ctx.serverBase === null) throw new HostError("No mangabaka-seeder server is configured.");
const res = await ctx.transport.fetch(`${ctx.serverBase}/api/upload`, {
method: "POST",
headers: { "content-type": "application/json" },
headers: { "content-type": "application/json", [API_CALL_HEADER]: "1" },
body: JSON.stringify({ url: image.sourceUrl, ...body }),
timeoutMs: UPLOAD_TIMEOUT_MS,
});
+5 -2
View File
@@ -5,8 +5,11 @@ const NAMED: Readonly<Record<string, string>> = { amp: "&", lt: "<", gt: ">", qu
export function decodeEntities(s: string): string {
return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
const e = entity.toLowerCase();
if (e.startsWith("#x")) return String.fromCodePoint(parseInt(e.slice(2), 16));
if (e.startsWith("#")) return String.fromCodePoint(parseInt(e.slice(1), 10));
if (e.startsWith("#")) {
const code = e.startsWith("#x") ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
// Malformed pages carry entities like &#99999999;: leave them as written rather than throw.
return Number.isInteger(code) && code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;
}
return NAMED[e] ?? match;
});
}
+5
View File
@@ -9,6 +9,11 @@ export function findSource(url: URL, sources: readonly Source[] = SOURCES): Sour
}
/** What `/api/fetch` may fetch: every source's hosts plus Wayback (spec §4.10). */
/** Where cover images may come from: what `/api/upload` will rehost (directly or as a Wayback copy). */
export function allowedImageHosts(sources: readonly Source[] = SOURCES): Set<string> {
return new Set(sources.flatMap((s) => s.imageHosts));
}
export function allowedFetchHosts(sources: readonly Source[] = SOURCES): Set<string> {
return new Set([...sources.flatMap((s) => s.hosts), WAYBACK_HOST]);
}
+2
View File
@@ -18,6 +18,8 @@ export interface Source {
readonly name: string;
/** Hosts this source fetches from; the server proxy allowlist is built from these. */
readonly hosts: readonly string[];
/** Hosts its cover images live on, current and historical; the only sources `/api/upload` rehosts from. */
readonly imageHosts: readonly string[];
matches(url: URL): boolean;
resolve(url: URL, ctx: SourceContext): Promise<SourceSeries>;
currentCovers(series: SourceSeries, ctx: SourceContext): Promise<CoverCandidate[]>;
+2 -1
View File
@@ -2,7 +2,7 @@ import type { CoverCandidate, SourceSeries } from "../../model";
import { SourceError, type Source, type SourceContext } from "../source";
import { extractCovers, parseSeriesPage } from "./parse";
import {
IMAGE_HOST, archivePrefixes, fetchUrlFor, isSameWebtoonsSeries, isWebtoonsSeriesUrl, mangabakaLanguage,
IMAGE_HOST, IMAGE_HOSTS, archivePrefixes, fetchUrlFor, isSameWebtoonsSeries, isWebtoonsSeriesUrl, mangabakaLanguage,
normalizeImageUrl, parseWebtoonsUrl, type WebtoonsRef,
} from "./urls";
@@ -27,6 +27,7 @@ export const webtoons: Source = {
id: "webtoons",
name: "WEBTOON",
hosts: ["www.webtoons.com", "m.webtoons.com", IMAGE_HOST],
imageHosts: [...IMAGE_HOSTS],
matches: (url) => isWebtoonsSeriesUrl(url),
+1 -1
View File
@@ -3,7 +3,7 @@ import { LinkError } from "../../links";
export const WEBTOONS_HOSTS: ReadonlySet<string> = new Set(["www.webtoons.com", "m.webtoons.com", "webtoons.com"]);
export const IMAGE_HOST = "swebtoon-phinf.pstatic.net";
/** Hosts that serve the same image paths; only IMAGE_HOST works without a Webtoons Referer (spec §2.1). */
const IMAGE_HOSTS: ReadonlySet<string> = new Set([
export const IMAGE_HOSTS: ReadonlySet<string> = new Set([
IMAGE_HOST, "webtoon-phinf.pstatic.net", "swebtoon.phinf.naver.net", "webtoon.phinf.naver.net",
]);
/** Webtoons path segment → MangaBaka language code. */
+2
View File
@@ -15,6 +15,8 @@ export interface Transport {
}
export const MANGABAKA_API_HOST = "api.mangabaka.org";
/** Our page and userscript send this on every API call; a plain navigation can't (see server app.ts). */
export const API_CALL_HEADER = "x-mbseed";
export const WAYBACK_HOST = "web.archive.org";
/** Hosts a browser page may call directly because they send CORS headers (spec §2.3, §2.4). */
export const CORS_OPEN_HOSTS: ReadonlySet<string> = new Set([MANGABAKA_API_HOST, "litterbox.catbox.moe"]);
+51 -10
View File
@@ -1,8 +1,8 @@
// The portable request handler (spec §4.10): web-standard APIs only, so a Workers entry can wrap it later.
import { uploadToCatbox } from "../core/hosting/catbox";
import { ImageError, MAX_IMAGE_BYTES, probeImage, readCapped, sha256Hex } from "../core/images";
import { allowedFetchHosts } from "../core/sources/registry";
import { TOOL_USER_AGENT, userAgentFor, type Transport } from "../core/transport";
import { allowedFetchHosts, allowedImageHosts } from "../core/sources/registry";
import { API_CALL_HEADER, TOOL_USER_AGENT, WAYBACK_HOST, userAgentFor, type Transport } from "../core/transport";
import { resolveAssetPath } from "./assets-path";
import { RateLimiter } from "./rate-limit";
import { CONTENT_TYPES, COVER_NAME, StorageFullError, type Storage } from "./storage";
@@ -24,6 +24,16 @@ export const MAX_REDIRECTS = 5;
export const UPSTREAM_TIMEOUT_MS = 30_000;
/** Pages whose scripts may call our API besides our own (the userscript runs on mangabaka.org). */
const EXTRA_ORIGINS = new Set(["https://mangabaka.org"]);
/** Covers are a few hundred pixels to ~2000 wide; anything outside this is not one (or not an image at all). */
const MAX_IMAGE_SIDE = 10_000;
/** Proxied bodies are data for our scripts, never a document: they must not render or run on our origin. */
const INERT_HEADERS = {
"content-type": "application/octet-stream",
"x-content-type-options": "nosniff",
"content-security-policy": "sandbox; default-src 'none'",
"content-disposition": "attachment",
"cache-control": "no-store",
} as const;
const error = (status: number, message: string) =>
Response.json({ error: message }, { status, headers: { "x-mbseed-error": "1", "cache-control": "no-store" } });
@@ -33,7 +43,30 @@ export function createApp(env: ServerEnv): (req: Request) => Promise<Response> {
// A run is ~60–100 proxied requests over several minutes (spec §5).
const limits = env.limits ?? { fetch: new RateLimiter(240, 60_000), upload: new RateLimiter(60, 60_000) };
const upstream: Upstream = env.upstream ?? ((url, init) => fetch(url, init));
const ownOrigin = (req: Request) => env.publicUrl ?? new URL(req.url).origin;
const imageHosts = allowedImageHosts();
/** Behind a TLS proxy Bun sees http://host while the browser says https://host, so compare hosts. */
const isOwnOrigin = (req: Request, origin: string) => {
let host: string;
try {
host = new URL(origin).host;
} catch {
return false;
}
const own = [new URL(req.url).host, req.headers.get("host") ?? ""];
if (env.publicUrl) own.push(new URL(env.publicUrl).host);
return own.includes(host);
};
/** `/api/upload` rehosts cover images only: a source image host, or Wayback's copy of one. */
const isImageSource = (u: URL) => {
if (imageHosts.has(u.hostname)) return true;
if (u.hostname !== WAYBACK_HOST) return false;
const inner = /^\/web\/\d{1,14}(?:id_|im_)?\/(.+)$/.exec(u.pathname + u.search)?.[1];
try {
return inner !== undefined && imageHosts.has(new URL(inner).hostname);
} catch {
return false;
}
};
const allowed = (u: URL) =>
(u.protocol === "https:" || u.protocol === "http:") && hosts.has(u.hostname) && ["", "80", "443"].includes(u.port);
@@ -74,11 +107,7 @@ export function createApp(env: ServerEnv): (req: Request) => Promise<Response> {
}
return new Response(body as Uint8Array<ArrayBuffer>, {
status: res.status,
headers: {
"content-type": res.headers.get("content-type") ?? "application/octet-stream",
"cache-control": "no-store",
"x-mbseed-final-url": url.toString(),
},
headers: { ...INERT_HEADERS, "x-mbseed-final-url": url.toString() },
});
}
return error(502, "Too many redirects.");
@@ -107,6 +136,13 @@ export function createApp(env: ServerEnv): (req: Request) => Promise<Response> {
return error(400, "Bad userhash.");
}
if (body.host === "store" && !env.storage) return error(404, "This server has no storage.");
let source: URL;
try {
source = new URL(body.url);
} catch {
return error(400, "Not a URL.");
}
if (!isImageSource(source)) return error(403, "Only cover images from a supported source (or their Wayback copies) can be rehosted.");
// The client sends the source URL, never bytes, so this can't host arbitrary files (spec §4.6).
const fetched = await proxyGet(body.url);
@@ -114,7 +150,9 @@ export function createApp(env: ServerEnv): (req: Request) => Promise<Response> {
if (!fetched.ok) return error(502, `Could not fetch the image (HTTP ${fetched.status}).`);
const bytes = new Uint8Array(await fetched.arrayBuffer());
const info = probeImage(bytes);
if (!info) return error(415, "That URL is not an image.");
if (!info || info.width < 1 || info.height < 1 || info.width > MAX_IMAGE_SIDE || info.height > MAX_IMAGE_SIDE) {
return error(415, "That URL is not a cover image.");
}
const sha256 = await sha256Hex(bytes);
if (body.host === "store") {
@@ -156,7 +194,10 @@ export function createApp(env: ServerEnv): (req: Request) => Promise<Response> {
if (path.startsWith("/api/")) {
const origin = req.headers.get("origin");
if (origin && origin !== ownOrigin(req) && !EXTRA_ORIGINS.has(origin)) return error(403, "Cross-origin requests are not allowed.");
if (origin && !isOwnOrigin(req, origin) && !EXTRA_ORIGINS.has(origin)) return error(403, "Cross-origin requests are not allowed.");
// Only our scripts send this header; a link or a form can't, so a navigation can't make us fetch.
const fromOurScript = req.headers.get(API_CALL_HEADER) === "1";
if ((path === "/api/fetch" || path === "/api/upload") && !fromOurScript) return error(403, "API calls only.");
if (path === "/api/config" && isRead) {
return Response.json({ storage: env.storage !== null, storageFull: env.storage?.isFull() ?? false }, { headers: { "cache-control": "no-store" } });
}
+5 -2
View File
@@ -1,4 +1,4 @@
import { CORS_OPEN_HOSTS, TransportError, type Transport } from "../core/transport";
import { API_CALL_HEADER, CORS_OPEN_HOSTS, TransportError, type Transport } from "../core/transport";
type RawFetch = (url: string, init?: RequestInit) => Promise<Response>;
@@ -15,7 +15,10 @@ export function pageTransport(origin: string, rawFetch: RawFetch = (url, init) =
if ((init.method ?? "GET").toUpperCase() !== "GET") {
throw new TransportError(`The page can't ${init.method} to ${target.hostname} (no CORS).`, url);
}
const res = await rawFetch(`${origin}/api/fetch?url=${encodeURIComponent(target.toString())}`, { signal: init.signal });
const res = await rawFetch(`${origin}/api/fetch?url=${encodeURIComponent(target.toString())}`, {
signal: init.signal,
headers: { [API_CALL_HEADER]: "1" },
});
if (res.headers.get("x-mbseed-error")) {
const body = (await res.json().catch(() => ({}))) as { error?: string };
throw new TransportError(`Server proxy: ${body.error ?? `HTTP ${res.status}`}`, url, res.status);
+4
View File
@@ -23,3 +23,7 @@ test("finds tags and meta content by property or name", () => {
test("textOf strips tags and collapses whitespace", () => {
expect(textOf("<span>Tower</span>\n of &amp; God ")).toBe("Tower of & God");
});
test("an out-of-range numeric entity is left as written instead of throwing", () => {
expect(decodeEntities("a &#99999999; b &#x110000; c")).toBe("a &#99999999; b &#x110000; c");
});
+58 -3
View File
@@ -25,7 +25,9 @@ function setup(over: Partial<ServerEnv> = {}, routes: Record<string, () => Respo
upstream,
...over,
});
const call = (path: string, init?: RequestInit) => app(new Request(`${ORIGIN}${path}`, init));
// Our page and the userscript always send x-mbseed: 1 on API calls; a navigation can't.
const call = (path: string, init: RequestInit = {}) =>
app(new Request(`${ORIGIN}${path}`, { ...init, headers: { "x-mbseed": "1", ...(init.headers as Record<string, string> | undefined) } }));
return { app, call, calls };
}
const fetchPath = (url: string) => `/api/fetch?url=${encodeURIComponent(url)}`;
@@ -40,14 +42,13 @@ describe("/api/config", () => {
});
describe("/api/fetch", () => {
test("fetches allowlisted hosts with the right User-Agent and passes status and type through", async () => {
test("fetches allowlisted hosts with the right User-Agent and passes the status through", async () => {
const { call, calls } = setup({}, {
"https://www.webtoons.com/-/-/-/list?title_no=95": () => new Response("<html>", { headers: { "content-type": "text/html" } }),
"https://web.archive.org/cdx/search/cdx?url=x": () => new Response("", { status: 503 }),
});
const page = await call(fetchPath("https://www.webtoons.com/-/-/-/list?title_no=95"));
expect(page.status).toBe(200);
expect(page.headers.get("content-type")).toBe("text/html");
expect(await page.text()).toBe("<html>");
expect(new Headers(calls[0]!.init!.headers).get("user-agent")).toMatch(/^Mozilla/);
expect((await call(fetchPath("https://web.archive.org/cdx/search/cdx?url=x"))).status).toBe(503);
@@ -164,3 +165,57 @@ describe("covers and assets", () => {
expect((await call("/..%2f.env")).status).toBe(404);
});
});
describe("final review fixes", () => {
test("proxied responses can never render on our origin (stored XSS via archived HTML)", async () => {
const { call } = setup({}, { "https://web.archive.org/web/2026id_/https://evil.example/x.html": () =>
new Response("<script>steal(localStorage)</script>", { headers: { "content-type": "text/html" } }) });
const res = await call(fetchPath("https://web.archive.org/web/2026id_/https://evil.example/x.html"), { headers: { "x-mbseed": "1" } });
expect(res.status).toBe(200);
expect(res.headers.get("content-type")).toBe("application/octet-stream");
expect(res.headers.get("x-content-type-options")).toBe("nosniff");
expect(res.headers.get("content-security-policy")).toContain("sandbox");
expect(res.headers.get("content-disposition")).toBe("attachment");
});
test("a plain navigation to /api/fetch or /api/upload is refused (no x-mbseed header)", async () => {
const { app, calls } = setup({}, { [POSTER]: () => new Response("x") });
const plain = (path: string, init?: RequestInit) => app(new Request(`${ORIGIN}${path}`, init)); // no x-mbseed
expect((await plain(fetchPath(POSTER))).status).toBe(403);
expect((await plain("/api/upload", { method: "POST", headers: { "content-type": "text/plain" }, body: JSON.stringify({ url: POSTER, host: "store" }) })).status).toBe(403);
expect(calls).toHaveLength(0);
});
test("uploads only take source image hosts, or their Wayback copies", async () => {
const poster = await fixtureBytes("images/poster-95.jpg");
const archivedOld = "https://web.archive.org/web/20150312154616id_/http://swebtoon.phinf.naver.net/20140710_28/x.jpg";
const { call, calls } = setup({}, {
[POSTER]: () => new Response(poster as Uint8Array<ArrayBuffer>),
[archivedOld]: () => new Response(poster as Uint8Array<ArrayBuffer>),
});
for (const url of [
"https://www.webtoons.com/-/-/-/list?title_no=95",
"https://web.archive.org/web/2026id_/https://evil.example/payload.gif",
"https://web.archive.org/cdx/search/cdx?url=x",
]) {
expect((await call("/api/upload", uploadInit({ url, host: "store" }))).status).toBe(403);
}
expect(calls).toHaveLength(0);
expect((await call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(200);
expect((await call("/api/upload", uploadInit({ url: archivedOld, host: "store" }))).status).toBe(200);
});
test("a GIF header in front of arbitrary bytes is not accepted as an image", async () => {
const fake = new TextEncoder().encode("GIF89a\x00\x00\x00\x00 arbitrary payload ...");
const { call } = setup({}, { [POSTER]: () => new Response(fake) });
expect((await call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(415);
});
test("behind a TLS proxy without MBSEED_PUBLIC_URL, our own https Origin is accepted", async () => {
const app = createApp({ storage: null, publicUrl: null, assets: async () => null, clientIp: () => "1",
upstream: async () => new Response("x") });
const res = await app(new Request(`http://mbseed.example.org${fetchPath(POSTER)}`,
{ headers: { origin: "https://mbseed.example.org", host: "mbseed.example.org", "x-mbseed": "1" } }));
expect(res.status).toBe(200);
});
});
+11
View File
@@ -127,3 +127,14 @@ describe("waybackHistory", () => {
expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(1);
});
});
test("one unreadable snapshot is a warning; the history goes on", async () => {
const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n`;
const page2015 = await fixtureText("wayback/tower-of-god-2015.html");
const broken = () => new Response(new ReadableStream({ start(c) { c.error(new Error("connection reset mid-body")); } }));
const fake = waybackFake(cdx, (u) => (u.includes("/2015") ? broken() : text(page2015)));
const warnings: string[] = [];
const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: (m) => warnings.push(m) });
expect(found.length).toBeGreaterThan(0);
expect(warnings).toEqual([expect.stringContaining("2015-03")]);
});