From 2debb3d513fac6dfc1951160cb2633e7bd778b49 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Wed, 14 Oct 2026 23:39:19 +0200 Subject: [PATCH] feat(server): web-standard handler (config, allowlisted fetch proxy, upload, covers, assets) Co-Authored-By: Claude Opus 5.5 --- src/server/app.ts | 187 ++++++++++++++++++++++++++++++++++++++++ test/server-app.test.ts | 166 +++++++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 src/server/app.ts create mode 100644 test/server-app.test.ts diff --git a/src/server/app.ts b/src/server/app.ts new file mode 100644 index 0000000..1f571a3 --- /dev/null +++ b/src/server/app.ts @@ -0,0 +1,187 @@ +// The portable request handler (spec §4.10): web-standard APIs only, so a Workers entry can wrap it later. +import { uploadToCatbox } from "../core/hosting/catbox"; +import { ImageError, MAX_IMAGE_BYTES, probeImage, readCapped, sha256Hex } from "../core/images"; +import { allowedFetchHosts } from "../core/sources/registry"; +import { TOOL_USER_AGENT, userAgentFor, type Transport } from "../core/transport"; +import { resolveAssetPath } from "./assets-path"; +import { RateLimiter } from "./rate-limit"; +import { CONTENT_TYPES, COVER_NAME, StorageFullError, type Storage } from "./storage"; + +export type Upstream = (url: string, init?: RequestInit) => Promise; + +export interface ServerEnv { + storage: Storage | null; + publicUrl: string | null; + /** Serves a built file by its path relative to the assets root, or null. */ + assets(relativePath: string): Promise; + clientIp(req: Request): string; + upstream?: Upstream; + allowedHosts?: ReadonlySet; + limits?: { fetch: RateLimiter; upload: RateLimiter }; +} + +export const MAX_REDIRECTS = 5; +const UPSTREAM_TIMEOUT_MS = 30_000; +/** Pages whose scripts may call our API besides our own (the userscript runs on mangabaka.org). */ +const EXTRA_ORIGINS = new Set(["https://mangabaka.org"]); + +const error = (status: number, message: string) => + Response.json({ error: message }, { status, headers: { "x-mbseed-error": "1", "cache-control": "no-store" } }); + +export function createApp(env: ServerEnv): (req: Request) => Promise { + const hosts = env.allowedHosts ?? allowedFetchHosts(); + // A run is ~60–100 proxied requests over several minutes (spec §5). + const limits = env.limits ?? { fetch: new RateLimiter(240, 60_000), upload: new RateLimiter(60, 60_000) }; + const upstream: Upstream = env.upstream ?? ((url, init) => fetch(url, init)); + const ownOrigin = (req: Request) => env.publicUrl ?? new URL(req.url).origin; + + const allowed = (u: URL) => + (u.protocol === "https:" || u.protocol === "http:") && hosts.has(u.hostname) && ["", "80", "443"].includes(u.port); + + /** GET through the allowlist, re-checking every redirect hop. */ + async function proxyGet(target: string | null): Promise { + if (!target) return error(400, "Missing url."); + let url: URL; + try { + url = new URL(target); + } catch { + return error(400, "Not a URL."); + } + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + if (!allowed(url)) return error(403, `Host not allowed: ${url.hostname}`); + let res: Response; + try { + res = await upstream(url.toString(), { + redirect: "manual", + headers: { "user-agent": userAgentFor(url.toString()), accept: "*/*" }, + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + } catch (err) { + return error(502, `Upstream failed: ${err instanceof Error ? err.message : String(err)}`); + } + const location = res.headers.get("location"); + if (res.status >= 300 && res.status < 400 && location) { + await res.body?.cancel(); + url = new URL(location, url); + continue; + } + let body: Uint8Array; + try { + body = await readCapped(res, MAX_IMAGE_BYTES); + } catch (err) { + if (err instanceof ImageError) return error(413, "Response larger than 20 MB."); + throw err; + } + return new Response(body as Uint8Array, { + status: res.status, + headers: { + "content-type": res.headers.get("content-type") ?? "application/octet-stream", + "cache-control": "no-store", + "x-mbseed-final-url": url.toString(), + }, + }); + } + return error(502, "Too many redirects."); + } + + const catboxTransport: Transport = { + fetch: (url, init = {}) => + upstream(url, { + method: init.method, + headers: { "user-agent": TOOL_USER_AGENT, ...init.headers }, + body: init.body, + signal: AbortSignal.timeout(init.timeoutMs ?? UPSTREAM_TIMEOUT_MS), + }), + }; + + async function upload(req: Request): Promise { + let body: { url?: unknown; host?: unknown; userhash?: unknown }; + try { + body = await req.json(); + } catch { + return error(400, "Body must be JSON."); + } + if (typeof body.url !== "string") return error(400, "url is required."); + if (body.host !== "store" && body.host !== "catbox") return error(400, 'host must be "store" or "catbox".'); + if (body.userhash !== undefined && (typeof body.userhash !== "string" || !/^[A-Za-z0-9]{1,64}$/.test(body.userhash))) { + return error(400, "Bad userhash."); + } + if (body.host === "store" && !env.storage) return error(404, "This server has no storage."); + + // The client sends the source URL, never bytes, so this can't host arbitrary files (spec §4.6). + const fetched = await proxyGet(body.url); + if (fetched.headers.get("x-mbseed-error")) return fetched; + if (!fetched.ok) return error(502, `Could not fetch the image (HTTP ${fetched.status}).`); + const bytes = new Uint8Array(await fetched.arrayBuffer()); + const info = probeImage(bytes); + if (!info) return error(415, "That URL is not an image."); + const sha256 = await sha256Hex(bytes); + + if (body.host === "store") { + const name = `${sha256}.${info.ext}`; + try { + await env.storage!.put(name, bytes); + } catch (err) { + if (err instanceof StorageFullError) return error(507, "This server's storage is full; use litterbox or catbox."); + throw err; + } + return Response.json({ url: `${env.publicUrl}/covers/${name}`, sha256 }); + } + try { + const blob = new Blob([bytes as Uint8Array], { type: info.contentType }); + const { url } = await uploadToCatbox(catboxTransport, blob, `${sha256}.${info.ext}`, body.userhash as string | undefined); + return Response.json({ url, sha256 }); + } catch (err) { + return error(502, err instanceof Error ? err.message : "catbox upload failed."); + } + } + + async function cover(name: string): Promise { + if (!env.storage || !COVER_NAME.test(name)) return error(404, "Not found."); + const blob = await env.storage.get(name); + if (!blob) return error(404, "Not found."); + return new Response(blob, { + headers: { + "content-type": CONTENT_TYPES[name.split(".").pop()!]!, + "cache-control": "public, max-age=86400", + "x-content-type-options": "nosniff", + }, + }); + } + + async function route(req: Request): Promise { + const url = new URL(req.url); + const path = url.pathname; + const isRead = req.method === "GET" || req.method === "HEAD"; + + if (path.startsWith("/api/")) { + const origin = req.headers.get("origin"); + if (origin && origin !== ownOrigin(req) && !EXTRA_ORIGINS.has(origin)) return error(403, "Cross-origin requests are not allowed."); + if (path === "/api/config" && isRead) { + return Response.json({ storage: env.storage !== null, storageFull: env.storage?.isFull() ?? false }, { headers: { "cache-control": "no-store" } }); + } + if (path === "/api/fetch" && req.method === "GET") { + if (!limits.fetch.take(env.clientIp(req))) return error(429, "Too many requests; slow down."); + return proxyGet(url.searchParams.get("url")); + } + if (path === "/api/upload" && req.method === "POST") { + if (!limits.upload.take(env.clientIp(req))) return error(429, "Too many uploads; slow down."); + return upload(req); + } + return error(404, "No such API route."); + } + if (!isRead) return error(405, "Method not allowed."); + if (path.startsWith("/covers/")) return cover(path.slice("/covers/".length)); + const file = resolveAssetPath(path); + return (file && (await env.assets(file))) || error(404, "Not found."); + } + + return async (req) => { + try { + return await route(req); + } catch (err) { + console.error("request failed:", req.method, new URL(req.url).pathname, err); + return error(500, "Internal error."); + } + }; +} diff --git a/test/server-app.test.ts b/test/server-app.test.ts new file mode 100644 index 0000000..c46c06d --- /dev/null +++ b/test/server-app.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, test } from "bun:test"; +import { sha256Hex } from "../src/core/images"; +import { TOOL_USER_AGENT } from "../src/core/transport"; +import { createApp, type ServerEnv, type Upstream } from "../src/server/app"; +import { RateLimiter } from "../src/server/rate-limit"; +import { MemoryStorage } from "../src/server/storage"; +import { fixtureBytes } from "./helpers/fixtures"; + +const ORIGIN = "https://mbseed.example.org"; +const POSTER = "https://swebtoon-phinf.pstatic.net/20250204_46/17386458444547o1b2_JPEG/95.jpg"; + +function setup(over: Partial = {}, routes: Record Response | Promise> = {}) { + const calls: { url: string; init?: RequestInit }[] = []; + const upstream: Upstream = async (url, init) => { + calls.push({ url, init }); + const route = routes[url]; + if (!route) throw new Error(`unexpected upstream ${url}`); + return route(); + }; + const app = createApp({ + storage: new MemoryStorage(10 * 1024 * 1024), + publicUrl: ORIGIN, + assets: async (path) => (path === "index.html" ? new Response("page", { headers: { "content-type": "text/html" } }) : null), + clientIp: () => "203.0.113.9", + upstream, + ...over, + }); + const call = (path: string, init?: RequestInit) => app(new Request(`${ORIGIN}${path}`, init)); + return { app, call, calls }; +} +const fetchPath = (url: string) => `/api/fetch?url=${encodeURIComponent(url)}`; +const uploadInit = (body: unknown): RequestInit => ({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + +describe("/api/config", () => { + test("reports storage and whether it is full", async () => { + expect(await (await setup().call("/api/config")).json()).toEqual({ storage: true, storageFull: false }); + expect(await (await setup({ storage: null }).call("/api/config")).json()).toEqual({ storage: false, storageFull: false }); + expect(await (await setup({ storage: new MemoryStorage(10) }).call("/api/config")).json()).toEqual({ storage: true, storageFull: true }); + }); +}); + +describe("/api/fetch", () => { + test("fetches allowlisted hosts with the right User-Agent and passes status and type through", async () => { + const { call, calls } = setup({}, { + "https://www.webtoons.com/-/-/-/list?title_no=95": () => new Response("", { headers: { "content-type": "text/html" } }), + "https://web.archive.org/cdx/search/cdx?url=x": () => new Response("", { status: 503 }), + }); + const page = await call(fetchPath("https://www.webtoons.com/-/-/-/list?title_no=95")); + expect(page.status).toBe(200); + expect(page.headers.get("content-type")).toBe("text/html"); + expect(await page.text()).toBe(""); + expect(new Headers(calls[0]!.init!.headers).get("user-agent")).toMatch(/^Mozilla/); + expect((await call(fetchPath("https://web.archive.org/cdx/search/cdx?url=x"))).status).toBe(503); + expect(new Headers(calls[1]!.init!.headers).get("user-agent")).toBe(TOOL_USER_AGENT); + }); + + test.each(["https://evil.example/x", "http://169.254.169.254/latest", "ftp://www.webtoons.com/x", "https://www.webtoons.com:8443/x", "not a url"])( + "%p is refused without being fetched", async (url) => { + const { call, calls } = setup(); + expect([400, 403]).toContain((await call(fetchPath(url))).status); + expect(calls).toHaveLength(0); + }); + + test("follows redirects within the allowlist", async () => { + const { call } = setup({}, { + "https://www.webtoons.com/-/-/-/list?title_no=95": () => new Response(null, { status: 301, headers: { location: "/en/fantasy/tower-of-god/list?title_no=95" } }), + "https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95": () => new Response("canonical"), + }); + expect(await (await call(fetchPath("https://www.webtoons.com/-/-/-/list?title_no=95"))).text()).toBe("canonical"); + }); + + test("redirect to a non-allowlisted host is refused", async () => { + // Review Focus 2 + const { call, calls } = setup({}, { + "https://www.webtoons.com/x": () => new Response(null, { status: 302, headers: { location: "https://evil.example/" } }), + }); + expect((await call(fetchPath("https://www.webtoons.com/x"))).status).toBe(403); + expect(calls.map((c) => c.url)).toEqual(["https://www.webtoons.com/x"]); + }); + + test("responses over 20 MB are refused", async () => { + const { call } = setup({}, { [POSTER]: () => new Response("x", { headers: { "content-length": String(21 * 1024 * 1024) } }) }); + expect((await call(fetchPath(POSTER))).status).toBe(413); + }); + + test("rejects other origins; allows ours, mangabaka.org and none", async () => { + const { call } = setup({}, { [POSTER]: () => new Response("x") }); + expect((await call(fetchPath(POSTER), { headers: { origin: "https://evil.example" } })).status).toBe(403); + expect((await call(fetchPath(POSTER), { headers: { origin: ORIGIN } })).status).toBe(200); + expect((await call(fetchPath(POSTER), { headers: { origin: "https://mangabaka.org" } })).status).toBe(200); + expect((await call(fetchPath(POSTER))).status).toBe(200); + }); + + test("rate-limited per IP", async () => { + const { call } = setup({ limits: { fetch: new RateLimiter(1, 60_000), upload: new RateLimiter(1, 60_000) } }, { [POSTER]: () => new Response("x") }); + expect((await call(fetchPath(POSTER))).status).toBe(200); + expect((await call(fetchPath(POSTER))).status).toBe(429); + }); + + test("proxy errors are marked so the page can tell them from upstream answers", async () => { + const res = await setup().call(fetchPath("https://evil.example/")); + expect(res.headers.get("x-mbseed-error")).toBe("1"); + expect(await res.json()).toEqual({ error: expect.stringContaining("evil.example") }); + }); +}); + +describe("/api/upload", () => { + test("store: fetches the source itself, keeps it by hash, returns its public URL", async () => { + const poster = await fixtureBytes("images/poster-95.jpg"); + const sha = await sha256Hex(poster); + const { call } = setup({}, { [POSTER]: () => new Response(poster as Uint8Array) }); + const res = await call("/api/upload", uploadInit({ url: POSTER, host: "store" })); + expect(await res.json()).toEqual({ url: `${ORIGIN}/covers/${sha}.jpg`, sha256: sha }); + const cover = await call(`/covers/${sha}.jpg`); + expect(cover.headers.get("content-type")).toBe("image/jpeg"); + expect((await cover.arrayBuffer()).byteLength).toBe(poster.byteLength); + }); + + test("store full: 507", async () => { + const poster = await fixtureBytes("images/poster-95.jpg"); + const { call } = setup({ storage: new MemoryStorage(1000) }, { [POSTER]: () => new Response(poster as Uint8Array) }); + expect((await call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(507); + }); + + test("catbox: forwards the userhash to catbox only", async () => { + const poster = await fixtureBytes("images/poster-95.jpg"); + const { call, calls } = setup({}, { + [POSTER]: () => new Response(poster as Uint8Array), + "https://catbox.moe/user/api.php": () => new Response("https://files.catbox.moe/abc.jpg"), + }); + const res = await call("/api/upload", uploadInit({ url: POSTER, host: "catbox", userhash: "abc123" })); + expect((await res.json()).url).toBe("https://files.catbox.moe/abc.jpg"); + expect((calls[1]!.init!.body as FormData).get("userhash")).toBe("abc123"); + }); + + test.each([ + [{ url: POSTER, host: "imgur" }, 400], + [{ host: "store" }, 400], + [{ url: "https://evil.example/x.jpg", host: "store" }, 403], + [{ url: POSTER, host: "catbox", userhash: "../../x" }, 400], + ])("%o → %i", async (body, status) => { + expect((await setup().call("/api/upload", uploadInit(body))).status).toBe(status); + }); + + test("no storage configured: 404 for store", async () => { + expect((await setup({ storage: null }).call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(404); + }); + + test("not an image: 415", async () => { + const { call } = setup({}, { [POSTER]: () => new Response("image not found") }); + expect((await call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(415); + }); +}); + +describe("covers and assets", () => { + test.each(["/covers/../../x", `/covers/${"A".repeat(64)}.jpg`, `/covers/${"a".repeat(64)}.svg`, "/covers/"])("%p → 404", async (path) => { + expect((await setup().call(path)).status).toBe(404); + }); + test("static assets via env.assets; unknown paths 404; writes 405", async () => { + const { call } = setup(); + expect(await (await call("/")).text()).toBe("page"); + expect((await call("/nope.js")).status).toBe(404); + expect((await call("/", { method: "POST" })).status).toBe(405); + expect((await call("/..%2f.env")).status).toBe(404); + }); +});