feat(server): web-standard handler (config, allowlisted fetch proxy, upload, covers, assets)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/server/app.ts

@@ -0,0 +1,187 @@
+// The portable request handler (spec §4.10): web-standard APIs only, so a Workers entry can wrap it later.
+import { uploadToCatbox } from "../core/hosting/catbox";
+import { ImageError, MAX_IMAGE_BYTES, probeImage, readCapped, sha256Hex } from "../core/images";
+import { allowedFetchHosts } from "../core/sources/registry";
+import { TOOL_USER_AGENT, userAgentFor, type Transport } from "../core/transport";
+import { resolveAssetPath } from "./assets-path";
+import { RateLimiter } from "./rate-limit";
+import { CONTENT_TYPES, COVER_NAME, StorageFullError, type Storage } from "./storage";
+
+export type Upstream = (url: string, init?: RequestInit) => Promise<Response>;
+
+export interface ServerEnv {
+  storage: Storage | null;
+  publicUrl: string | null;
+  /** Serves a built file by its path relative to the assets root, or null. */
+  assets(relativePath: string): Promise<Response | null>;
+  clientIp(req: Request): string;
+  upstream?: Upstream;
+  allowedHosts?: ReadonlySet<string>;
+  limits?: { fetch: RateLimiter; upload: RateLimiter };
+}
+
+export const MAX_REDIRECTS = 5;
+const UPSTREAM_TIMEOUT_MS = 30_000;
+/** Pages whose scripts may call our API besides our own (the userscript runs on mangabaka.org). */
+const EXTRA_ORIGINS = new Set(["https://mangabaka.org"]);
+
+const error = (status: number, message: string) =>
+  Response.json({ error: message }, { status, headers: { "x-mbseed-error": "1", "cache-control": "no-store" } });
+
+export function createApp(env: ServerEnv): (req: Request) => Promise<Response> {
+  const hosts = env.allowedHosts ?? allowedFetchHosts();
+  // A run is ~60–100 proxied requests over several minutes (spec §5).
+  const limits = env.limits ?? { fetch: new RateLimiter(240, 60_000), upload: new RateLimiter(60, 60_000) };
+  const upstream: Upstream = env.upstream ?? ((url, init) => fetch(url, init));
+  const ownOrigin = (req: Request) => env.publicUrl ?? new URL(req.url).origin;
+
+  const allowed = (u: URL) =>
+    (u.protocol === "https:" || u.protocol === "http:") && hosts.has(u.hostname) && ["", "80", "443"].includes(u.port);
+
+  /** GET through the allowlist, re-checking every redirect hop. */
+  async function proxyGet(target: string | null): Promise<Response> {
+    if (!target) return error(400, "Missing url.");
+    let url: URL;
+    try {
+      url = new URL(target);
+    } catch {
+      return error(400, "Not a URL.");
+    }
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+      if (!allowed(url)) return error(403, `Host not allowed: ${url.hostname}`);
+      let res: Response;
+      try {
+        res = await upstream(url.toString(), {
+          redirect: "manual",
+          headers: { "user-agent": userAgentFor(url.toString()), accept: "*/*" },
+          signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
+        });
+      } catch (err) {
+        return error(502, `Upstream failed: ${err instanceof Error ? err.message : String(err)}`);
+      }
+      const location = res.headers.get("location");
+      if (res.status >= 300 && res.status < 400 && location) {
+        await res.body?.cancel();
+        url = new URL(location, url);
+        continue;
+      }
+      let body: Uint8Array;
+      try {
+        body = await readCapped(res, MAX_IMAGE_BYTES);
+      } catch (err) {
+        if (err instanceof ImageError) return error(413, "Response larger than 20 MB.");
+        throw err;
+      }
+      return new Response(body as Uint8Array<ArrayBuffer>, {
+        status: res.status,
+        headers: {
+          "content-type": res.headers.get("content-type") ?? "application/octet-stream",
+          "cache-control": "no-store",
+          "x-mbseed-final-url": url.toString(),
+        },
+      });
+    }
+    return error(502, "Too many redirects.");
+  }
+
+  const catboxTransport: Transport = {
+    fetch: (url, init = {}) =>
+      upstream(url, {
+        method: init.method,
+        headers: { "user-agent": TOOL_USER_AGENT, ...init.headers },
+        body: init.body,
+        signal: AbortSignal.timeout(init.timeoutMs ?? UPSTREAM_TIMEOUT_MS),
+      }),
+  };
+
+  async function upload(req: Request): Promise<Response> {
+    let body: { url?: unknown; host?: unknown; userhash?: unknown };
+    try {
+      body = await req.json();
+    } catch {
+      return error(400, "Body must be JSON.");
+    }
+    if (typeof body.url !== "string") return error(400, "url is required.");
+    if (body.host !== "store" && body.host !== "catbox") return error(400, 'host must be "store" or "catbox".');
+    if (body.userhash !== undefined && (typeof body.userhash !== "string" || !/^[A-Za-z0-9]{1,64}$/.test(body.userhash))) {
+      return error(400, "Bad userhash.");
+    }
+    if (body.host === "store" && !env.storage) return error(404, "This server has no storage.");
+
+    // The client sends the source URL, never bytes, so this can't host arbitrary files (spec §4.6).
+    const fetched = await proxyGet(body.url);
+    if (fetched.headers.get("x-mbseed-error")) return fetched;
+    if (!fetched.ok) return error(502, `Could not fetch the image (HTTP ${fetched.status}).`);
+    const bytes = new Uint8Array(await fetched.arrayBuffer());
+    const info = probeImage(bytes);
+    if (!info) return error(415, "That URL is not an image.");
+    const sha256 = await sha256Hex(bytes);
+
+    if (body.host === "store") {
+      const name = `${sha256}.${info.ext}`;
+      try {
+        await env.storage!.put(name, bytes);
+      } catch (err) {
+        if (err instanceof StorageFullError) return error(507, "This server's storage is full; use litterbox or catbox.");
+        throw err;
+      }
+      return Response.json({ url: `${env.publicUrl}/covers/${name}`, sha256 });
+    }
+    try {
+      const blob = new Blob([bytes as Uint8Array<ArrayBuffer>], { type: info.contentType });
+      const { url } = await uploadToCatbox(catboxTransport, blob, `${sha256}.${info.ext}`, body.userhash as string | undefined);
+      return Response.json({ url, sha256 });
+    } catch (err) {
+      return error(502, err instanceof Error ? err.message : "catbox upload failed.");
+    }
+  }
+
+  async function cover(name: string): Promise<Response> {
+    if (!env.storage || !COVER_NAME.test(name)) return error(404, "Not found.");
+    const blob = await env.storage.get(name);
+    if (!blob) return error(404, "Not found.");
+    return new Response(blob, {
+      headers: {
+        "content-type": CONTENT_TYPES[name.split(".").pop()!]!,
+        "cache-control": "public, max-age=86400",
+        "x-content-type-options": "nosniff",
+      },
+    });
+  }
+
+  async function route(req: Request): Promise<Response> {
+    const url = new URL(req.url);
+    const path = url.pathname;
+    const isRead = req.method === "GET" || req.method === "HEAD";
+
+    if (path.startsWith("/api/")) {
+      const origin = req.headers.get("origin");
+      if (origin && origin !== ownOrigin(req) && !EXTRA_ORIGINS.has(origin)) return error(403, "Cross-origin requests are not allowed.");
+      if (path === "/api/config" && isRead) {
+        return Response.json({ storage: env.storage !== null, storageFull: env.storage?.isFull() ?? false }, { headers: { "cache-control": "no-store" } });
+      }
+      if (path === "/api/fetch" && req.method === "GET") {
+        if (!limits.fetch.take(env.clientIp(req))) return error(429, "Too many requests; slow down.");
+        return proxyGet(url.searchParams.get("url"));
+      }
+      if (path === "/api/upload" && req.method === "POST") {
+        if (!limits.upload.take(env.clientIp(req))) return error(429, "Too many uploads; slow down.");
+        return upload(req);
+      }
+      return error(404, "No such API route.");
+    }
+    if (!isRead) return error(405, "Method not allowed.");
+    if (path.startsWith("/covers/")) return cover(path.slice("/covers/".length));
+    const file = resolveAssetPath(path);
+    return (file && (await env.assets(file))) || error(404, "Not found.");
+  }
+
+  return async (req) => {
+    try {
+      return await route(req);
+    } catch (err) {
+      console.error("request failed:", req.method, new URL(req.url).pathname, err);
+      return error(500, "Internal error.");
+    }
+  };
+}

test/server-app.test.ts

@@ -0,0 +1,166 @@
+import { describe, expect, test } from "bun:test";
+import { sha256Hex } from "../src/core/images";
+import { TOOL_USER_AGENT } from "../src/core/transport";
+import { createApp, type ServerEnv, type Upstream } from "../src/server/app";
+import { RateLimiter } from "../src/server/rate-limit";
+import { MemoryStorage } from "../src/server/storage";
+import { fixtureBytes } from "./helpers/fixtures";
+
+const ORIGIN = "https://mbseed.example.org";
+const POSTER = "https://swebtoon-phinf.pstatic.net/20250204_46/17386458444547o1b2_JPEG/95.jpg";
+
+function setup(over: Partial<ServerEnv> = {}, routes: Record<string, () => Response | Promise<Response>> = {}) {
+  const calls: { url: string; init?: RequestInit }[] = [];
+  const upstream: Upstream = async (url, init) => {
+    calls.push({ url, init });
+    const route = routes[url];
+    if (!route) throw new Error(`unexpected upstream ${url}`);
+    return route();
+  };
+  const app = createApp({
+    storage: new MemoryStorage(10 * 1024 * 1024),
+    publicUrl: ORIGIN,
+    assets: async (path) => (path === "index.html" ? new Response("<!doctype html>page", { headers: { "content-type": "text/html" } }) : null),
+    clientIp: () => "203.0.113.9",
+    upstream,
+    ...over,
+  });
+  const call = (path: string, init?: RequestInit) => app(new Request(`${ORIGIN}${path}`, init));
+  return { app, call, calls };
+}
+const fetchPath = (url: string) => `/api/fetch?url=${encodeURIComponent(url)}`;
+const uploadInit = (body: unknown): RequestInit => ({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
+
+describe("/api/config", () => {
+  test("reports storage and whether it is full", async () => {
+    expect(await (await setup().call("/api/config")).json()).toEqual({ storage: true, storageFull: false });
+    expect(await (await setup({ storage: null }).call("/api/config")).json()).toEqual({ storage: false, storageFull: false });
+    expect(await (await setup({ storage: new MemoryStorage(10) }).call("/api/config")).json()).toEqual({ storage: true, storageFull: true });
+  });
+});
+
+describe("/api/fetch", () => {
+  test("fetches allowlisted hosts with the right User-Agent and passes status and type through", async () => {
+    const { call, calls } = setup({}, {
+      "https://www.webtoons.com/-/-/-/list?title_no=95": () => new Response("<html>", { headers: { "content-type": "text/html" } }),
+      "https://web.archive.org/cdx/search/cdx?url=x": () => new Response("", { status: 503 }),
+    });
+    const page = await call(fetchPath("https://www.webtoons.com/-/-/-/list?title_no=95"));
+    expect(page.status).toBe(200);
+    expect(page.headers.get("content-type")).toBe("text/html");
+    expect(await page.text()).toBe("<html>");
+    expect(new Headers(calls[0]!.init!.headers).get("user-agent")).toMatch(/^Mozilla/);
+    expect((await call(fetchPath("https://web.archive.org/cdx/search/cdx?url=x"))).status).toBe(503);
+    expect(new Headers(calls[1]!.init!.headers).get("user-agent")).toBe(TOOL_USER_AGENT);
+  });
+
+  test.each(["https://evil.example/x", "http://169.254.169.254/latest", "ftp://www.webtoons.com/x", "https://www.webtoons.com:8443/x", "not a url"])(
+    "%p is refused without being fetched", async (url) => {
+      const { call, calls } = setup();
+      expect([400, 403]).toContain((await call(fetchPath(url))).status);
+      expect(calls).toHaveLength(0);
+    });
+
+  test("follows redirects within the allowlist", async () => {
+    const { call } = setup({}, {
+      "https://www.webtoons.com/-/-/-/list?title_no=95": () => new Response(null, { status: 301, headers: { location: "/en/fantasy/tower-of-god/list?title_no=95" } }),
+      "https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95": () => new Response("canonical"),
+    });
+    expect(await (await call(fetchPath("https://www.webtoons.com/-/-/-/list?title_no=95"))).text()).toBe("canonical");
+  });
+
+  test("redirect to a non-allowlisted host is refused", async () => {
+    // Review Focus 2
+    const { call, calls } = setup({}, {
+      "https://www.webtoons.com/x": () => new Response(null, { status: 302, headers: { location: "https://evil.example/" } }),
+    });
+    expect((await call(fetchPath("https://www.webtoons.com/x"))).status).toBe(403);
+    expect(calls.map((c) => c.url)).toEqual(["https://www.webtoons.com/x"]);
+  });
+
+  test("responses over 20 MB are refused", async () => {
+    const { call } = setup({}, { [POSTER]: () => new Response("x", { headers: { "content-length": String(21 * 1024 * 1024) } }) });
+    expect((await call(fetchPath(POSTER))).status).toBe(413);
+  });
+
+  test("rejects other origins; allows ours, mangabaka.org and none", async () => {
+    const { call } = setup({}, { [POSTER]: () => new Response("x") });
+    expect((await call(fetchPath(POSTER), { headers: { origin: "https://evil.example" } })).status).toBe(403);
+    expect((await call(fetchPath(POSTER), { headers: { origin: ORIGIN } })).status).toBe(200);
+    expect((await call(fetchPath(POSTER), { headers: { origin: "https://mangabaka.org" } })).status).toBe(200);
+    expect((await call(fetchPath(POSTER))).status).toBe(200);
+  });
+
+  test("rate-limited per IP", async () => {
+    const { call } = setup({ limits: { fetch: new RateLimiter(1, 60_000), upload: new RateLimiter(1, 60_000) } }, { [POSTER]: () => new Response("x") });
+    expect((await call(fetchPath(POSTER))).status).toBe(200);
+    expect((await call(fetchPath(POSTER))).status).toBe(429);
+  });
+
+  test("proxy errors are marked so the page can tell them from upstream answers", async () => {
+    const res = await setup().call(fetchPath("https://evil.example/"));
+    expect(res.headers.get("x-mbseed-error")).toBe("1");
+    expect(await res.json()).toEqual({ error: expect.stringContaining("evil.example") });
+  });
+});
+
+describe("/api/upload", () => {
+  test("store: fetches the source itself, keeps it by hash, returns its public URL", async () => {
+    const poster = await fixtureBytes("images/poster-95.jpg");
+    const sha = await sha256Hex(poster);
+    const { call } = setup({}, { [POSTER]: () => new Response(poster as Uint8Array<ArrayBuffer>) });
+    const res = await call("/api/upload", uploadInit({ url: POSTER, host: "store" }));
+    expect(await res.json()).toEqual({ url: `${ORIGIN}/covers/${sha}.jpg`, sha256: sha });
+    const cover = await call(`/covers/${sha}.jpg`);
+    expect(cover.headers.get("content-type")).toBe("image/jpeg");
+    expect((await cover.arrayBuffer()).byteLength).toBe(poster.byteLength);
+  });
+
+  test("store full: 507", async () => {
+    const poster = await fixtureBytes("images/poster-95.jpg");
+    const { call } = setup({ storage: new MemoryStorage(1000) }, { [POSTER]: () => new Response(poster as Uint8Array<ArrayBuffer>) });
+    expect((await call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(507);
+  });
+
+  test("catbox: forwards the userhash to catbox only", async () => {
+    const poster = await fixtureBytes("images/poster-95.jpg");
+    const { call, calls } = setup({}, {
+      [POSTER]: () => new Response(poster as Uint8Array<ArrayBuffer>),
+      "https://catbox.moe/user/api.php": () => new Response("https://files.catbox.moe/abc.jpg"),
+    });
+    const res = await call("/api/upload", uploadInit({ url: POSTER, host: "catbox", userhash: "abc123" }));
+    expect((await res.json()).url).toBe("https://files.catbox.moe/abc.jpg");
+    expect((calls[1]!.init!.body as FormData).get("userhash")).toBe("abc123");
+  });
+
+  test.each([
+    [{ url: POSTER, host: "imgur" }, 400],
+    [{ host: "store" }, 400],
+    [{ url: "https://evil.example/x.jpg", host: "store" }, 403],
+    [{ url: POSTER, host: "catbox", userhash: "../../x" }, 400],
+  ])("%o → %i", async (body, status) => {
+    expect((await setup().call("/api/upload", uploadInit(body))).status).toBe(status);
+  });
+
+  test("no storage configured: 404 for store", async () => {
+    expect((await setup({ storage: null }).call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(404);
+  });
+
+  test("not an image: 415", async () => {
+    const { call } = setup({}, { [POSTER]: () => new Response("<html>image not found</html>") });
+    expect((await call("/api/upload", uploadInit({ url: POSTER, host: "store" }))).status).toBe(415);
+  });
+});
+
+describe("covers and assets", () => {
+  test.each(["/covers/../../x", `/covers/${"A".repeat(64)}.jpg`, `/covers/${"a".repeat(64)}.svg`, "/covers/"])("%p → 404", async (path) => {
+    expect((await setup().call(path)).status).toBe(404);
+  });
+  test("static assets via env.assets; unknown paths 404; writes 405", async () => {
+    const { call } = setup();
+    expect(await (await call("/")).text()).toBe("<!doctype html>page");
+    expect((await call("/nope.js")).status).toBe(404);
+    expect((await call("/", { method: "POST" })).status).toBe(405);
+    expect((await call("/..%2f.env")).status).toBe(404);
+  });
+});