feat(core): model types and transport policy (pacing, retries, timeouts, token scope)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/core/model.ts

@@ -0,0 +1,44 @@
+export type CoverRole = "poster" | "square" | "banner" | "foreground" | "other";
+export type ImageType = "volume" | "volume_back" | "chapter" | "season" | "banner" | "audiobook" | "other";
+export type ContentRating = "safe" | "suggestive" | "erotica" | "pornographic";
+export const CONTENT_RATINGS: readonly ContentRating[] = ["safe", "suggestive", "erotica", "pornographic"];
+export type Shape = "tall" | "portrait" | "square" | "wide" | "other";
+export type ImageExt = "jpg" | "png" | "gif" | "webp";
+/** Wayback history: one snapshot per this many months (spec §4.3). */
+export type Granularity = 1 | 3 | 6 | 12;
+export const GRANULARITIES: readonly Granularity[] = [1, 3, 6, 12];
+
+export interface SourceSeries {
+  source: string;          // Source.id
+  externalId: string;      // e.g. Webtoons title_no "95"
+  title: string;
+  language: string;        // MangaBaka language code, e.g. "en", "zh-Hant", or "unknown"
+  canonicalUrl: string;
+  archiveUrls: string[];   // URL prefixes (no scheme) to look up in Wayback
+  mature: boolean;
+  flags: Record<string, boolean>; // e.g. { canvas: true }
+}
+
+export interface CoverCandidate {
+  url: string;             // normalised, hotlinkable
+  role: CoverRole;
+  current: boolean;        // shown on the live page now
+  seenAt?: string;         // earliest archive timestamp (YYYYMMDDhhmmss) if historical
+  lastSeenAt?: string;     // latest archive timestamp after merging
+  originalUrl?: string;    // as written in the page (absolute), before normalisation
+  archivedUrl?: string;    // Wayback copy of the image itself
+}
+
+export interface InspectedCover {
+  candidate: CoverCandidate;
+  fetchedFrom: string;     // URL that served the bytes (candidate.url, or archivedUrl)
+  live: boolean;           // true if fetchedFrom === candidate.url
+  width: number;
+  height: number;
+  sha256: string;          // lowercase hex
+  contentType: string;     // image/jpeg | image/png | image/gif | image/webp
+  ext: ImageExt;
+  bytes: Uint8Array;
+  shape: Shape;
+  type: ImageType;         // suggested MangaBaka type
+}

src/core/transport.ts

@@ -0,0 +1,140 @@
+import { REPO_URL, VERSION } from "./version";
+
+export interface FetchInit {
+  method?: string;
+  headers?: Record<string, string>;
+  body?: BodyInit | null;
+  signal?: AbortSignal;
+  /** Overrides the policy timeout for this one request (uploads). */
+  timeoutMs?: number;
+}
+
+/** How a front end reaches the network (spec §4.2). Returns a standard Response. */
+export interface Transport {
+  fetch(url: string, init?: FetchInit): Promise<Response>;
+}
+
+export const MANGABAKA_API_HOST = "api.mangabaka.org";
+export const WAYBACK_HOST = "web.archive.org";
+/** Hosts a browser page may call directly because they send CORS headers (spec §2.3, §2.4). */
+export const CORS_OPEN_HOSTS: ReadonlySet<string> = new Set([MANGABAKA_API_HOST, "litterbox.catbox.moe"]);
+
+export const BROWSER_USER_AGENT =
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36";
+export const TOOL_USER_AGENT = `mangabaka-seeder/${VERSION} (+${REPO_URL})`;
+
+/** Wayback flags fake browser User-Agents as bots (spec §2.2); sources get a browser one. */
+export function userAgentFor(url: string): string {
+  return hostOf(url) === WAYBACK_HOST ? TOOL_USER_AGENT : BROWSER_USER_AGENT;
+}
+
+export function hostOf(url: string): string {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return "";
+  }
+}
+
+export class TransportError extends Error {
+  constructor(message: string, readonly url: string, readonly status?: number) {
+    super(message);
+    this.name = "TransportError";
+  }
+}
+
+export interface PolicyOptions {
+  /** Minimum gap between requests to one host. Default: 10 s for Wayback, 1 s otherwise. */
+  minIntervalMs?: (host: string) => number;
+  timeoutMs?: number;
+  retries?: number;
+  backoffMs?: number;
+  now?: () => number;
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+}
+
+export const defaultInterval = (host: string): number => (host === WAYBACK_HOST ? 10_000 : 1_000);
+
+export function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) return reject(signal.reason);
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal!.reason);
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+export function assertTokenScope(host: string, headers?: Record<string, string>): void {
+  if (!headers) return;
+  const carriesToken = Object.keys(headers).some((k) => ["x-api-key", "authorization"].includes(k.toLowerCase()));
+  if (carriesToken && host !== MANGABAKA_API_HOST) {
+    throw new TransportError(`refusing to send credentials to ${host || "an unknown host"}`, host);
+  }
+}
+
+const RETRYABLE_METHODS = new Set(["GET", "HEAD"]);
+const isRetryableStatus = (status: number) => status === 429 || status >= 500;
+
+/** Wraps any transport with the shared behaviour of spec §4.2. */
+export function withPolicy(inner: Transport, opts: PolicyOptions = {}): Transport {
+  const interval = opts.minIntervalMs ?? defaultInterval;
+  const timeoutMs = opts.timeoutMs ?? 20_000;
+  const retries = opts.retries ?? 2;
+  const backoffMs = opts.backoffMs ?? 2_000;
+  const now = opts.now ?? Date.now;
+  const sleep = opts.sleep ?? abortableSleep;
+  const nextSlot = new Map<string, number>();
+
+  async function waitTurn(host: string, signal?: AbortSignal) {
+    const t = now();
+    const at = Math.max(t, nextSlot.get(host) ?? 0);
+    nextSlot.set(host, at + interval(host));
+    if (at > t) await sleep(at - t, signal);
+  }
+
+  function attempt(url: string, init: FetchInit): Promise<Response> {
+    const { timeoutMs: perRequest, ...rest } = init;
+    const timeout = AbortSignal.timeout(perRequest ?? timeoutMs);
+    const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout;
+    return inner.fetch(url, { ...rest, signal });
+  }
+
+  return {
+    async fetch(url, init = {}) {
+      const host = hostOf(url);
+      assertTokenScope(host, init.headers);
+      const method = (init.method ?? "GET").toUpperCase();
+      const attempts = RETRYABLE_METHODS.has(method) ? retries + 1 : 1;
+      let lastError: unknown;
+      for (let i = 0; i < attempts; i++) {
+        await waitTurn(host, init.signal);
+        try {
+          const res = await attempt(url, init);
+          if (!isRetryableStatus(res.status) || i === attempts - 1) return res;
+          await res.body?.cancel();
+          await sleep(retryDelay(res, backoffMs, i), init.signal);
+        } catch (err) {
+          if (init.signal?.aborted) throw err;
+          lastError = err;
+          if (i < attempts - 1) await sleep(backoffMs * 2 ** i, init.signal);
+        }
+      }
+      throw new TransportError(`request to ${host} failed: ${describe(lastError)}`, url);
+    },
+  };
+}
+
+function retryDelay(res: Response, base: number, attempt: number): number {
+  const seconds = Number(res.headers.get("retry-after"));
+  return Number.isFinite(seconds) && seconds > 0 ? Math.min(seconds, 120) * 1000 : base * 2 ** attempt;
+}
+
+function describe(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}

test/helpers/fake-transport.ts

@@ -0,0 +1,35 @@
+import type { FetchInit, Transport } from "../../src/core/transport";
+
+type Matcher = string | RegExp | ((url: string, init: FetchInit) => boolean);
+type Responder = (url: string, init: FetchInit) => Response | Promise<Response>;
+
+/** Serves canned responses; throws on any request without a route, so tests can't hit the network. */
+export class FakeTransport implements Transport {
+  readonly calls: { url: string; init: FetchInit }[] = [];
+  private readonly routes: { match: Matcher; respond: Responder }[] = [];
+
+  on(match: Matcher, respond: Responder): this {
+    this.routes.push({ match, respond });
+    return this;
+  }
+
+  async fetch(url: string, init: FetchInit = {}): Promise<Response> {
+    this.calls.push({ url, init });
+    for (const { match, respond } of this.routes) {
+      const hit = typeof match === "string" ? match === url : match instanceof RegExp ? match.test(url) : match(url, init);
+      if (hit) return respond(url, init);
+    }
+    throw new Error(`FakeTransport: no route for ${init.method ?? "GET"} ${url}`);
+  }
+
+  urls(): string[] {
+    return this.calls.map((c) => c.url);
+  }
+}
+
+export const text = (body: string, status = 200, headers: Record<string, string> = {}) =>
+  new Response(body, { status, headers: { "content-type": "text/html; charset=utf-8", ...headers } });
+export const json = (body: unknown, status = 200) =>
+  new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
+export const bytes = (body: Uint8Array, contentType = "image/jpeg", status = 200) =>
+  new Response(body as Uint8Array<ArrayBuffer>, { status, headers: { "content-type": contentType } });

test/transport.test.ts

@@ -0,0 +1,104 @@
+import { describe, expect, test } from "bun:test";
+import {
+  TOOL_USER_AGENT, TransportError, userAgentFor, withPolicy, type PolicyOptions,
+} from "../src/core/transport";
+import { FakeTransport, text } from "./helpers/fake-transport";
+
+function fakeClock() {
+  let now = 0;
+  const slept: number[] = [];
+  const opts: PolicyOptions = {
+    now: () => now,
+    sleep: async (ms) => { slept.push(ms); now += ms; },
+  };
+  return { opts, slept, advance: (ms: number) => { now += ms; } };
+}
+
+describe("withPolicy", () => {
+  test("spaces requests to the same host, not across hosts", async () => {
+    const fake = new FakeTransport().on(() => true, () => text("ok"));
+    const clock = fakeClock();
+    const t = withPolicy(fake, clock.opts);
+    await t.fetch("https://www.webtoons.com/a");
+    await t.fetch("https://www.webtoons.com/b");
+    await t.fetch("https://swebtoon-phinf.pstatic.net/c");
+    await t.fetch("https://www.webtoons.com/d");
+    expect(clock.slept).toEqual([1000, 1000]);
+  });
+
+  test("waits 10 s between Wayback requests", async () => {
+    const fake = new FakeTransport().on(() => true, () => text("ok"));
+    const clock = fakeClock();
+    const t = withPolicy(fake, clock.opts);
+    await t.fetch("https://web.archive.org/a");
+    await t.fetch("https://web.archive.org/b");
+    expect(clock.slept).toEqual([10_000]);
+  });
+
+  test("retries GET on 429/5xx, honouring Retry-After", async () => {
+    let n = 0;
+    const fake = new FakeTransport().on(() => true, () =>
+      ++n === 1 ? text("slow down", 429, { "retry-after": "7" }) : n === 2 ? text("oops", 503) : text("ok"));
+    const clock = fakeClock();
+    const res = await withPolicy(fake, { ...clock.opts, minIntervalMs: () => 0 }).fetch("https://www.webtoons.com/x");
+    expect(res.status).toBe(200);
+    expect(fake.calls.length).toBe(3);
+    expect(clock.slept).toEqual([7000, 4000]); // Retry-After, then backoff 2000 * 2^1
+  });
+
+  test("returns the last 5xx after the retries are used up", async () => {
+    const fake = new FakeTransport().on(() => true, () => text("down", 502));
+    const clock = fakeClock();
+    const res = await withPolicy(fake, { ...clock.opts, minIntervalMs: () => 0 }).fetch("https://www.webtoons.com/x");
+    expect(res.status).toBe(502);
+    expect(fake.calls.length).toBe(3);
+  });
+
+  test("never retries POST", async () => {
+    const fake = new FakeTransport().on(() => true, () => text("bad gateway", 502));
+    const clock = fakeClock();
+    const res = await withPolicy(fake, clock.opts).fetch("https://api.mangabaka.org/v0/my/submissions/series-images/1", {
+      method: "POST", body: "{}",
+    });
+    expect(res.status).toBe(502);
+    expect(fake.calls.length).toBe(1);
+  });
+
+  test("retries network errors on GET, then throws TransportError", async () => {
+    const fake = new FakeTransport().on(() => true, () => { throw new TypeError("connection reset"); });
+    const clock = fakeClock();
+    const t = withPolicy(fake, { ...clock.opts, minIntervalMs: () => 0 });
+    await expect(t.fetch("https://www.webtoons.com/x")).rejects.toBeInstanceOf(TransportError);
+    expect(fake.calls.length).toBe(3);
+  });
+
+  test("times out a hanging request", async () => {
+    const hang = { fetch: (_url: string, init?: { signal?: AbortSignal }) =>
+      new Promise<Response>((_, reject) => init?.signal?.addEventListener("abort", () => reject(init.signal!.reason))) };
+    const t = withPolicy(hang, { timeoutMs: 30, retries: 0, minIntervalMs: () => 0 });
+    await expect(t.fetch("https://www.webtoons.com/x")).rejects.toBeInstanceOf(TransportError);
+  });
+
+  test("refuses to send a token anywhere but api.mangabaka.org", async () => {
+    const fake = new FakeTransport().on(() => true, () => text("ok"));
+    const t = withPolicy(fake, { minIntervalMs: () => 0 });
+    await expect(t.fetch("https://evil.example/x", { headers: { "X-Api-Key": "mb-secret" } })).rejects.toThrow(/credentials/);
+    expect(fake.calls.length).toBe(0);
+    await t.fetch("https://api.mangabaka.org/v1/series/1", { headers: { "x-api-key": "mb-secret" } });
+    expect(fake.calls.length).toBe(1);
+  });
+
+  test("a caller abort is not retried", async () => {
+    const controller = new AbortController();
+    const fake = new FakeTransport().on(() => true, () => { controller.abort(); throw new DOMException("Aborted", "AbortError"); });
+    const t = withPolicy(fake, { minIntervalMs: () => 0 });
+    await expect(t.fetch("https://www.webtoons.com/x", { signal: controller.signal })).rejects.toThrow();
+    expect(fake.calls.length).toBe(1);
+  });
+});
+
+test("Wayback gets the honest User-Agent, everything else a browser one", () => {
+  expect(userAgentFor("https://web.archive.org/cdx/search/cdx?url=x")).toBe(TOOL_USER_AGENT);
+  expect(TOOL_USER_AGENT).toMatch(/^mangabaka-seeder\/\d+\.\d+\.\d+ \(\+https:\/\//);
+  expect(userAgentFor("https://www.webtoons.com/en/x")).toMatch(/^Mozilla\/5\.0/);
+});