From dfb21a6c0167c46459b88cfad1f0ea89583dc3ee Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Wed, 14 Oct 2026 23:36:34 +0200 Subject: [PATCH] feat(core): model types and transport policy (pacing, retries, timeouts, token scope) Co-Authored-By: Claude Opus 5.5 --- src/core/model.ts | 44 +++++++++++ src/core/transport.ts | 140 +++++++++++++++++++++++++++++++++ test/helpers/fake-transport.ts | 35 +++++++++ test/transport.test.ts | 104 ++++++++++++++++++++++++ 4 files changed, 323 insertions(+) create mode 100644 src/core/model.ts create mode 100644 src/core/transport.ts create mode 100644 test/helpers/fake-transport.ts create mode 100644 test/transport.test.ts diff --git a/src/core/model.ts b/src/core/model.ts new file mode 100644 index 0000000..0e2276f --- /dev/null +++ b/src/core/model.ts @@ -0,0 +1,44 @@ +export type CoverRole = "poster" | "square" | "banner" | "foreground" | "other"; +export type ImageType = "volume" | "volume_back" | "chapter" | "season" | "banner" | "audiobook" | "other"; +export type ContentRating = "safe" | "suggestive" | "erotica" | "pornographic"; +export const CONTENT_RATINGS: readonly ContentRating[] = ["safe", "suggestive", "erotica", "pornographic"]; +export type Shape = "tall" | "portrait" | "square" | "wide" | "other"; +export type ImageExt = "jpg" | "png" | "gif" | "webp"; +/** Wayback history: one snapshot per this many months (spec §4.3). */ +export type Granularity = 1 | 3 | 6 | 12; +export const GRANULARITIES: readonly Granularity[] = [1, 3, 6, 12]; + +export interface SourceSeries { + source: string; // Source.id + externalId: string; // e.g. Webtoons title_no "95" + title: string; + language: string; // MangaBaka language code, e.g. "en", "zh-Hant", or "unknown" + canonicalUrl: string; + archiveUrls: string[]; // URL prefixes (no scheme) to look up in Wayback + mature: boolean; + flags: Record; // e.g. { canvas: true } +} + +export interface CoverCandidate { + url: string; // normalised, hotlinkable + role: CoverRole; + current: boolean; // shown on the live page now + seenAt?: string; // earliest archive timestamp (YYYYMMDDhhmmss) if historical + lastSeenAt?: string; // latest archive timestamp after merging + originalUrl?: string; // as written in the page (absolute), before normalisation + archivedUrl?: string; // Wayback copy of the image itself +} + +export interface InspectedCover { + candidate: CoverCandidate; + fetchedFrom: string; // URL that served the bytes (candidate.url, or archivedUrl) + live: boolean; // true if fetchedFrom === candidate.url + width: number; + height: number; + sha256: string; // lowercase hex + contentType: string; // image/jpeg | image/png | image/gif | image/webp + ext: ImageExt; + bytes: Uint8Array; + shape: Shape; + type: ImageType; // suggested MangaBaka type +} diff --git a/src/core/transport.ts b/src/core/transport.ts new file mode 100644 index 0000000..b027b0e --- /dev/null +++ b/src/core/transport.ts @@ -0,0 +1,140 @@ +import { REPO_URL, VERSION } from "./version"; + +export interface FetchInit { + method?: string; + headers?: Record; + body?: BodyInit | null; + signal?: AbortSignal; + /** Overrides the policy timeout for this one request (uploads). */ + timeoutMs?: number; +} + +/** How a front end reaches the network (spec §4.2). Returns a standard Response. */ +export interface Transport { + fetch(url: string, init?: FetchInit): Promise; +} + +export const MANGABAKA_API_HOST = "api.mangabaka.org"; +export const WAYBACK_HOST = "web.archive.org"; +/** Hosts a browser page may call directly because they send CORS headers (spec §2.3, §2.4). */ +export const CORS_OPEN_HOSTS: ReadonlySet = new Set([MANGABAKA_API_HOST, "litterbox.catbox.moe"]); + +export const BROWSER_USER_AGENT = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36"; +export const TOOL_USER_AGENT = `mangabaka-seeder/${VERSION} (+${REPO_URL})`; + +/** Wayback flags fake browser User-Agents as bots (spec §2.2); sources get a browser one. */ +export function userAgentFor(url: string): string { + return hostOf(url) === WAYBACK_HOST ? TOOL_USER_AGENT : BROWSER_USER_AGENT; +} + +export function hostOf(url: string): string { + try { + return new URL(url).hostname; + } catch { + return ""; + } +} + +export class TransportError extends Error { + constructor(message: string, readonly url: string, readonly status?: number) { + super(message); + this.name = "TransportError"; + } +} + +export interface PolicyOptions { + /** Minimum gap between requests to one host. Default: 10 s for Wayback, 1 s otherwise. */ + minIntervalMs?: (host: string) => number; + timeoutMs?: number; + retries?: number; + backoffMs?: number; + now?: () => number; + sleep?: (ms: number, signal?: AbortSignal) => Promise; +} + +export const defaultInterval = (host: string): number => (host === WAYBACK_HOST ? 10_000 : 1_000); + +export function abortableSleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(signal.reason); + const onAbort = () => { + clearTimeout(timer); + reject(signal!.reason); + }; + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +export function assertTokenScope(host: string, headers?: Record): void { + if (!headers) return; + const carriesToken = Object.keys(headers).some((k) => ["x-api-key", "authorization"].includes(k.toLowerCase())); + if (carriesToken && host !== MANGABAKA_API_HOST) { + throw new TransportError(`refusing to send credentials to ${host || "an unknown host"}`, host); + } +} + +const RETRYABLE_METHODS = new Set(["GET", "HEAD"]); +const isRetryableStatus = (status: number) => status === 429 || status >= 500; + +/** Wraps any transport with the shared behaviour of spec §4.2. */ +export function withPolicy(inner: Transport, opts: PolicyOptions = {}): Transport { + const interval = opts.minIntervalMs ?? defaultInterval; + const timeoutMs = opts.timeoutMs ?? 20_000; + const retries = opts.retries ?? 2; + const backoffMs = opts.backoffMs ?? 2_000; + const now = opts.now ?? Date.now; + const sleep = opts.sleep ?? abortableSleep; + const nextSlot = new Map(); + + async function waitTurn(host: string, signal?: AbortSignal) { + const t = now(); + const at = Math.max(t, nextSlot.get(host) ?? 0); + nextSlot.set(host, at + interval(host)); + if (at > t) await sleep(at - t, signal); + } + + function attempt(url: string, init: FetchInit): Promise { + const { timeoutMs: perRequest, ...rest } = init; + const timeout = AbortSignal.timeout(perRequest ?? timeoutMs); + const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout; + return inner.fetch(url, { ...rest, signal }); + } + + return { + async fetch(url, init = {}) { + const host = hostOf(url); + assertTokenScope(host, init.headers); + const method = (init.method ?? "GET").toUpperCase(); + const attempts = RETRYABLE_METHODS.has(method) ? retries + 1 : 1; + let lastError: unknown; + for (let i = 0; i < attempts; i++) { + await waitTurn(host, init.signal); + try { + const res = await attempt(url, init); + if (!isRetryableStatus(res.status) || i === attempts - 1) return res; + await res.body?.cancel(); + await sleep(retryDelay(res, backoffMs, i), init.signal); + } catch (err) { + if (init.signal?.aborted) throw err; + lastError = err; + if (i < attempts - 1) await sleep(backoffMs * 2 ** i, init.signal); + } + } + throw new TransportError(`request to ${host} failed: ${describe(lastError)}`, url); + }, + }; +} + +function retryDelay(res: Response, base: number, attempt: number): number { + const seconds = Number(res.headers.get("retry-after")); + return Number.isFinite(seconds) && seconds > 0 ? Math.min(seconds, 120) * 1000 : base * 2 ** attempt; +} + +function describe(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/test/helpers/fake-transport.ts b/test/helpers/fake-transport.ts new file mode 100644 index 0000000..56cfb6a --- /dev/null +++ b/test/helpers/fake-transport.ts @@ -0,0 +1,35 @@ +import type { FetchInit, Transport } from "../../src/core/transport"; + +type Matcher = string | RegExp | ((url: string, init: FetchInit) => boolean); +type Responder = (url: string, init: FetchInit) => Response | Promise; + +/** Serves canned responses; throws on any request without a route, so tests can't hit the network. */ +export class FakeTransport implements Transport { + readonly calls: { url: string; init: FetchInit }[] = []; + private readonly routes: { match: Matcher; respond: Responder }[] = []; + + on(match: Matcher, respond: Responder): this { + this.routes.push({ match, respond }); + return this; + } + + async fetch(url: string, init: FetchInit = {}): Promise { + this.calls.push({ url, init }); + for (const { match, respond } of this.routes) { + const hit = typeof match === "string" ? match === url : match instanceof RegExp ? match.test(url) : match(url, init); + if (hit) return respond(url, init); + } + throw new Error(`FakeTransport: no route for ${init.method ?? "GET"} ${url}`); + } + + urls(): string[] { + return this.calls.map((c) => c.url); + } +} + +export const text = (body: string, status = 200, headers: Record = {}) => + new Response(body, { status, headers: { "content-type": "text/html; charset=utf-8", ...headers } }); +export const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +export const bytes = (body: Uint8Array, contentType = "image/jpeg", status = 200) => + new Response(body as Uint8Array, { status, headers: { "content-type": contentType } }); diff --git a/test/transport.test.ts b/test/transport.test.ts new file mode 100644 index 0000000..b3e690f --- /dev/null +++ b/test/transport.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, test } from "bun:test"; +import { + TOOL_USER_AGENT, TransportError, userAgentFor, withPolicy, type PolicyOptions, +} from "../src/core/transport"; +import { FakeTransport, text } from "./helpers/fake-transport"; + +function fakeClock() { + let now = 0; + const slept: number[] = []; + const opts: PolicyOptions = { + now: () => now, + sleep: async (ms) => { slept.push(ms); now += ms; }, + }; + return { opts, slept, advance: (ms: number) => { now += ms; } }; +} + +describe("withPolicy", () => { + test("spaces requests to the same host, not across hosts", async () => { + const fake = new FakeTransport().on(() => true, () => text("ok")); + const clock = fakeClock(); + const t = withPolicy(fake, clock.opts); + await t.fetch("https://www.webtoons.com/a"); + await t.fetch("https://www.webtoons.com/b"); + await t.fetch("https://swebtoon-phinf.pstatic.net/c"); + await t.fetch("https://www.webtoons.com/d"); + expect(clock.slept).toEqual([1000, 1000]); + }); + + test("waits 10 s between Wayback requests", async () => { + const fake = new FakeTransport().on(() => true, () => text("ok")); + const clock = fakeClock(); + const t = withPolicy(fake, clock.opts); + await t.fetch("https://web.archive.org/a"); + await t.fetch("https://web.archive.org/b"); + expect(clock.slept).toEqual([10_000]); + }); + + test("retries GET on 429/5xx, honouring Retry-After", async () => { + let n = 0; + const fake = new FakeTransport().on(() => true, () => + ++n === 1 ? text("slow down", 429, { "retry-after": "7" }) : n === 2 ? text("oops", 503) : text("ok")); + const clock = fakeClock(); + const res = await withPolicy(fake, { ...clock.opts, minIntervalMs: () => 0 }).fetch("https://www.webtoons.com/x"); + expect(res.status).toBe(200); + expect(fake.calls.length).toBe(3); + expect(clock.slept).toEqual([7000, 4000]); // Retry-After, then backoff 2000 * 2^1 + }); + + test("returns the last 5xx after the retries are used up", async () => { + const fake = new FakeTransport().on(() => true, () => text("down", 502)); + const clock = fakeClock(); + const res = await withPolicy(fake, { ...clock.opts, minIntervalMs: () => 0 }).fetch("https://www.webtoons.com/x"); + expect(res.status).toBe(502); + expect(fake.calls.length).toBe(3); + }); + + test("never retries POST", async () => { + const fake = new FakeTransport().on(() => true, () => text("bad gateway", 502)); + const clock = fakeClock(); + const res = await withPolicy(fake, clock.opts).fetch("https://api.mangabaka.org/v0/my/submissions/series-images/1", { + method: "POST", body: "{}", + }); + expect(res.status).toBe(502); + expect(fake.calls.length).toBe(1); + }); + + test("retries network errors on GET, then throws TransportError", async () => { + const fake = new FakeTransport().on(() => true, () => { throw new TypeError("connection reset"); }); + const clock = fakeClock(); + const t = withPolicy(fake, { ...clock.opts, minIntervalMs: () => 0 }); + await expect(t.fetch("https://www.webtoons.com/x")).rejects.toBeInstanceOf(TransportError); + expect(fake.calls.length).toBe(3); + }); + + test("times out a hanging request", async () => { + const hang = { fetch: (_url: string, init?: { signal?: AbortSignal }) => + new Promise((_, reject) => init?.signal?.addEventListener("abort", () => reject(init.signal!.reason))) }; + const t = withPolicy(hang, { timeoutMs: 30, retries: 0, minIntervalMs: () => 0 }); + await expect(t.fetch("https://www.webtoons.com/x")).rejects.toBeInstanceOf(TransportError); + }); + + test("refuses to send a token anywhere but api.mangabaka.org", async () => { + const fake = new FakeTransport().on(() => true, () => text("ok")); + const t = withPolicy(fake, { minIntervalMs: () => 0 }); + await expect(t.fetch("https://evil.example/x", { headers: { "X-Api-Key": "mb-secret" } })).rejects.toThrow(/credentials/); + expect(fake.calls.length).toBe(0); + await t.fetch("https://api.mangabaka.org/v1/series/1", { headers: { "x-api-key": "mb-secret" } }); + expect(fake.calls.length).toBe(1); + }); + + test("a caller abort is not retried", async () => { + const controller = new AbortController(); + const fake = new FakeTransport().on(() => true, () => { controller.abort(); throw new DOMException("Aborted", "AbortError"); }); + const t = withPolicy(fake, { minIntervalMs: () => 0 }); + await expect(t.fetch("https://www.webtoons.com/x", { signal: controller.signal })).rejects.toThrow(); + expect(fake.calls.length).toBe(1); + }); +}); + +test("Wayback gets the honest User-Agent, everything else a browser one", () => { + expect(userAgentFor("https://web.archive.org/cdx/search/cdx?url=x")).toBe(TOOL_USER_AGENT); + expect(TOOL_USER_AGENT).toMatch(/^mangabaka-seeder\/\d+\.\d+\.\d+ \(\+https:\/\//); + expect(userAgentFor("https://www.webtoons.com/en/x")).toMatch(/^Mozilla\/5\.0/); +});