From a6c02cb548120809015de3031b9dbf3a1daeda08 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Wed, 14 Oct 2026 23:38:22 +0200 Subject: [PATCH] feat(core): image hosts (server, litterbox, catbox, direct) and per-image rehosting Co-Authored-By: Claude Opus 5.5 --- src/core/hosting/catbox.ts | 28 +++++++++ src/core/hosting/direct.ts | 11 ++++ src/core/hosting/host.ts | 66 +++++++++++++++++++++ src/core/hosting/litterbox.ts | 22 +++++++ src/core/hosting/registry.ts | 8 +++ src/core/hosting/rehost.ts | 46 +++++++++++++++ src/core/hosting/server.ts | 8 +++ test/hosting.test.ts | 106 ++++++++++++++++++++++++++++++++++ 8 files changed, 295 insertions(+) create mode 100644 src/core/hosting/catbox.ts create mode 100644 src/core/hosting/direct.ts create mode 100644 src/core/hosting/host.ts create mode 100644 src/core/hosting/litterbox.ts create mode 100644 src/core/hosting/registry.ts create mode 100644 src/core/hosting/rehost.ts create mode 100644 src/core/hosting/server.ts create mode 100644 test/hosting.test.ts diff --git a/src/core/hosting/catbox.ts b/src/core/hosting/catbox.ts new file mode 100644 index 0000000..02c0b79 --- /dev/null +++ b/src/core/hosting/catbox.ts @@ -0,0 +1,28 @@ +import type { Transport } from "../transport"; +import { HostError, UPLOAD_TIMEOUT_MS, fileFor, viaServer, type ImageHost } from "./host"; + +export const CATBOX_API = "https://catbox.moe/user/api.php"; + +/** Used directly by the userscript and by our server (for the page). */ +export async function uploadToCatbox(transport: Transport, file: Blob, name: string, userhash?: string): Promise<{ url: string }> { + const form = new FormData(); + form.set("reqtype", "fileupload"); + if (userhash) form.set("userhash", userhash); + form.set("fileToUpload", file, name); + const res = await transport.fetch(CATBOX_API, { method: "POST", body: form, timeoutMs: UPLOAD_TIMEOUT_MS }); + const body = (await res.text()).trim(); + if (!res.ok || !/^https:\/\/files\.catbox\.moe\/\S+$/.test(body)) { + throw new HostError(`catbox: ${body.slice(0, 200) || `HTTP ${res.status}`}`, res.status); + } + return { url: body }; +} + +export const catbox: ImageHost = { + id: "catbox", + lifetime: () => "permanent", + upload(image, ctx) { + const userhash = ctx.options.catboxUserhash?.trim() || undefined; + if (ctx.catboxVia === "server") return viaServer(image, ctx, { host: "catbox", ...(userhash ? { userhash } : {}) }); + return uploadToCatbox(ctx.transport, fileFor(image), `${image.sha256}.${image.ext}`, userhash); + }, +}; diff --git a/src/core/hosting/direct.ts b/src/core/hosting/direct.ts new file mode 100644 index 0000000..20aea8e --- /dev/null +++ b/src/core/hosting/direct.ts @@ -0,0 +1,11 @@ +import { HostError, type ImageHost } from "./host"; + +/** No rehosting: submit the source URL itself. */ +export const direct: ImageHost = { + id: "direct", + lifetime: () => "as long as the source keeps it", + async upload(image) { + if (!image.live) throw new HostError("This image is only on archive.org now; pick a host that rehosts it."); + return { url: image.sourceUrl }; + }, +}; diff --git a/src/core/hosting/host.ts b/src/core/hosting/host.ts new file mode 100644 index 0000000..350e566 --- /dev/null +++ b/src/core/hosting/host.ts @@ -0,0 +1,66 @@ +import type { Transport } from "../transport"; + +export type HostId = "server" | "litterbox" | "catbox" | "direct"; +export type LitterboxTime = "1h" | "12h" | "24h" | "72h"; +export const LITTERBOX_TIMES: readonly LitterboxTime[] = ["1h", "12h", "24h", "72h"]; + +export interface HostOptions { + litterboxTime?: LitterboxTime; + catboxUserhash?: string; +} + +/** One inspected image, ready to rehost. `sourceUrl` is where its bytes came from. */ +export interface HostImage { + sourceUrl: string; + live: boolean; + bytes: Uint8Array; + sha256: string; + contentType: string; + ext: string; +} + +export interface HostContext { + transport: Transport; + /** Our server's origin, or null when there is none (userscript without a server). */ + serverBase: string | null; + /** catbox has no CORS: the page goes through our server, the userscript straight there (spec §4.6). */ + catboxVia: "server" | "direct"; + options: HostOptions; +} + +export interface ImageHost { + readonly id: HostId; + /** Shown next to the host in the UI. */ + lifetime(opts: HostOptions): string; + upload(image: HostImage, ctx: HostContext): Promise<{ url: string }>; +} + +export class HostError extends Error { + constructor(message: string, readonly status?: number) { + super(message); + this.name = "HostError"; + } +} + +export const UPLOAD_TIMEOUT_MS = 120_000; + +export const fileFor = (image: HostImage): Blob => new Blob([image.bytes as Uint8Array], { type: image.contentType }); + +/** `POST /api/upload`: the server fetches `sourceUrl` itself, so it can't be used to host arbitrary bytes. */ +export async function viaServer( + image: HostImage, + ctx: HostContext, + body: { host: "store" | "catbox"; userhash?: string }, +): Promise<{ url: string }> { + if (ctx.serverBase === null) throw new HostError("No mangabaka-seeder server is configured."); + const res = await ctx.transport.fetch(`${ctx.serverBase}/api/upload`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ url: image.sourceUrl, ...body }), + timeoutMs: UPLOAD_TIMEOUT_MS, + }); + const reply = (await res.json().catch(() => null)) as { url?: string; sha256?: string; error?: string } | null; + if (!res.ok || !reply?.url) throw new HostError(reply?.error ?? `Upload failed (HTTP ${res.status}).`, res.status); + if (reply.sha256 !== image.sha256) throw new HostError("The server fetched different bytes than the ones checked; not using them."); + return { url: reply.url }; +} diff --git a/src/core/hosting/litterbox.ts b/src/core/hosting/litterbox.ts new file mode 100644 index 0000000..63548f5 --- /dev/null +++ b/src/core/hosting/litterbox.ts @@ -0,0 +1,22 @@ +import { HostError, UPLOAD_TIMEOUT_MS, fileFor, type ImageHost, type LitterboxTime } from "./host"; + +export const LITTERBOX_API = "https://litterbox.catbox.moe/resources/internals/api.php"; +const LABEL: Readonly> = { "1h": "1 hour", "12h": "12 hours", "24h": "24 hours", "72h": "72 hours" }; + +/** Temporary (≤ 72 h); CORS is open, so the page uploads straight from the browser. */ +export const litterbox: ImageHost = { + id: "litterbox", + lifetime: (opts) => LABEL[opts.litterboxTime ?? "72h"], + async upload(image, ctx) { + const form = new FormData(); + form.set("reqtype", "fileupload"); + form.set("time", ctx.options.litterboxTime ?? "72h"); + form.set("fileToUpload", fileFor(image), `${image.sha256}.${image.ext}`); + const res = await ctx.transport.fetch(LITTERBOX_API, { method: "POST", body: form, timeoutMs: UPLOAD_TIMEOUT_MS }); + const body = (await res.text()).trim(); + if (!res.ok || !/^https:\/\/litter\.catbox\.moe\/\S+$/.test(body)) { + throw new HostError(`litterbox: ${body.slice(0, 200) || `HTTP ${res.status}`}`, res.status); + } + return { url: body }; + }, +}; diff --git a/src/core/hosting/registry.ts b/src/core/hosting/registry.ts new file mode 100644 index 0000000..6939c36 --- /dev/null +++ b/src/core/hosting/registry.ts @@ -0,0 +1,8 @@ +import { catbox } from "./catbox"; +import { direct } from "./direct"; +import type { HostId, ImageHost } from "./host"; +import { litterbox } from "./litterbox"; +import { serverHost } from "./server"; + +/** Adding a host: one file in this folder plus one line here. */ +export const HOSTS: Readonly> = { server: serverHost, litterbox, catbox, direct }; diff --git a/src/core/hosting/rehost.ts b/src/core/hosting/rehost.ts new file mode 100644 index 0000000..b19f44e --- /dev/null +++ b/src/core/hosting/rehost.ts @@ -0,0 +1,46 @@ +import { HostError, type HostContext, type HostId, type HostImage, type HostOptions, type ImageHost } from "./host"; + +export interface RehostResult { + hosted: Map; + failures: { sha256: string; message: string }[]; + allFailed: boolean; +} + +/** litterbox and catbox uploads are remembered by hash; server storage is re-uploaded on purpose (restarts its 30 days). */ +export function cacheKey(host: HostId, options: HostOptions, sha256: string): string | null { + if (host === "litterbox") return `litterbox:${options.litterboxTime ?? "72h"}:${sha256}`; + if (host === "catbox") return `catbox:${sha256}`; + return null; +} + +/** Failures are per image (spec §4.6). A full store (507) stops the rest: they would fail the same way. */ +export async function rehostAll( + images: readonly HostImage[], + host: ImageHost, + ctx: HostContext, + cache: Map, + onProgress?: (done: number, total: number) => void, +): Promise { + const hosted = new Map(); + const failures: { sha256: string; message: string }[] = []; + let stopped: string | null = null; + let done = 0; + for (const image of images) { + if (stopped) { + failures.push({ sha256: image.sha256, message: stopped }); + } else { + const key = cacheKey(host.id, ctx.options, image.sha256); + try { + const url = (key && cache.get(key)) || (await host.upload(image, ctx)).url; + if (key) cache.set(key, url); + hosted.set(image.sha256, url); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + failures.push({ sha256: image.sha256, message }); + if (err instanceof HostError && err.status === 507) stopped = message; + } + } + onProgress?.(++done, images.length); + } + return { hosted, failures, allFailed: images.length > 0 && hosted.size === 0 }; +} diff --git a/src/core/hosting/server.ts b/src/core/hosting/server.ts new file mode 100644 index 0000000..243aefd --- /dev/null +++ b/src/core/hosting/server.ts @@ -0,0 +1,8 @@ +import { viaServer, type ImageHost } from "./host"; + +/** Our server's covers folder; swept after 30 days (spec §4.6). */ +export const serverHost: ImageHost = { + id: "server", + lifetime: () => "30 days", + upload: (image, ctx) => viaServer(image, ctx, { host: "store" }), +}; diff --git a/test/hosting.test.ts b/test/hosting.test.ts new file mode 100644 index 0000000..96a066d --- /dev/null +++ b/test/hosting.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "bun:test"; +import { catbox } from "../src/core/hosting/catbox"; +import { direct } from "../src/core/hosting/direct"; +import { HostError, type HostContext, type HostImage } from "../src/core/hosting/host"; +import { litterbox } from "../src/core/hosting/litterbox"; +import { HOSTS } from "../src/core/hosting/registry"; +import { rehostAll } from "../src/core/hosting/rehost"; +import { serverHost } from "../src/core/hosting/server"; +import { FakeTransport, json, text } from "./helpers/fake-transport"; + +const image = (n = 1, live = true): HostImage => ({ + sourceUrl: `https://swebtoon-phinf.pstatic.net/x/${n}.jpg`, live, bytes: new Uint8Array([n, 2, 3]), + sha256: String(n).repeat(64).slice(0, 64), contentType: "image/jpeg", ext: "jpg", +}); +const ctx = (transport: FakeTransport, over: Partial = {}): HostContext => + ({ transport, serverBase: "https://mbseed.example.org", catboxVia: "direct", options: {}, ...over }); +const LITTERBOX = "https://litterbox.catbox.moe/resources/internals/api.php"; +const CATBOX = "https://catbox.moe/user/api.php"; +const UPLOAD = "https://mbseed.example.org/api/upload"; + +test("lifetimes shown next to each host", () => { + expect(HOSTS.server.lifetime({})).toBe("30 days"); + expect(HOSTS.litterbox.lifetime({})).toBe("72 hours"); + expect(HOSTS.litterbox.lifetime({ litterboxTime: "12h" })).toBe("12 hours"); + expect(HOSTS.catbox.lifetime({})).toBe("permanent"); + expect(HOSTS.direct.lifetime({})).toBe("as long as the source keeps it"); +}); + +test("direct: the live source URL; refuses archive-only images", async () => { + const fake = new FakeTransport(); + expect(await direct.upload(image(), ctx(fake))).toEqual({ url: image().sourceUrl }); + await expect(direct.upload(image(1, false), ctx(fake))).rejects.toBeInstanceOf(HostError); + expect(fake.calls).toHaveLength(0); +}); + +describe("litterbox", () => { + test("multipart upload of the checked bytes, 72h by default", async () => { + const fake = new FakeTransport().on(LITTERBOX, () => text("https://litter.catbox.moe/abc123.jpg\n")); + expect(await litterbox.upload(image(), ctx(fake))).toEqual({ url: "https://litter.catbox.moe/abc123.jpg" }); + const form = fake.calls[0]!.init.body as FormData; + expect(fake.calls[0]!.init.method).toBe("POST"); + expect(form.get("reqtype")).toBe("fileupload"); + expect(form.get("time")).toBe("72h"); + expect((form.get("fileToUpload") as File).size).toBe(3); + }); + test("an error body is a HostError", async () => { + const fake = new FakeTransport().on(LITTERBOX, () => text("File too large", 413)); + await expect(litterbox.upload(image(), ctx(fake))).rejects.toThrow(/File too large/); + }); +}); + +describe("catbox", () => { + test("direct (userscript): multipart with the userhash", async () => { + const fake = new FakeTransport().on(CATBOX, () => text("https://files.catbox.moe/xyz.jpg")); + const url = await catbox.upload(image(), ctx(fake, { options: { catboxUserhash: "abc123" } })); + expect(url).toEqual({ url: "https://files.catbox.moe/xyz.jpg" }); + expect((fake.calls[0]!.init.body as FormData).get("userhash")).toBe("abc123"); + }); + test("via our server (page): sends the source URL, never bytes; checks the sha256", async () => { + const fake = new FakeTransport().on(UPLOAD, () => json({ url: "https://files.catbox.moe/xyz.jpg", sha256: image().sha256 })); + await catbox.upload(image(), ctx(fake, { catboxVia: "server", options: { catboxUserhash: "abc123" } })); + expect(JSON.parse(fake.calls[0]!.init.body as string)).toEqual({ url: image().sourceUrl, host: "catbox", userhash: "abc123" }); + }); +}); + +describe("server storage", () => { + test("posts the source URL with host store", async () => { + const fake = new FakeTransport().on(UPLOAD, () => + json({ url: `https://mbseed.example.org/covers/${image().sha256}.jpg`, sha256: image().sha256 })); + expect((await serverHost.upload(image(), ctx(fake))).url).toContain("/covers/"); + expect(JSON.parse(fake.calls[0]!.init.body as string)).toEqual({ url: image().sourceUrl, host: "store" }); + }); + test("rejects different bytes", async () => { + const fake = new FakeTransport().on(UPLOAD, () => json({ url: "https://x/covers/y.jpg", sha256: "0".repeat(64) })); + await expect(serverHost.upload(image(), ctx(fake))).rejects.toThrow(/different bytes/); + }); + test("storage full is HostError 507", async () => { + const fake = new FakeTransport().on(UPLOAD, () => json({ error: "server storage is full" }, 507)); + await expect(serverHost.upload(image(), ctx(fake))).rejects.toMatchObject({ status: 507 }); + }); + test("no server configured", async () => { + await expect(serverHost.upload(image(), ctx(new FakeTransport(), { serverBase: null }))).rejects.toBeInstanceOf(HostError); + }); +}); + +describe("rehostAll", () => { + test("failures are per image; the cache avoids uploading twice", async () => { + let n = 0; + const fake = new FakeTransport().on(LITTERBOX, () => + ++n === 2 ? text("error", 500) : text(`https://litter.catbox.moe/${n}.jpg`)); + const cache = new Map(); + const first = await rehostAll([image(1), image(2), image(3)], litterbox, ctx(fake), cache); + expect([...first.hosted.keys()]).toEqual([image(1).sha256, image(3).sha256]); + expect(first.failures).toEqual([{ sha256: image(2).sha256, message: expect.stringContaining("litterbox") }]); + expect(first.allFailed).toBe(false); + await rehostAll([image(1)], litterbox, ctx(fake), cache); + expect(fake.calls).toHaveLength(3); + }); + test("a full store stops the remaining uploads", async () => { + const fake = new FakeTransport().on(UPLOAD, () => json({ error: "server storage is full" }, 507)); + const res = await rehostAll([image(1), image(2)], serverHost, ctx(fake), new Map()); + expect(fake.calls).toHaveLength(1); + expect(res.failures).toHaveLength(2); + expect(res.allFailed).toBe(true); + }); +});