diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..b969289 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,22 @@ +{ + "permissions": { + "allow": [ + "Bash(git status:*)", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)", + "Bash(git rev-parse:*)", "Bash(git add:*)", "Bash(git commit:*)" + ], + "ask": [ + "Bash(git push:*)", "Bash(git merge:*)", "Bash(git rebase:*)", "Bash(git switch:*)", + "Bash(git restore:*)", "Bash(git commit --amend:*)", "Bash(rm:*)", + "Bash(bun add:*)", "Bash(bun remove:*)", "Bash(bun update:*)", "Bash(bun upgrade:*)", + "Bash(bunx:*)", "Bash(npx:*)" + ], + "deny": [ + "Bash(git push --force:*)", "Bash(git push -f:*)", "Bash(git push --force-with-lease:*)", + "Bash(git reset --hard:*)", "Bash(git clean:*)", "Bash(git add -f:*)", "Bash(git add --force:*)", + "Bash(git commit --no-verify:*)", "Bash(git commit -n:*)", + "Bash(rm -r:*)", "Bash(rm -R:*)", "Bash(rm -rf:*)", "Bash(rm -fr:*)", "Bash(rm -Rf:*)", + "Bash(rm -fR:*)", "Bash(rm --recursive:*)", "Bash(sudo:*)", + "Read(./.env)", "Read(./.env.*)", "Read(./.dev.vars)", "Read(./secrets/**)" + ] + } +} diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 0000000..5542eff --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1,2 @@ +sandbox_mode = "workspace-write" +approval_policy = "on-request" diff --git a/.codex/rules/project.rules b/.codex/rules/project.rules new file mode 100644 index 0000000..6a1daeb --- /dev/null +++ b/.codex/rules/project.rules @@ -0,0 +1,18 @@ +# Rendered from docs/PERMISSIONS.md — change both in one commit. +# Deny +prefix_rule(pattern = ["git", "push", ["--force", "-f", "--force-with-lease"]], decision = "forbidden", + justification = "Force-push rewrites shared history.") +prefix_rule(pattern = ["git", "reset", "--hard"], decision = "forbidden", justification = "Discards work.") +prefix_rule(pattern = ["git", "clean"], decision = "forbidden", justification = "Deletes untracked files.") +prefix_rule(pattern = ["git", "add", ["-f", "--force"]], decision = "forbidden", + justification = "Stages ignored files, e.g. secrets.") +prefix_rule(pattern = ["git", "commit", ["--no-verify", "-n"]], decision = "forbidden", justification = "Skips hooks.") +prefix_rule(pattern = ["rm", ["-r", "-R", "-rf", "-fr", "-Rf", "-fR", "--recursive"]], decision = "forbidden", + justification = "Recursive delete.") +prefix_rule(pattern = ["sudo"], decision = "forbidden", justification = "No privilege escalation.") +# Ask +prefix_rule(pattern = ["git", ["push", "merge", "rebase", "switch", "restore"]], decision = "prompt") +prefix_rule(pattern = ["git", "commit", "--amend"], decision = "prompt") +prefix_rule(pattern = ["rm"], decision = "prompt") +prefix_rule(pattern = ["bun", ["add", "remove", "update", "upgrade"]], decision = "prompt") +prefix_rule(pattern = [["bunx", "npx"]], decision = "prompt") diff --git a/.gemini/settings.json b/.gemini/settings.json new file mode 100644 index 0000000..d699300 --- /dev/null +++ b/.gemini/settings.json @@ -0,0 +1,18 @@ +{ + "tools": { + "allowed": [ + "run_shell_command(git status)", "run_shell_command(git diff)", "run_shell_command(git log)", + "run_shell_command(git show)", "run_shell_command(git branch --list)", + "run_shell_command(git rev-parse)", "run_shell_command(git add)", "run_shell_command(git commit)" + ], + "exclude": [ + "run_shell_command(git push --force)", "run_shell_command(git push -f)", + "run_shell_command(git push --force-with-lease)", "run_shell_command(git reset --hard)", + "run_shell_command(git clean)", "run_shell_command(git add -f)", "run_shell_command(git add --force)", + "run_shell_command(git commit --no-verify)", "run_shell_command(git commit -n)", + "run_shell_command(rm -r)", "run_shell_command(rm -R)", "run_shell_command(rm -rf)", + "run_shell_command(rm -fr)", "run_shell_command(rm -Rf)", "run_shell_command(rm -fR)", + "run_shell_command(rm --recursive)", "run_shell_command(sudo)" + ] + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9bbd6a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules/ +dist/ +.env +.env.* +.dev.vars +.wrangler/ +CLAUDE.local.md +.claude/settings.local.json +.worktrees/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..46ea9d5 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,89 @@ +# mangabaka-seeder + +Seeds cover edits into MangaBaka from a Webtoons series. TypeScript on **Bun** (not Node): +a pure core, a static page served by a Cloudflare Worker, and a userscript. No code yet — the +approved design is `docs/specs/001-webtoons-covers.md`. + +`AGENTS.md` is the single source of truth for agents here. `CLAUDE.md` and `GEMINI.md` only point here. +Deeper context on demand: `docs/ARCHITECTURE.md`, `docs/DECISIONS.md`, `docs/TODO.md`. + +## Commands + +TODO(maintainer): confirm commands once tooling exists. Planned (spec §6), none verified yet: +`bun install --frozen-lockfile`, `bun test`, `bun test test/.test.ts`, `bunx tsc --noEmit`. +Use `bun`/`bunx` — never `npm`, `npx`, `node` or `vitest`. + +## Workflow + +Required skills: `superpowers`, `impeccable`. Check at session start; install if missing. +Specs → `docs/specs/NNN-slug.md` (permanent). Plans → `docs/plans/NNN-slug.md` while in progress; +`git rm` them before opening a merge request — the spec is the record. +(Overrides the superpowers default paths.) + +Anything beyond a trivial fix: brainstorm → spec (sign-off) → plan (sign-off) → TDD → review. +TDD means failing test first, watch it fail, minimal code, watch it pass, commit. +Tick plan checkboxes as you go so an interrupted session is resumable. +Trivial changes (typo, version bump, one-line config): say "trivial, skipping spec" and proceed. + +Use impeccable for the static page (`src/web/`) and the userscript panel — `craft` for new, +`audit`/`polish` on existing, `harden` before shipping. Design context: `PRODUCT.md`, `DESIGN.md`. + +## Rules + +- `src/core/` runs in a browser, a Worker, a userscript and Bun: no DOM, no `Bun.*`, no `node:*`, + no `GM_*`. All network goes through the injected `Transport`; hashing uses `crypto.subtle`. +- `core/` never imports from `web/`, `worker/`, `userscript/` or `cli/`. +- The MangaBaka token goes straight to `api.mangabaka.org` (CORS is open) — never through the Worker. +- MangaBaka `images` **replaces the whole set**: a save body must include every existing image. + Saves always use `save_mode: "review"`. +- Never call the MangaBaka save endpoint (`POST /v0/my/submissions/…` without `/preview`) yourself: + it files a real edit request. Tests use the fake transport; tests never touch the network. +- Fixtures in `test/fixtures/` are captured from real responses, never hand-written. +- `/api/fetch` only proxies allowlisted hosts; `/api/upload` takes a source URL, never raw bytes. +- Webtoons image URLs: strip the `?type=` query and use the `swebtoon-phinf.pstatic.net` host: + +```ts +// CORRECT https://swebtoon-phinf.pstatic.net/20250204_46/17386458444547o1b2_JPEG/95.jpg +// WRONG https://webtoon-phinf.pstatic.net/20250204_46/17386458444547o1b2_JPEG/95.jpg?type=crop540_540 +// (403 without a Webtoons Referer, and a cropped variant) +``` + +## Architecture + +src/core/ -> pure pipeline: sources, Wayback history, inspect, dedupe, seed, MangaBaka client +src/web/ -> static page · src/worker/ -> assets, `/api/fetch`, `/api/upload` → R2 (30-day expiry) +src/userscript/ -> panel on mangabaka.org, `GM_xmlhttpRequest` · src/cli/ -> later + +Full picture: `docs/ARCHITECTURE.md` and the spec — read them before your first change in a session. + +## Boundaries + +Never commit `.env*`, `.dev.vars`, tokens, or Cloudflare credentials. +Never force-push, rewrite pushed history, or commit directly to `main`. + +Ask first: new runtime dependencies · `wrangler deploy` or any Cloudflare/R2 configuration · +anything sent to MangaBaka with a real token · anything destructive (`rm -rf`, `git reset --hard`) · +work the approved plan does not cover. + +If the task turns out much larger than the plan assumed: stop, report, re-plan. + +## Docs duties + +Same commit as the code, not "later": +- `docs/ARCHITECTURE.md` — when a module, boundary, or data flow changes. +- `docs/DECISIONS.md` — append-only; any choice a future maintainer would reverse-engineer. +- `docs/TODO.md` — anything noticed and deliberately not fixed. No `TODO` comments in source. +- `docs/PERMISSIONS.md` — together with `.claude/settings.json`, `.codex/` and + `.gemini/settings.json`, in one commit. + +## Git + +Branches: `feat|fix|chore/`. Commits: Conventional Commits (`docs: …`, `feat: …`). +Small, each leaving tests green. + +## Done means verified + +Do not report success on work you have not run. Before saying done: +tests run *in this session* and seen passing · typecheck clean · matches the approved spec · +docs duties complete · `git status` clean and the diff is only what you intended. +"It should work" is not verification. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..20275b8 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,9 @@ +# CLAUDE.md + +**`AGENTS.md` is the single source of truth for this repository.** Read it now and follow it exactly. + +@AGENTS.md + +This file exists only so Claude Code finds the rules, and for Claude-specific configuration that +AGENTS.md cannot express. It contains no project rules of its own — add those to `AGENTS.md`. +If anything here ever conflicts with `AGENTS.md`, `AGENTS.md` wins. diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..a9364e8 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,7 @@ +# Design + +No UI exists yet, so there is no design system to record. + +TODO(maintainer): establish the visual direction with impeccable when building `src/web/` (the +first UI task), then record tokens, typography and components here from what ships. The +userscript panel lives inside mangabaka.org's pages and should sit comfortably next to them. diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..a97a85e --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,7 @@ +# GEMINI.md + +**`AGENTS.md` is the single source of truth for this repository.** Read `./AGENTS.md` before doing +anything else and follow it exactly. + +This file exists only so Gemini CLI finds the rules. It contains no project rules of its own — add +those to `AGENTS.md`. If anything here ever conflicts with `AGENTS.md`, `AGENTS.md` wins. diff --git a/HANDOVER.md b/HANDOVER.md deleted file mode 100644 index fea061b..0000000 --- a/HANDOVER.md +++ /dev/null @@ -1,138 +0,0 @@ -# Handover - -Read this first if you are picking up this repo. It records the state of the -work and the context from the design conversation (2026-10-14) that isn't -obvious from the code. - -**Read these, in this order:** - -1. this file -2. [`TODO.md`](TODO.md) -3. [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md), the approved design spec - with the external facts we verified -4. [`README.md`](README.md), user-facing behaviour. - -## Status - -- **Phase:** the design is complete and **approved by the user**. No code - exists yet. -- **Branch:** `feat/webtoons-covers`. The repo had no commits before this - branch, and nothing has been pushed. -- **Next step:** write the implementation plan, using the - `superpowers:writing-plans` skill, from `docs/ARCHITECTURE.md`. Then - implement it with test-driven development. -- **Approval gates:** the user's global preference is "once I approve a - design, carry it through spec → plan → execution with no extra review stops". - In this session, though, they explicitly said *"Do not write the plan - afterwards. Wait for my approval."* and later approved the spec ("everything - seems good"). Unless they say otherwise, **show the plan before - implementing.** - -## What we're building (in the user's words, then refined) - -> "a nice and easy to use seeder for adding new data to mangabaka. The first -> thing we should be able to do is given a webtoons url, get all covers related -> to the series in name, and request them added as main cover english." -> -> "think of this as a mangabaka version of /home/user/yambs" - -The user is a regular MangaBaka contributor and uses the normal edit-request -flow, not admin access. At the time of the conversation they **did not** have -API access (fewer than 25 approved submissions). - -## Decisions made with the user - -| Topic | Decision | Source | -|---|---|---| -| Stack | **TypeScript / Node** (≥ 22). The user picked this over Go, despite yambs being Go, to keep a browser extension or userscript possible later. | user | -| Submission path | **Both.** The API when `MANGABAKA_TOKEN` is set; otherwise a local review page for manual entry. | user | -| Which images | **All:** poster, square thumbnail, banner/background and foreground art. | user ("3") | -| Old covers | **Also find historical covers** through the Wayback Machine. | user | -| Main cover | The current poster (the square for canvas series) gets `is_default: true`, and existing covers get `is_default: false`. The reviewer can reject this. | Claude, accepted | -| Rehosting | **Required.** MangaBaka may not be able to fetch Webtoons URLs. The image host is pluggable: **catbox.moe by default**, an S3-compatible bucket, or `direct`. Temporary hosts like litterbox were rejected because links can expire before review. imgur is banned by MangaBaka. | user ("1") | -| Content rating | **The user sets it**, default `safe`: through a flag, an interactive prompt or a per-card selector. `--yes` requires an explicit `--content-rating`. Mature titles show a warning. | user | -| Image bytes | Kept in memory between inspection and upload. **Sizes up to about 650 KB have been seen**, so don't assume small images. There is a 20 MB per-image safety cap. | user, confirmed | -| Extensibility | **New sources** (Tapas, Tappytoon, Lezhin, Manta, …) are one module under `src/sources/` plus one registry line. **Image hosts** use the same pattern under `src/hosting/`. No pipeline changes. | user requirement | -| Safety | Nothing is uploaded or sent to MangaBaka without explicit user action. API saves always use `save_mode: "review"`. | Claude, accepted | -| Not now | Editing fields other than images; following Webtoons across language editions. | Claude, accepted | - -## Key facts (details in ARCHITECTURE §2) - -- **MangaBaka:** - - **Endpoints:** site `https://mangabaka.org`, API `https://api.mangabaka.org`. - `api.mangabaka.dev` is deprecated. - - **OpenAPI spec:** `https://mangabaka.org/api.json`, about 700 KB. Download - it again; the earlier copy was in a temporary scratchpad and is gone. - - **Cover submissions:** `GET`, `POST`, and `POST …/preview` on - `/v0/my/submissions/series-images/{id}`, with the `x-api-key` header. - `images` replaces the whole set. These endpoints return 403 unless the user - has the contributor role or at least 25 approved submissions. - - **Web cover form:** `/data/submissions/series/{id}/images`. It can't be - pre-filled, and SvelteKit's CSRF check blocks cross-site POSTs. - - **Test series:** Tower of God is **series 9999**. Its link is - `https://www.webtoons.com/-/-/-/list?title_no=95`, so links are stored - canonicalised. - - **Finding a series:** there's no reverse lookup by URL. Use - `/v2/series/match?q=` or `/v2/series/search`, then `/v1/series/{id}/links`. - - **Rate limits:** about 30 requests per minute for search, 180 otherwise. -- **Webtoons:** - - **Image fields:** `og:image` is the poster (480×623 when the `?type=` - query is stripped), and `twitter:image` is the square thumbnail. - - **Hero art:** `div.detail_bg` style and `.detail_header span.thmb img`. - - **Hotlinking:** `webtoon-phinf` needs a Webtoons Referer, but - `swebtoon-phinf` doesn't. Rewrite the host. - - **Old hosts:** rewrite `swebtoon.phinf.naver.net` to - `swebtoon-phinf.pstatic.net`. - - **No JSON API:** the app API needs signed requests, so scrape the HTML. -- **Wayback:** querying CDX with `collapse=timestamp:6` and fetching - `id_` snapshots works. - - Tower of God has about 2000 snapshots (2015–2026), which reference five - distinct cover images. - - Old images tested from 2014, 2015 and 2019 are **still live** on - `swebtoon-phinf.pstatic.net`. - - The 2021 one is gone from Wayback's own copy, but is live on pstatic. - - Some historical `og:image`s are landing-page art rather than covers. This - is why the tool classifies by shape and the user reviews the list. - -### Tower of God historical `og:image`s found (useful as test data) - -``` -2015 http://swebtoon.phinf.naver.net/20140710_28/1404979632969uKBA8_JPEG/01_EC8BA0EC9D98ED8391.jpg -2017 http://swebtoon.phinf.naver.net/20151006_141/1444125046528RTfRM_JPEG/_EB93BBEB80AF_EBA38CEABCA5_E29587EABCB1_EB9384EB84B0_ipa.jpg - (twitter:image …/20151006_243/1444125042491PuNza_JPEG/…_mob.jpg) -2019 https://swebtoon-phinf.pstatic.net/20190318_291/1552868599909GoVLY_JPEG/10_EC8DB8EB84A4EC9DBC_ipad.jpg - (twitter:image …/20190318_26/1552868584759vl9A1_JPEG/11_EC8DB8EB84A4EC9DBC_mobile.jpg) -2021 https://swebtoon-phinf.pstatic.net/20210127_106/1611705089085OYcUB_JPEG/1Tower-of-God-Mobile-Landing-Page.jpg -2023+ https://swebtoon-phinf.pstatic.net/20230324_114/1679612146968KhLLz_JPEG/6TowerOfGod_landingpage_mobile.jpg -live https://swebtoon-phinf.pstatic.net/20250204_46/17386458444547o1b2_JPEG/95.jpg (poster, 480×623) -``` - -Other test URLs: - -- canvas: `https://www.webtoons.com/en/canvas/crow-time/list?title_no=693372` -- mature: title_no `6065` (filename contains spaces and `)`; URL-encode it) -- FR example: Violet Romance, `2Thumb_Poster_9782.jpg`. - -## Open questions and risks - -1. **When MangaBaka copies images.** We don't know if it copies the image when - the request is submitted or when it is approved. Permanent hosting (catbox) - is the mitigation. -2. **Whether MangaBaka accepts catbox URLs.** Not verified. If it doesn't, try - `s3`, or `direct` (`swebtoon-phinf.pstatic.net`). -3. **The "main cover" mapping.** Mapping "main cover" to `is_default` is - inferred from the API spec, not confirmed by MangaBaka. -4. **Request limits.** Rate limits and region blocks on Webtoons or Wayback at - scale haven't been tested. -5. **Personal access tokens.** We don't know where a user creates a PAT in the - MangaBaka UI. The submission endpoints say `auth: pat`. - -## References - -- yambs (the model project, Go): `/home/user/yambs`, with the same - sources → seed → actions shape. -- MangaBaka API docs: https://mangabaka.org/data/api; explorer: - https://mangabaka.org/data/api/explorer -- MangaBaka support: https://mangabaka.org/discord -- catbox API: `POST https://catbox.moe/user/api.php` with `reqtype=fileupload`, - `fileToUpload` and optional `userhash`. diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 0000000..d551df1 --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,20 @@ +# Product + +**What:** a tool that turns a MangaBaka series link plus a Webtoons series link into a cover edit +request for [MangaBaka](https://mangabaka.org): every current and historical cover, deduplicated +against what MangaBaka already has, with the current poster as the main cover. + +**Who:** regular MangaBaka contributors. Most have **no API access** (it needs the contributor role +or 25 approved submissions), so the manual path — copy hosted URLs into MangaBaka's cover form — +is the common case, not an edge case. + +**Surfaces:** a static web page (primary); a userscript panel on mangabaka.org series pages; a CLI later. + +**Principles** +- The user is in control: nothing is uploaded or submitted without an explicit action, and the + submission always goes to MangaBaka review. +- The user decides the content rating (default `safe`); the tool only warns on mature titles. +- Show the evidence for each cover (image, size, where and when it was seen) so the user can drop + landing-page art that isn't a cover. + +Design spec: `docs/specs/001-webtoons-covers.md`. diff --git a/README.md b/README.md index 557cbf5..d4341de 100644 --- a/README.md +++ b/README.md @@ -1,155 +1,72 @@ # mangabaka-seeder -`mbseed` is a command-line tool for seeding edits to the -[MangaBaka](https://mangabaka.org) manga/comics database, in the spirit of -[yambs](https://codeberg.org/derat/yambs) for MusicBrainz. +A tool for seeding edits to the [MangaBaka](https://mangabaka.org) manga/comics +database, in the spirit of [yambs](https://codeberg.org/derat/yambs) for +MusicBrainz. -Today it does one thing well: **given a Webtoons series URL, it finds every -cover that series has used, current and historical, and requests them added to -the matching MangaBaka series**. The current poster becomes the main (default) -cover for the edition's language. +> **Status:** designed, not built yet. This README describes the planned +> behaviour; see [the spec](docs/specs/001-webtoons-covers.md). -More sources (Tapas, Tappytoon, …) and more kinds of edits are planned; see -[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for how they plug in. +Today's goal is one thing done well: **given a MangaBaka series link and a +Webtoons series link, find every cover the Webtoons series has used, current +and historical, and request them added to the MangaBaka series**. The current +poster becomes the main (default) cover for the edition's language. + +## Two ways to use it + +- **Web page** (primary). Open the page, paste the two links, review the + covers. The page is hosted on Cloudflare; a small Worker behind it reads + Webtoons and the Wayback Machine for you, because browsers can't. +- **Userscript.** Install `mbseed.user.js` in Tampermonkey or Violentmonkey. + On a MangaBaka series page it adds a **Seed covers** panel; paste the + Webtoons link. It reads Webtoons directly from your browser, no proxy needed. + +A command-line version (Bun) may follow. ## How it works -1. **Read the series.** It reads the Webtoons page: title, `title_no`, language, - and official or canvas. -2. **Find the MangaBaka series.** It searches MangaBaka by title and picks the - series whose Webtoons link has the same `title_no`. -3. **Collect covers.** +1. **Read both links.** The MangaBaka link gives the series id; the Webtoons + link gives the title, `title_no`, language, and official or canvas. If the + MangaBaka series has no link to that `title_no`, you're warned. +2. **Collect covers.** - **Current:** the portrait poster, the square thumbnail, and the banner and hero art. - - **Historical:** the cover images in archived copies of the series page on - the [Wayback Machine](https://web.archive.org), up to one snapshot per - month. -4. **Check each image.** - - It downloads each image to get its size and hash. - - It classifies the image by shape: portrait → `volume`, square → `other`, - wide → `banner`. - - It drops duplicates and images MangaBaka already has. -5. **Build the submission.** The submission keeps all existing covers and adds - the new ones. The current poster is the default, or the square for canvas - series, which have no poster. Every new cover gets a note recording where - and when it was seen. -6. **Rehost the images.** MangaBaka may not be able to download from Webtoons' - image server, so new covers are re-uploaded to a durable image host: - [catbox.moe](https://catbox.moe) by default, or your own S3-compatible - bucket. -7. **Hand it to you.** It either submits through the MangaBaka API (with your - confirmation) or opens a local review page to help you enter it by hand. + - **Historical:** cover images in archived copies of the series page on the + [Wayback Machine](https://web.archive.org), up to one snapshot per month. +3. **Check each image.** It downloads each image to get its size and hash, + classifies it by shape (portrait → `volume`, square → `other`, wide → + `banner`), and drops duplicates and images MangaBaka already has. +4. **Build the submission.** Existing covers are kept and new ones added. The + current poster is the default (the square for canvas series, which have no + poster). Every new cover gets a note recording where and when it was seen. +5. **Rehost the images** when you click **Rehost selected**. They go to our + storage and **expire after 30 days** — long enough for a MangaBaka review. + If a review takes longer, run the tool again. +6. **Hand it to you.** + - **Without API access (most users):** copy the hosted URLs and paste them + into MangaBaka's cover form for the series (the page links to it). + - **With a personal access token:** the tool shows MangaBaka's preview diff + and, after you confirm, submits the request for review. -Nothing is uploaded or sent to MangaBaka without your confirmation, and API -submissions always go to review (`save_mode: "review"`). You choose the content -rating; the default is `safe`. - -## Installation - -Requires Node.js 22 or newer. - -```sh -npm install -npm run build -npm link # puts `mbseed` on your PATH -``` - -## Usage - -``` -Usage: mbseed [options] - -Options: - --action What to do with the seed - (default: submit if MANGABAKA_TOKEN is set, else serve) - --series MangaBaka series id (skips automatic matching) - --no-history Only use covers currently on the source page - --history-limit Max archived snapshots to inspect (default 60) - --content-rating safe | suggestive | erotica | pornographic - (default: asked interactively, Enter = safe; - required together with --yes) - --image-host - Where to rehost new covers (default catbox) - --json With --action print: print the seed as JSON - --yes With --action submit: don't ask for confirmation - --addr With --action serve: listen address (default localhost:8999) - --verbose Log every HTTP request - -h, --help Show help -``` - -Environment: - -| Variable | Purpose | -|---|---| -| `MANGABAKA_TOKEN` | Personal access token (`mb-…`). Enables `--action submit`. | -| `MANGABAKA_API` | API base URL (default `https://api.mangabaka.org`). | -| `MBSEED_IMAGE_HOST` | Default for `--image-host`. | -| `CATBOX_USERHASH` | Optional. Attaches catbox uploads to your catbox account so you can manage or delete them. | -| `MBSEED_S3_ENDPOINT`, `MBSEED_S3_BUCKET`, `MBSEED_S3_REGION`, `MBSEED_S3_ACCESS_KEY_ID`, `MBSEED_S3_SECRET_ACCESS_KEY`, `MBSEED_S3_PUBLIC_URL` | Settings for `--image-host s3` (Cloudflare R2, Backblaze B2, MinIO or AWS). | - -### Without API access (most users) +Nothing is uploaded or sent to MangaBaka without your action, and submissions +always go to review. You choose the content rating (default `safe`); titles +Webtoons marks as mature show a warning. Your token is sent only to +MangaBaka's API, never to our Worker. MangaBaka's submissions API is only open to contributors, or to users with at -least 25 approved submissions. Without a token, `mbseed` starts a local review -page: - -```sh -mbseed https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95 -# → Review page at http://localhost:8999 -``` - -The page shows each new cover with its thumbnail, size, suggested type, -language, note and a content-rating selector (preset to `safe`). To use it: - -1. Tick the covers you want. -2. Click **Rehost selected**. -3. Copy the hosted URLs. -4. Use the link to open MangaBaka's cover form for the series and paste them in. - The form can't be pre-filled automatically. - -### With API access - -```sh -export MANGABAKA_TOKEN=mb-... -mbseed https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95 -``` - -`mbseed` asks for the content rating (Enter = `safe`), fetches the current -cover set and shows MangaBaka's preview diff. After you confirm, it rehosts the -images and submits the request for review. For unattended runs, pass both -`--yes` and `--content-rating`. If the API refuses access -(403), it falls back to the review page. - -### Just looking - -```sh -mbseed --action print https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95 -mbseed --action print --json > seed.json -``` +least 25 approved submissions. ## Supported sources -| Source | URLs | Current covers | Historical covers | +| Source | Links | Current covers | Historical covers | |---|---|---|---| | WEBTOON (official & canvas) | `https://www.webtoons.com//…/list?title_no=N`, `m.webtoons.com` | poster, square, banner, foreground | Wayback Machine | -Each Webtoons language edition has its own `title_no`. Run `mbseed` once per -edition URL; the covers are tagged with that edition's language. +Each Webtoons language edition has its own `title_no`. Run the tool once per +edition; the covers are tagged with that edition's language. ## Notes and limitations -- **Image hosting.** By default, covers are re-uploaded to catbox.moe, where - files are permanent and need no account. The hosted URL still has to work - when a MangaBaka reviewer looks at the request. - - **Avoid temporary hosts** (e.g. litterbox): links can expire before review. - - **imgur is rejected** by MangaBaka. - - **Skip rehosting** with `--image-host direct` if you know MangaBaka can - fetch the Webtoons URLs directly. - - **Uploads are cached by image hash**, so rerunning doesn't upload the same - image twice. -- **Content rating.** The tool can't judge content, so you choose the rating. - For titles Webtoons marks as mature, it warns you to consider a rating above - `safe`. - **Some covers may only exist in the archive.** If an old cover is no longer on Webtoons' server, its archive.org copy is used instead. - **Not everything is a cover.** Some historical images are promotional or @@ -159,11 +76,6 @@ edition URL; the covers are tagged with that edition's language. ## Development -```sh -npm test # vitest, fixture-based, no network -npm run typecheck -npm run dev -- --action print # run from source with tsx -``` - -To add a new source, see [docs/ARCHITECTURE.md § 4.1](docs/ARCHITECTURE.md#41-the-source-interface-extension-point). -To add a new image host, see [§ 4.6](docs/ARCHITECTURE.md#46-image-hosting-hosting). +Working on this repo, by hand or with an AI agent: start at +[`AGENTS.md`](AGENTS.md). Design: [`docs/specs/`](docs/specs/); current +architecture: [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md). diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 735577c..0000000 --- a/TODO.md +++ /dev/null @@ -1,67 +0,0 @@ -# TODO - -Context: [HANDOVER.md](HANDOVER.md). Design: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). - -## Next - -- [ ] Write the implementation plan from `docs/ARCHITECTURE.md` with - `superpowers:writing-plans`, and **show it to the user before - implementing**. - -## Implementation (rough order; the plan will refine it) - -- [ ] **Scaffold.** `package.json` (bin `mbseed`), TypeScript, vitest, tsx, - cheerio, image-size, aws4fetch, `.gitignore`. Scripts: `build`, `dev`, - `test`, `typecheck`. -- [ ] **Capture test fixtures from real responses:** - - Webtoons pages: Tower of God (official), crow-time (canvas), title_no 6065 - (mature), and a 2015 Wayback snapshot - - Wayback CDX output - - MangaBaka `match`, `search`, `links` and `images` responses for series 9999. -- [ ] **Re-download the OpenAPI spec** from `https://mangabaka.org/api.json`. - Derive `mangabaka/types.ts` from the schemas we use: `SaveSubmissionBody`, - the series-images snapshot and preview, and `V1_Series_Cover_Image`. -- [ ] `model.ts` and `http.ts`: User-Agent, per-host rate limiting, retries, - timeouts, credential scoping. -- [ ] `sources/source.ts` and `registry.ts`: the `Source` interface and - `findSource`. -- [ ] `sources/webtoons/`: `urls.ts` (canonicalisation, image URL - normalisation), `parse.ts` (pure extraction), `index.ts`. -- [ ] `history/wayback.ts`: CDX query, `id_` fetches, merging observations. -- [ ] `images.ts`: download with archive fallback, dimensions, sha256, 20 MB - cap, classification by shape. -- [ ] `covers.ts`: dedupe within the candidates and against existing MangaBaka - covers, default selection, notes. -- [ ] `mangabaka/client.ts` and `resolve.ts`. -- [ ] `hosting/`: the `ImageHost` interface, the upload cache, and the - `catbox`, `s3` and `direct` hosts. -- [ ] `seed/covers.ts`: `buildCoverSeed` and `withHostedUrls`; content-rating - resolution. -- [ ] `actions/print.ts`, `actions/serve.ts` (review page, `/rehost` with a - per-run token) and `actions/submit.ts` (rating → snapshot → preview → - confirm → rehost → save; 409 and 403 handling). -- [ ] `cli.ts` and `pipeline.ts`: flags as in the README, exit codes 0/1/2/3. -- [ ] Manual smoke test: - `mbseed --action print https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95`. -- [ ] Update README and ARCHITECTURE wherever the implementation differs. - -## Verify with real services (once possible) - -- [ ] Does MangaBaka accept catbox.moe URLs? Does it accept - `swebtoon-phinf.pstatic.net`, which would make `direct` viable? -- [ ] Does MangaBaka copy images when the request is submitted, or when it is - approved? -- [ ] Is "main cover" really `is_default: true`? Ask on the MangaBaka Discord - if unclear. -- [ ] Where are personal access tokens created in the MangaBaka UI? Add that - to the README. - -## Later - -- [ ] More sources: Tapas, Tappytoon, Lezhin, Manta. -- [ ] Our own image host as an `ImageHost` implementation. -- [ ] Following Webtoons language editions of the same series. -- [ ] Edits beyond covers (series fields, links), via other submission - resources. -- [ ] A web frontend, browser extension or userscript (the reason for choosing - TypeScript). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 887c17e..6fdba53 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,501 +1,40 @@ # Architecture -`mangabaka-seeder` is a command-line tool that gathers data about a series from -an external site (a "source") and turns it into an edit request ("submission") -for [MangaBaka](https://mangabaka.org). It is modeled on -[yambs](https://codeberg.org/derat/yambs), the MusicBrainz seeder: sources turn -into seeds, and an action decides what happens to them. The user always stays -in control of what is submitted. +How `mangabaka-seeder` is built today. Update in the same commit as any structural change. -The first feature is **covers**: given a Webtoons series URL, collect every -cover that series has used (current and historical) and request them added to -the matching MangaBaka series, with the current poster as the default (main) -cover for the edition's language. +## Overview +No code exists yet. The design is in [`specs/001-webtoons-covers.md`](specs/001-webtoons-covers.md): +one pure TypeScript core (`src/core/`) that turns a MangaBaka series link and a Webtoons series +link into a cover submission, used by three front ends — a static page served by a Cloudflare +Worker (primary), a userscript, and later a Bun CLI. Replace this section with reality as modules land. -This document is the design spec. It records the facts we verified about the -external services, the module boundaries, and the extension points. - ---- - -## 1. Goals and non-goals - -**Goals** - -- One command: `mbseed ` produces a ready-to-review cover submission. -- Find historical covers, not just the current one. -- Work for everyone. With a MangaBaka API token, submit through the API. Without - one, fall back to a local review page that helps with manual entry. -- New sources (Tapas, Tappytoon, Lezhin, Manta, …) can be added as one new - module under `src/sources/`, with no changes to the pipeline, the MangaBaka - client or the actions. -- Never change MangaBaka data silently. API submissions always go through - preview, explicit confirmation and `save_mode: "review"`. -- Never upload anything without explicit user action. The user chooses the - content rating (default `safe`). -- Image hosts are pluggable too (catbox.moe by default, any S3-compatible - bucket, or no rehosting). - -**Non-goals (for now)** - -- Editing series fields other than images, such as titles, links or tags. The - submission layer is keyed by MangaBaka "resource" so this can come later. -- Following a Webtoons series across language editions. Each `title_no` belongs - to one language edition; run the tool once per edition URL. -- Hosting images on our own infrastructure by default. We rehost on a public - image host (catbox.moe), or on a user-configured S3-compatible bucket. - ---- - -## 2. Verified external facts - -These were verified on 2026-10-14. Treat them as assumptions that the fixtures -in `test/fixtures/` pin down. - -### 2.1 Webtoons - -| What | Where | +## Map (planned — spec §4) +| Path | Responsibility | |---|---| -| Portrait poster (official series; 480×623 original) | `meta[property="og:image"]` | -| Square thumbnail | `meta[name="twitter:image"]` | -| Hero background (1920×321, official only) | `div.detail_bg` inline `style` → `background:url('…')` | -| Hero foreground (1200×240 PNG; on canvas series, the 1080×1080 square) | `.detail_header span.thmb img[src]` | -| Title | `meta[property="og:title"]` (fallback `h1.subj` / `h3.subj`) | -| Canonical URL | `link[rel="canonical"]` | -| `title_no` | `title_no` query parameter | -| Language | first path segment (`/en/`, `/fr/`, `/zh-hant/`, …) | -| Canvas vs official | `/canvas/` (or `/challenge/`) in the path; inline JS `isChallenge : true` | +| `src/core/` | Pure pipeline: sources, Wayback history, image inspection, dedupe, seed, MangaBaka client. No DOM, Bun, Node or `GM_*` APIs. | +| `src/web/` | Static page UI. | +| `src/worker/` | Cloudflare Worker: static assets, `/api/fetch` allowlisted proxy, `/api/upload` → R2. | +| `src/userscript/` | Userscript on mangabaka.org; network via `GM_xmlhttpRequest`. | +| `src/cli/` | Later: Bun CLI. | +| `test/fixtures/` | Saved real responses; tests never hit the network. | -- **Canvas series** have no portrait poster. Their `og:image` is a 1080×1080 - square. -- **Image hosts.** `webtoon-phinf.pstatic.net` returns 403 without a Webtoons - `Referer`; `swebtoon-phinf.pstatic.net` serves the same path with no Referer. -- **Original size.** The `?type=…` query selects a resized variant. Removing it - returns the original. `crop540_540` crops posters, so always strip the query. -- **Old hosts.** Old snapshots reference `swebtoon.phinf.naver.net`, which - becomes `swebtoon-phinf.pstatic.net` with the same path. -- **No limits so far.** A browser User-Agent is enough. Age-gated titles still - serve full HTML, and no cookies are required. -- **No usable JSON API.** The app API needs signed requests. We scrape HTML. +## Data flow +Links → series id + `source.resolve` → current + Wayback candidates → inspect (bytes, size, +sha256) → dedupe against MangaBaka → seed → user reviews → rehost (explicit) → submit or manual entry. -### 2.2 Historical covers (Wayback Machine) - -- **Snapshot index.** The CDX API at - `https://web.archive.org/cdx/search/cdx?url=&matchType=prefix&filter=statuscode:200&collapse=timestamp:6&fl=timestamp,original` - returns at most one snapshot per month. -- **Raw pages.** `https://web.archive.org/web/id_/` returns the - page as it was archived, without Wayback's toolbar. -- **History exists.** For Tower of God, snapshots from 2015 to 2025 reference - five distinct cover images. All the older ones tested are **still served live** - from `swebtoon-phinf.pstatic.net`. -- **Fallback.** If an image is no longer live, it is often still at - `https://web.archive.org/web/id_/`. -- **Not every image is a cover.** Some historical `og:image`s are landing-page - art, so candidates are classified by shape and reviewed by the user. - -### 2.3 MangaBaka - -- **Endpoints.** The site is `https://mangabaka.org` and the API is - `https://api.mangabaka.org`. The OpenAPI spec is at - `https://mangabaka.org/api.json`, with docs at - `https://mangabaka.org/data/api`. -- **Public reads.** - - `GET /v2/series/match?q=` and `GET /v2/series/search?q=…` find - series. They are limited to about 30 requests per minute. - - `GET /v1/series/{id}` and `GET /v1/series/{id}/links` return the series - and its links. Links are canonicalised, e.g. - `https://www.webtoons.com/-/-/-/list?title_no=95`. - - `GET /v1/series/{id}/images` returns covers with `type`, `language`, - `index`, `note`, `content_rating`, `is_default` and `hashes.sha256`. -- **Image `type` values:** `volume` (default), `volume_back`, `chapter`, - `season`, `banner`, `audiobook`, `other`. There is no "main" type; the main - cover is the one with `is_default: true`. -- **Submissions API.** Auth is `x-api-key: mb-…`. All endpoints are v0 and - marked internal. - - `GET /v0/my/submissions/series-images/{id}` returns the current image set - and a `version` token. - - `POST /v0/my/submissions/series-images/{id}/preview` is a dry run that - returns a diff. - - `POST /v0/my/submissions/series-images/{id}` saves, with body - `{ version, user_note, save_mode, data: { images: [...] } }`. - - `images` **replaces the whole set**, so it must contain the existing images. - - A stale `version` gives 409. - - Users without the contributor role and with fewer than 25 approved - submissions get 403. -- **Web form.** Without API access, users use - `https://mangabaka.org/data/submissions/series/{id}/images`. It cannot be - pre-filled with query parameters, and its SvelteKit CSRF check blocks - cross-site POSTs. -- **Image URLs.** The spec describes the cover `url` as "Cover image URL. imgur - is rejected (aggressive rate limiting). Acts as the cover identity on save." - - We don't know whether MangaBaka can fetch from `pstatic.net` (it is - hotlink-protected on some hosts). - - We don't know whether MangaBaka copies the image when the request is - submitted or when a reviewer approves it. - - So by default we **rehost** each new image on a host that keeps it - (see §4.6). -- **`content_rating`:** one of `safe`, `suggestive`, `erotica`, - `pornographic`. The user chooses it (§4.7). - ---- - -## 3. Pipeline - -``` - ┌──────────────┐ SourceSeries ┌──────────────────┐ - URL ────▶ │ source.resolve│ ───────────────▶ │ mangabaka.resolve │──▶ MangaBaka series id - └──────────────┘ └──────────────────┘ - │ - ▼ - source.currentCovers + history (Wayback, via source.extractCovers) - │ - ▼ CoverCandidate[] - images.inspect (download, dimensions, sha256, liveness) - │ - ▼ InspectedCover[] - covers.dedupe (within candidates + against existing MangaBaka covers) - │ - ▼ - seed.buildCoverSeed ──▶ CoverSeed (existing images + new images, - │ content_rating chosen by the user) - ▼ - action: print | serve | submit - │ - ▼ (only after explicit user action: serve button / submit confirm) - imageHost.upload ──▶ rehosted URLs replace source URLs in the seed -``` - -Each stage is a pure function or a small module with injected I/O (`fetch`), -so it can be tested with fixtures. - ---- - -## 4. Modules - -``` -src/ - cli.ts argument parsing (node:util parseArgs), wiring, exit codes - pipeline.ts runs the stages above; the only place they are composed - http.ts fetch wrapper: User-Agent, per-host rate limit, retries, timeouts - model.ts shared types (SourceSeries, CoverCandidate, InspectedCover, CoverSeed) - sources/ - source.ts Source interface + helpers - registry.ts list of sources; findSource(url) - webtoons/ - index.ts Webtoons Source implementation - parse.ts pure HTML → { series, covers } extraction - urls.ts URL parsing, canonicalisation, image URL normalisation - history/ - wayback.ts generic Wayback history provider (CDX + archived pages) - images.ts download + probe dimensions + sha256 + classify by shape - covers.ts dedupe, merge observations, pick default - mangabaka/ - client.ts typed API client (public reads + submissions) - resolve.ts source series → MangaBaka series id - types.ts API types (subset of the OpenAPI spec we use) - hosting/ - host.ts ImageHost interface + upload cache (sha256 → url) - registry.ts list of hosts; selectHost(name, env) - catbox.ts catbox.moe (default) - s3.ts S3-compatible bucket (R2, B2, MinIO, AWS) via aws4fetch - direct.ts no rehosting: submit source URLs as-is - seed/ - covers.ts CoverSeed builder: existing + new images → submission body - actions/ - print.ts human-readable or --json output - serve.ts local review page (node:http, single HTML page, no framework) - submit.ts snapshot → preview → confirm → save (review mode) -test/ - fixtures/ saved HTML / CDX / JSON responses - *.test.ts vitest -``` - -### 4.1 The `Source` interface (extension point) - -```ts -export interface Source { - /** Stable id, e.g. "webtoons". Used in notes and CLI output. */ - readonly id: string; - /** Human-readable name, e.g. "WEBTOON". */ - readonly name: string; - - /** True if this source handles the URL. */ - matches(url: URL): boolean; - - /** Fetches the series page and returns normalised series info. */ - resolve(url: URL, ctx: SourceContext): Promise<SourceSeries>; - - /** Covers currently shown by the source. */ - currentCovers(series: SourceSeries, ctx: SourceContext): Promise<CoverCandidate[]>; - - /** - * Optional: pure extraction of cover candidates from a series page's HTML. - * Sources that implement this get Wayback history for free. The history - * provider runs it on archived copies of `series.archiveUrls`. - */ - extractCovers?(html: string, pageUrl: URL): CoverCandidate[]; - - /** Optional source-specific history beyond Wayback (e.g. an API with old art). */ - historicalCovers?(series: SourceSeries, ctx: SourceContext): Promise<CoverCandidate[]>; - - /** Normalises an image URL to its largest, hotlinkable form. */ - normalizeImageUrl(url: string): string; - - /** True if a MangaBaka series link points at this source series. */ - isSameSeries(linkUrl: string, series: SourceSeries): boolean; -} - -export interface SourceSeries { - source: string; // Source.id - externalId: string; // e.g. Webtoons title_no "95" - title: string; - language: string; // MangaBaka language code, e.g. "en", "zh-Hant" - canonicalUrl: string; - archiveUrls: string[]; // URL prefixes to look up in Wayback - mature: boolean; // source flags the series as mature/age-gated (§4.7) - flags: Record<string, boolean>; // source-specific, e.g. { canvas: true } -} - -export interface CoverCandidate { - url: string; // normalised - role: "poster" | "square" | "banner" | "foreground" | "other"; - current: boolean; // shown on the live page now - seenAt?: string; // archive timestamp (YYYYMMDD…) if historical - archivedUrl?: string; // Wayback fallback for the image itself -} -``` - -**Adding a source** (e.g. Tapas) means: - -1. Create `src/sources/tapas/` that implements `Source`. Put the HTML - extraction in a pure `parse.ts` and give it fixture tests. -2. Add it to the array in `src/sources/registry.ts`. -3. Map its language codes to MangaBaka codes. - -The pipeline, Wayback history, image inspection, deduplication, MangaBaka -resolution and all actions work unchanged. A source with an API (Tappytoon, for -example) can skip `extractCovers` and implement `historicalCovers`, or neither. - -### 4.2 History - -`history/wayback.ts` knows nothing about any one source. For each prefix in -`series.archiveUrls` it: - -1. queries CDX with `collapse=timestamp:6` (one snapshot per month, capped by - `--history-limit`, default 60); -2. fetches each snapshot with the `id_` modifier; -3. calls `source.extractCovers(html, originalUrl)`; -4. normalises the image URLs with `source.normalizeImageUrl` and tags each - candidate with `seenAt` and `archivedUrl`. - -Candidates that share a normalised URL are merged, and the merged candidate -records the first and last time it was seen. Wayback failures (timeouts, 5xx, -rate limits) produce a warning and an empty history; they are never fatal. -`--no-history` skips this stage. - -### 4.3 Image inspection and classification - -`images.ts` downloads each candidate (live URL first, then `archivedUrl`) and -records its `width`, `height`, `sha256`, `contentType` and `live` status. -It keeps the bytes in memory for a later upload (§4.6). Covers we've seen -range from about 20 KB to 650 KB. A few dozen candidates fit easily in memory, -and downloads over 20 MB are refused as a safety cap. Dimensions -come from the image header, read with the `image-size` package. Candidates that -cannot be downloaded at all are dropped with a warning. - -Classification by aspect ratio (h / w): - -| Shape | Ratio | MangaBaka `type` | +## External dependencies +| Service | Reached via | If unavailable | |---|---|---| -| portrait | ≥ 1.15 | `volume` | -| square-ish | 0.85 – 1.15 | `other` | -| wide | < 0.5 | `banner` | -| otherwise | – | `other` | +| `api.mangabaka.org` | direct (CORS open) | Nothing works. | +| Webtoons HTML / `swebtoon-phinf.pstatic.net` | Worker proxy or `GM_xmlhttpRequest` | Nothing works. | +| Wayback (`web.archive.org`) | same | Warning; current covers only. | +| Cloudflare R2 | Worker binding | Rehost fails; `direct` host still possible. | -### 4.4 Dedupe and default selection (`covers.ts`) +## Invariants +- `src/core/` never imports from `web/`, `worker/`, `userscript/` or `cli/`. +- The MangaBaka token is only ever sent to `api.mangabaka.org`, never through the Worker. +- Nothing is uploaded or submitted without an explicit user action; saves use `save_mode: "review"`. +- `/api/fetch` only fetches allowlisted hosts; `/api/upload` only takes a source URL, never bytes. -- **Duplicates.** Images with the same `sha256` are merged. The merged entry - keeps the live URL and the widest `seenAt` range. -- **Already on MangaBaka.** Candidates whose `sha256` matches an existing - MangaBaka image are dropped and reported as "already present". -- **Default selection.** - - The **default** is the current live poster. - - If there is none (canvas series), it is the current live square. - - The new default gets `is_default: true`, and every existing image gets - `is_default: false`. This is the "main cover" for the language edition. - - The reviewer on MangaBaka can still reject that change. -- **Notes.** Every new image gets a note, e.g. - `"WEBTOON poster (current)"` or `"WEBTOON poster, seen 2019-03 – 2021-01"`. - -### 4.5 MangaBaka series resolution (`mangabaka/resolve.ts`) - -1. If `--series <id>` is given, use it, and warn if no link of that series - passes `source.isSameSeries`. -2. Otherwise call `/v2/series/match?q=<title>`, then `/v2/series/search`. For - each result, fetch `/v1/series/{id}/links` and keep the series whose link - passes `source.isSameSeries`. For Webtoons that means the same `title_no`. -3. If no series or several series match, stop with exit code 2 and print the - candidates (id, title, URL) so the user can rerun with `--series`. - -### 4.6 Image hosting (`hosting/`) - -MangaBaka must be able to download every image we submit, and it may not do so -until a reviewer approves the request. So new images are **rehosted** on a host -that keeps them. This is a second extension point, shaped like sources: - -```ts -export interface ImageHost { - readonly id: string; // "catbox" | "s3" | "direct" | … - /** Throws a descriptive error if required config/env is missing. */ - checkConfig(env: NodeJS.ProcessEnv): void; - /** Uploads the bytes and returns a public, durable URL. */ - upload(image: { bytes: Uint8Array; sha256: string; contentType: string; - filename: string }, ctx: HostContext): Promise<string>; -} -``` - -| Host | Selected by | Notes | -|---|---|---| -| `catbox` (default) | `--image-host catbox` | Uploads to `https://catbox.moe/user/api.php` (`reqtype=fileupload`, multipart `fileToUpload`). Free, no account, files are permanent. Set `CATBOX_USERHASH` to attach uploads to your catbox account so you can delete them later. | -| `s3` | `--image-host s3` | Any S3-compatible bucket (Cloudflare R2, Backblaze B2, MinIO, AWS). Signed `PUT` with `aws4fetch`. Key is `<source>/<externalId>/<sha256>.<ext>`. Configured by `MBSEED_S3_ENDPOINT`, `MBSEED_S3_BUCKET`, `MBSEED_S3_REGION` (default `auto`), `MBSEED_S3_ACCESS_KEY_ID`, `MBSEED_S3_SECRET_ACCESS_KEY` and `MBSEED_S3_PUBLIC_URL`. The public URL is `<PUBLIC_URL>/<key>`. | -| `direct` | `--image-host direct` | No upload: submits the normalised source URL. Use it if MangaBaka turns out to fetch `swebtoon-phinf.pstatic.net` fine. | - -The upload rules: - -- **Reuse earlier uploads.** Uploads are cached in - `$XDG_CACHE_HOME/mangabaka-seeder/uploads.json`, keyed by host id and - `sha256`, so rerunning never uploads the same image twice. -- **Upload the bytes we checked.** We send the bytes downloaded in §4.3, not a - URL for the host to fetch. That way the host gets exactly the image whose - hash we checked against MangaBaka. -- **No upload without an explicit user action.** Images go up only when the - user clicks *Rehost selected* in `serve`, or answers *yes* in `submit`. - `print` never uploads. -- **Failures are per image.** If an upload fails, that image is reported and - left out of the submission, and the others continue. If every upload fails, - the action aborts with exit code 1. -- **Adding a host** (e.g. a future "ours" server) is one file in `hosting/` plus - one line in `hosting/registry.ts`. - -### 4.7 Content rating - -MangaBaka needs a `content_rating` per image. The tool can't judge this -reliably, so **the user chooses it**, with `safe` as the default: - -- **`--content-rating <rating>`** sets it for all new images. -- **`submit`** with no flag on a terminal asks - `Content rating for new covers [safe/suggestive/erotica/pornographic] (safe):`, - and Enter means `safe`. -- **`submit --yes`** requires `--content-rating` explicitly, so an unattended - run never assumes a rating; without it, exit code 2. On a non-interactive - stdin without `--yes`, the same rule applies. -- **`serve`** has a rating selector on each card, preset to the CLI value - (default `safe`). Selected ratings are shown next to each copied URL. -- **`print`** shows the rating it would use (`--content-rating` or `safe`). -- **Mature titles.** If the source marks the series as mature (e.g. Webtoons - `ageGradeNotice : true`), the prompt and the review page show a warning so - the user considers a rating above `safe`. The default is still `safe`. - -### 4.8 Seed (`seed/covers.ts`) - -`buildCoverSeed(existing, newCovers, series, { contentRating })` returns: - -```ts -interface CoverSeed { - seriesId: number; - userNote: string; // "Add WEBTOON covers from <canonicalUrl>" - images: SubmissionImage[]; // existing (unchanged except is_default) + new - added: InspectedCover[]; // for display - skipped: { cover: InspectedCover; reason: string }[]; -} -``` - -Each new image becomes: -`{ id: null, series_id, work_id: null, url, index: null, index_numeric: null, -language: series.language, type, note, content_rating, is_default }`. -`url` starts as the normalised source URL. `withHostedUrls(seed, urlMap)` -swaps in the rehosted URLs once the upload has happened. - -### 4.9 Actions - -| Action | Default when | Effect | -|---|---|---| -| `print` | `--action print` | Prints series, candidates, the planned seed, and skips. `--json` prints the seed as JSON. No uploads or writes. | -| `serve` | no `MANGABAKA_TOKEN` | Serves `http://localhost:8999` (see below). | -| `submit` | `MANGABAKA_TOKEN` set | Submits through the API (see below). | - -**`serve`** shows a review page with one card per new cover. Each card has: - -- a thumbnail, its dimensions, the suggested type, language and note -- a content-rating selector -- a checkbox. - -The page also has: - -- **"Rehost selected"**, which uploads the selected images to the configured - host and replaces the shown URLs with the hosted ones. With - `--image-host direct` it is hidden. -- **"Copy all selected"**, which copies the URLs. -- a link to `https://mangabaka.org/data/submissions/series/{id}/images`. - -It is a single HTML string rendered server-side with inline JS and no -framework. It binds to localhost only, and its single POST endpoint -(`/rehost`) checks a random per-run token embedded in the page. - -**`submit`**: - -1. Pick the content rating (§4.7). -2. `GET` the snapshot (`version` plus current images) and rebuild the seed on - top of it. -3. `POST …/preview` with the source URLs and print the diff. -4. Ask "Upload N images to <host> and submit for review? [y/N]". `--yes` skips - this. -5. Rehost (§4.6), swap in the hosted URLs, and `POST` with - `save_mode: "review"`. -6. Handle the outcome: - - **409:** fetch the snapshot again, rebuild, preview again and ask again, - at most once. The upload cache prevents re-uploading. - - **403:** explain the 25-approved-submissions requirement and fall back to - `serve`. - ---- - -## 5. Cross-cutting concerns - -- **HTTP.** All network access goes through `http.ts`. It: - - sends a browser User-Agent; - - limits each host to 1 request per second, and Wayback to 1 every 1.5 s; - - times out after 20 s; - - retries twice with backoff on 429/5xx; - - never sends credentials to any host other than the one they belong to. -- **Logging.** Progress goes to stderr and results to stdout, so - `print --json` can be piped. `--verbose` logs every request. -- **Exit codes.** - - 0: success, including "nothing new to add" - - 1: unexpected error - - 2: bad usage, an unsupported URL, or an ambiguous or missing series match - - 3: the user declined to submit. -- **Config.** All configuration comes from environment variables: - - `MANGABAKA_TOKEN` - - `MANGABAKA_API` (default `https://api.mangabaka.org`, for testing) - - `MBSEED_IMAGE_HOST`, the default for `--image-host` - - `CATBOX_USERHASH` - - the `MBSEED_S3_*` variables (§4.6). -- **Secrets.** The MangaBaka token is sent only to the MangaBaka API, and the - S3 credentials only to the S3 endpoint. - ---- - -## 6. Testing - -- **Unit tests with vitest**, run against fixtures saved from real responses: - - Webtoons pages (official, canvas, an archived 2015 snapshot) and CDX output - - MangaBaka match, links, images, snapshot and preview JSON. -- **Pure functions under test:** - - parsing, URL normalisation and canonicalisation - - classification, dedupe and default selection - - seed building, content-rating resolution and the language mapping - - the upload cache. -- **I/O modules** (`pipeline`, `mangabaka/client`, `actions/submit`, - `history/wayback`, `hosting/catbox`, `hosting/s3`) are tested with an injected fake `fetch`. -- **No network in tests.** A manual smoke test is documented in the README: - `mbseed --action print https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95`. +## Known rough edges +See the open questions in spec §7 and [`TODO.md`](TODO.md). diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md new file mode 100644 index 0000000..f208c12 --- /dev/null +++ b/docs/DECISIONS.md @@ -0,0 +1,61 @@ +# Decisions + +Append-only. Never edit or delete an entry — supersede it with a new one that links back. +One entry per decision a future maintainer would otherwise have to reverse-engineer. +No index: `grep -n '^## ' docs/DECISIONS.md` lists every entry (date, title, line number). + +## 2026-10-14 — Adopt AGENTS.md as single source of truth +**Status:** accepted +**Context:** Agents had no durable project context and no consistent workflow between sessions. +The design context lived in `HANDOVER.md`, which is now split into this file, the spec and `docs/TODO.md`. +**Decision:** `AGENTS.md` is canonical; `CLAUDE.md`/`GEMINI.md` are pointers. Specs are permanent in +`docs/specs/`. Plans live in `docs/plans/` only while work is in progress and are removed before the +merge request. Both override the superpowers default paths. +**Alternatives:** Per-tool rule files (drift, synchronisation debt); no rules file (agents re-explore +the repo every session — measurably slower and more tokens). +**Consequences:** Rule changes happen in one file; pointer files must stay thin. Plans are working +artefacts: kept after merge they go stale and contradict the code, so the spec is the record. + +## 2026-10-14 — Temporary image hosting: Cloudflare R2 with a 30-day expiry +**Status:** accepted (supersedes "Rehosting on catbox.moe" below) +**Context:** A static page can't upload to catbox.moe (no CORS). The user said temporary hosting is +fine with an expiry of at most a month: if moderators take longer, the user reruns. +**Decision:** The Worker rehosts into R2 at `covers/<sha256>.<ext>`, with a bucket lifecycle rule +deleting objects after 30 days. `direct` (no rehost) stays as an option. +**Alternatives:** litterbox (CORS works, but 72h maximum — likely shorter than a review); +0x0.st (CORS and MangaBaka acceptance unverified); catbox (no CORS, permanent). +**Consequences:** Needs a Cloudflare account with a card on file (R2). Expected cost is $0 on +free tiers; abuse of the Worker is the cost risk, hence the allowlist and the per-IP rate limit. + +## 2026-10-14 — Static page + Cloudflare Worker, and a userscript; Bun; links given directly +**Status:** accepted (supersedes "Stack: TypeScript / Node CLI" and "Automatic series matching" below) +**Context:** The user wants a static web page first, with the CLI as secondary, Bun instead of Node, +and to pass both the MangaBaka and Webtoons links. Checked 2026-10-14: `api.mangabaka.org` sends +`access-control-allow-origin: *` (submissions too); Webtoons HTML, pstatic images and Wayback do not. +**Decision:** One pure core with an injected `Transport`. The page uses a Worker proxy (allowlisted +hosts) for Webtoons/Wayback and calls MangaBaka directly. The userscript uses `GM_xmlhttpRequest` +and needs no proxy for fetching. The user chose to pursue both front ends. Tooling is Bun +(`bun test`, `bun build`). No automatic MangaBaka search; a `title_no` link check only warns. +**Alternatives:** Local Bun server only (not a hosted page); userscript only. +**Consequences:** Two front ends to keep in step; the Worker is public infrastructure to protect. +Cloudflare IPs may be throttled by Webtoons/Wayback — the userscript doesn't have that risk. + +## 2026-10-14 — Original design decisions (from the design conversation) +**Status:** partly superseded — see the two entries above +**Context:** Recorded in the former `HANDOVER.md`. +**Decision:** +- Stack: TypeScript, chosen over Go (yambs is Go) to keep a browser extension or userscript possible. + *Runtime superseded: Bun, not Node.* +- Submission path: the API when a token is available, otherwise help with manual entry. +- Which images: all — poster, square thumbnail, banner/background and foreground art. +- Historical covers through the Wayback Machine. +- Main cover: the current poster (square for canvas) gets `is_default: true`, existing covers `false`. +- Rehosting on catbox.moe by default. *Superseded: R2, 30 days.* +- Content rating chosen by the user, default `safe`; mature titles show a warning. +- Image bytes kept in memory; sizes up to ~650 KB seen; 20 MB cap. +- Extensibility: new sources and image hosts are one module plus one registry line. +- Safety: nothing uploaded or sent without explicit user action; `save_mode: "review"`. +- Not now: fields other than images; following language editions. +- Automatic series matching via `/v2/series/match`. *Superseded: links given directly.* +**Alternatives:** not recorded +**Consequences:** See the spec. diff --git a/docs/PERMISSIONS.md b/docs/PERMISSIONS.md new file mode 100644 index 0000000..de82caf --- /dev/null +++ b/docs/PERMISSIONS.md @@ -0,0 +1,45 @@ +# Agent permissions + +Source of truth for what AI coding agents may run here. Tool files are renderings of this page: +`.claude/settings.json`, `.codex/config.toml`, `.codex/rules/project.rules`, `.gemini/settings.json`. +Change this page and every tool file in the same commit. + +## Verified commands + +Run in this session. Only `worked` rows may appear in a tier, a tool file, or `AGENTS.md` Commands. + +| Command | Result | +|---|---| +| `git status`, `git log`, `git diff`, `git mv` | worked | +| `bun --version` | worked (1.3.14) | +| `bun test` | failed — no test files yet (no `package.json`, no code) | +| `bun install --frozen-lockfile` | not run — no `package.json` / `bun.lock` yet | +| `bunx tsc --noEmit` | not run — `typescript` not installed yet | + +TODO(maintainer): confirm every row above that is not worked, once the Bun scaffold exists, then +add the verified ones to Allow here and in each tool file. + +## Tiers + +| Tier | Commands | Why | +|---|---|---| +| Allow | `git status`, `git diff`, `git log`, `git show`, `git branch --list`, `git rev-parse`, `git add`, `git commit` | Inspection and the dev loop; no prompt needed. | +| Ask | `git push`, `git merge`, `git rebase`, `git switch`, `git restore`, `git commit --amend`, `rm` (any); `bun add`, `bun remove`, `bun update`, `bun upgrade`; `bunx`, `npx` (download and run code, incl. `wrangler deploy`) | Leaves the machine, rewrites work, changes dependencies, or deploys — a human decides. | +| Deny | `git push --force` / `-f` / `--force-with-lease`, `git reset --hard`, `git clean`, `git add -f` / `--force`, `git commit --no-verify` / `-n`, `rm -r` / `-R` / `-rf` / `-fr` / `-Rf` / `-fR` / `--recursive`, `sudo`; reading `.env`, `.env.*`, `.dev.vars`, `secrets/**` | Destroys work, skips checks, or exposes secrets. | + +## Known gaps +- Allowlists save prompts; they are not a security boundary. Use the tool's sandbox for isolation. +- Prefix rules are bypassable: `git -C dir push`, flags placed later (`git push origin main --force`), + `+refspec` pushes. Claude Code's `:*` matches at a word boundary, so each flag cluster must be + listed — which is why `git clean` is denied whole and `rm` always asks. +- Allowed test, lint, and build commands run code from the repo; an agent that can edit + `package.json` or the tests can run anything through them. +- Denying reads of `.env` / `.dev.vars` does not stop a script that opens them itself, and + git can print staged or committed secrets (`git show HEAD:.env`). `git add -f` is denied so an agent + cannot stage an ignored secret; keep real secrets out of the working tree. +- Codex: loaded only once the project is trusted; `approval_policy` here can loosen a stricter user + setting. No `allow` rules (in Codex that means "outside the sandbox"); `git add`/`git commit` + ask for escalation because `.git/` is read-only in `workspace-write`. Codex cannot deny file reads. + `--force-with-lease=<ref>`, `+refspec`, and later-placed flags only match the Ask rule. +- Gemini: ignored until the folder is trusted; `tools.exclude` is deprecated but is the only + project-level deny. No Ask tier (anything not allowed asks). Gemini cannot deny file reads. diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 0000000..ff79173 --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,59 @@ +# TODO + +Durable backlog. Agents append here instead of leaving `TODO` comments in source. +Small, specific, independently actionable. Remove only when actually done. +Design: [specs/001-webtoons-covers.md](specs/001-webtoons-covers.md). + +## Now +- [ ] User signs off the revised spec `docs/specs/001-webtoons-covers.md` (status: draft). +- [ ] Write the implementation plan with `superpowers:writing-plans` into `docs/plans/001-webtoons-covers.md`, + and **show it to the user before implementing** (the user asked for this in the design session). + +## Next (rough order; the plan will refine it) +- [ ] **Scaffold with Bun.** `package.json`, `bun.lock`, TypeScript, `wrangler`, `.gitignore`. + Scripts for test, typecheck, build (page, Worker, userscript). Then fill the Commands in + `AGENTS.md` and the Verified commands in `docs/PERMISSIONS.md` from runs that worked. +- [ ] **Capture fixtures from real responses:** Webtoons pages (Tower of God, crow-time canvas, + title_no 6065 mature, a 2015 Wayback snapshot), Wayback CDX output, MangaBaka + `series`, `links`, `images` for 9999, and a snapshot/preview if a token is available. +- [ ] **Re-download the OpenAPI spec** from `https://mangabaka.org/api.json`; derive + `core/mangabaka/types.ts` (`SaveSubmissionBody`, series-images snapshot and preview, + `V1_Series_Cover_Image`). +- [ ] `core/`: model, transport (rate limit, retries, timeouts, token scoping), links, sources + + registry, webtoons (urls, parse, index), history/wayback, images, covers, mangabaka client, + hosting (worker, direct), seed. +- [ ] `worker/`: static assets, `/api/fetch` allowlist, `/api/upload` → R2, per-IP rate limit. +- [ ] `web/`: two-link form, review cards, rehost, copy, token flow (preview → confirm → save). +- [ ] `userscript/`: panel on mangabaka.org series pages, `GM_xmlhttpRequest` transport, token in + `GM_setValue`, build to `mbseed.user.js` with metadata banner. +- [ ] Manual smoke test against Tower of God (9999 / title_no 95). +- [ ] Update README, `docs/ARCHITECTURE.md` and `PRODUCT.md`/`DESIGN.md` to match reality. + +## Verify with real services (once possible) +- [ ] Do Webtoons and Wayback serve requests from Cloudflare Worker IPs? +- [ ] Does MangaBaka accept our R2 URLs? Does it accept `swebtoon-phinf.pstatic.net` (`direct`)? +- [ ] Does MangaBaka copy images on submission or on approval? +- [ ] Is "main cover" really `is_default: true`? Ask on the MangaBaka Discord if unclear. +- [ ] Where are personal access tokens created in the MangaBaka UI? Add it to the README. +- [ ] Does the MangaBaka cover form accept fields filled by a userscript? + +## Someday +- [ ] Bun CLI on the same core. +- [ ] More sources: Tapas, Tappytoon, Lezhin, Manta. +- [ ] Following Webtoons language editions of the same series. +- [ ] Edits beyond covers (series fields, links), via other submission resources. +- [ ] CI: check `AGENTS.md`, `CLAUDE.md`, `GEMINI.md` and the `docs/` paths exist; fail if + `AGENTS.md` exceeds 150 lines. + +## Recurring +- [ ] Maintain `AGENTS.md`: + - **Add a rule only in response to a real failure.** When an agent repeatedly gets something wrong, + that is a gap; add one line, ideally with a code example. Do not pre-empt hypothetical mistakes. + - **Update in the same PR as the convention change.** A stale rules file is worse than none — it + actively misleads. + - **Prune quarterly.** Delete rules whose convention has changed or whose mistake no longer occurs. + - **Optional, cheap validation:** run the same task twice in clean sessions, with and without the + file, and compare steps and token use. + +## Known issues +- None yet (no code). diff --git a/docs/plans/README.md b/docs/plans/README.md new file mode 100644 index 0000000..c4ace6f --- /dev/null +++ b/docs/plans/README.md @@ -0,0 +1,11 @@ +# Plans + +Implementation plans from the superpowers writing-plans workflow. **Transient:** a plan lives only on +its feature branch and is removed with `git rm` before the merge request is opened. This README stays. + +- Naming: `NNN-short-slug.md`, matching its spec number. +- Links back to its spec. Bite-sized checkbox tasks with exact file paths, test commands, and + commit checkpoints. +- Written for an engineer with zero context for this codebase — spell out the obvious. +- Tick checkboxes as you go so another agent can resume an interrupted session. +- Front matter: `status: draft | approved | in-progress`. There is no `done` — a finished plan is deleted. diff --git a/docs/specs/001-webtoons-covers.md b/docs/specs/001-webtoons-covers.md new file mode 100644 index 0000000..759c9ac --- /dev/null +++ b/docs/specs/001-webtoons-covers.md @@ -0,0 +1,561 @@ +--- +status: draft +--- + +# 001 — Webtoons covers + +`mangabaka-seeder` gathers data about a series from an external site (a +"source") and turns it into an edit request ("submission") for +[MangaBaka](https://mangabaka.org). It is modelled on +[yambs](https://codeberg.org/derat/yambs), the MusicBrainz seeder: sources turn +into seeds, and an action decides what happens to them. The user always stays +in control of what is submitted. + +The first feature is **covers**: given a MangaBaka series link and a Webtoons +series link, collect every cover the Webtoons series has used (current and +historical) and request them added to the MangaBaka series, with the current +poster as the default (main) cover for the edition's language. + +This spec records the facts we verified about the external services, the +module boundaries and the extension points. Revised 2026-10-14: the user gives +both links directly, the product is a static web page first plus a userscript, +the runtime is Bun, and images go to a temporary host with a 30-day expiry. + +--- + +## 1. Goals and non-goals + +**Goals** + +- **Two links in, a reviewable cover submission out.** The user pastes a + MangaBaka series link and a Webtoons series link. There is no automatic + series matching. +- **Two front ends on one core:** + 1. a **static web page** (the primary product), served by a Cloudflare Worker + that also proxies the sites browsers cannot read; + 2. a **userscript** (Tampermonkey/Violentmonkey) on mangabaka.org, which needs + no proxy because `GM_xmlhttpRequest` is not bound by CORS. + A Bun CLI on the same core comes later. +- **Find historical covers**, not just the current one. +- **Work for everyone.** With a MangaBaka personal access token, submit through + the API. Without one, help the user enter the covers by hand. +- **New sources** (Tapas, Tappytoon, Lezhin, Manta, …) are one new module under + `src/core/sources/`, with no changes to the pipeline, the MangaBaka client or + the front ends. +- **Never change MangaBaka data silently.** API submissions always go through + preview, explicit confirmation and `save_mode: "review"`. +- **Never upload anything without explicit user action.** The user chooses the + content rating (default `safe`). + +**Non-goals (for now)** + +- Editing series fields other than images, such as titles, links or tags. The + submission layer is keyed by MangaBaka "resource" so this can come later. +- Following a Webtoons series across language editions. Each `title_no` belongs + to one language edition; run the tool once per edition link. +- Permanent image hosting. Rehosted images only need to outlive the MangaBaka + review; they expire after 30 days (§4.6). + +--- + +## 2. Verified external facts + +Verified on 2026-10-14. Treat them as assumptions that the fixtures in +`test/fixtures/` pin down. + +### 2.1 Webtoons + +| What | Where | +|---|---| +| Portrait poster (official series; 480×623 original) | `meta[property="og:image"]` | +| Square thumbnail | `meta[name="twitter:image"]` | +| Hero background (1920×321, official only) | `div.detail_bg` inline `style` → `background:url('…')` | +| Hero foreground (1200×240 PNG; on canvas series, the 1080×1080 square) | `.detail_header span.thmb img[src]` | +| Title | `meta[property="og:title"]` (fallback `h1.subj` / `h3.subj`) | +| Canonical URL | `link[rel="canonical"]` | +| `title_no` | `title_no` query parameter | +| Language | first path segment (`/en/`, `/fr/`, `/zh-hant/`, …) | +| Canvas vs official | `/canvas/` (or `/challenge/`) in the path; inline JS `isChallenge : true` | +| Mature | inline JS `ageGradeNotice : true` | + +- **Canvas series** have no portrait poster. Their `og:image` is a 1080×1080 + square. +- **Image hosts.** `webtoon-phinf.pstatic.net` returns 403 without a Webtoons + `Referer`; `swebtoon-phinf.pstatic.net` serves the same path with no Referer. +- **Original size.** The `?type=…` query selects a resized variant. Removing it + returns the original. `crop540_540` crops posters, so always strip the query. +- **Old hosts.** Old snapshots reference `swebtoon.phinf.naver.net`, which + becomes `swebtoon-phinf.pstatic.net` with the same path. +- **No limits so far.** A browser User-Agent is enough. Age-gated titles still + serve full HTML, and no cookies are required. (Tested from a residential IP, + not from Cloudflare's; see §7.) +- **No usable JSON API.** The app API needs signed requests. We scrape HTML. +- **CORS:** neither the series HTML nor `swebtoon-phinf` images send + `Access-Control-Allow-Origin`. A page can show the images in `<img>` but + cannot read their bytes. + +### 2.2 Historical covers (Wayback Machine) + +- **Snapshot index.** The CDX API at + `https://web.archive.org/cdx/search/cdx?url=<page>&matchType=prefix&filter=statuscode:200&collapse=timestamp:6&fl=timestamp,original` + returns at most one snapshot per month. +- **Raw pages.** `https://web.archive.org/web/<ts>id_/<original>` returns the + page as it was archived, without Wayback's toolbar. +- **History exists.** Tower of God has about 2000 snapshots (2015–2026) that + reference five distinct cover images. All the older ones tested are **still + served live** from `swebtoon-phinf.pstatic.net`. +- **Fallback.** If an image is no longer live, it is often still at + `https://web.archive.org/web/<ts>id_/<image-url>`. +- **Not every image is a cover.** Some historical `og:image`s are landing-page + art, so candidates are classified by shape and reviewed by the user. +- **CORS:** no `Access-Control-Allow-Origin` seen on snapshots. CDX returned + 503 during the check, so treat Wayback as unavailable from a plain page and + flaky in general. + +### 2.3 MangaBaka + +- **Endpoints.** The site is `https://mangabaka.org` and the API is + `https://api.mangabaka.org` (`api.mangabaka.dev` is deprecated). The OpenAPI + spec is at `https://mangabaka.org/api.json` (~700 KB), with docs at + `https://mangabaka.org/data/api`. +- **CORS is open.** `api.mangabaka.org` answers with + `access-control-allow-origin: *` and allows any header (including + `x-api-key`) and `POST`, on both public reads and + `/v0/my/submissions/…`. So the page and the userscript talk to MangaBaka + **directly**, never through the Worker. +- **Public reads.** + - `GET /v1/series/{id}` and `GET /v1/series/{id}/links` return the series + and its links. Links are canonicalised, e.g. + `https://www.webtoons.com/-/-/-/list?title_no=95`. + - `GET /v1/series/{id}/images` returns covers with `type`, `language`, + `index`, `note`, `content_rating`, `is_default` and `hashes.sha256`. + - Rate limits: about 180 requests per minute (search, which we no longer + use, is about 30). +- **Image `type` values:** `volume` (default), `volume_back`, `chapter`, + `season`, `banner`, `audiobook`, `other`. There is no "main" type; the main + cover is the one with `is_default: true`. +- **Submissions API.** Auth is `x-api-key: mb-…` (a personal access token). + All endpoints are v0 and marked internal. + - `GET /v0/my/submissions/series-images/{id}` returns the current image set + and a `version` token. + - `POST /v0/my/submissions/series-images/{id}/preview` is a dry run that + returns a diff. + - `POST /v0/my/submissions/series-images/{id}` saves, with body + `{ version, user_note, save_mode, data: { images: [...] } }`. + - `images` **replaces the whole set**, so it must contain the existing images. + - A stale `version` gives 409. + - Users without the contributor role and with fewer than 25 approved + submissions get 403. +- **Web form.** Without API access, users use + `https://mangabaka.org/data/submissions/series/{id}/images`. It cannot be + pre-filled with query parameters, and its SvelteKit CSRF check blocks + cross-site POSTs. (A userscript running *on* that page can still fill its + fields; see §4.9.) +- **Image URLs.** The spec describes the cover `url` as "Cover image URL. imgur + is rejected (aggressive rate limiting). Acts as the cover identity on save." + We don't know whether MangaBaka copies the image on submission or on + approval, so the hosted URL must survive the review (§4.6). +- **`content_rating`:** one of `safe`, `suggestive`, `erotica`, + `pornographic`. The user chooses it (§4.7). +- **Test series:** Tower of God is **series 9999** + (`https://mangabaka.org/manhwa/9999/Tower-of-God`; Webtoons `title_no=95`). + +### 2.4 Image hosts + +| Host | Browser CORS | Expiry | Verdict | +|---|---|---|---| +| catbox.moe | no | permanent | Not usable from a page, and permanent is more than we need. | +| litterbox.catbox.moe | yes | 1h / 12h / 24h / 72h only | 72h is likely shorter than a review. | +| 0x0.st | unverified | configurable | Not used. | +| **Cloudflare R2** (our bucket) | n/a (behind our Worker) | 30-day lifecycle rule | **Chosen.** | + +--- + +## 3. Pipeline + +``` + MangaBaka link ──▶ series id ──▶ GET /v1/series/{id}, /links, /images (direct; CORS open) + │ sanity check: a link has the same title_no + Webtoons link ──▶ source.resolve ───────┤ + │ ▼ + source.currentCovers + history (Wayback, via source.extractCovers) + │ + ▼ CoverCandidate[] + images.inspect (bytes → dimensions, sha256, liveness) + │ + ▼ InspectedCover[] + covers.dedupe (within candidates + against existing MangaBaka covers) + │ + ▼ + seed.buildCoverSeed ──▶ CoverSeed (existing + new images, user's content rating) + │ + ▼ front end: review UI → (explicit action) → rehost → submit / fill form +``` + +Every network call goes through an injected `Transport` (§4.2), so every stage +runs unchanged in the Worker-backed page, the userscript, the CLI and tests. + +--- + +## 4. Modules + +``` +src/ + core/ pure TypeScript; no DOM, no Bun/Node APIs, no GM_* + model.ts shared types (SourceSeries, CoverCandidate, InspectedCover, CoverSeed) + transport.ts Transport interface; per-host rate limit, retries, timeouts + pipeline.ts runs the stages above; the only place they are composed + links.ts parse the two input links (MangaBaka series id, source URL) + sources/ + source.ts Source interface + helpers + registry.ts list of sources; findSource(url) + webtoons/ + index.ts Webtoons Source implementation + parse.ts pure HTML → { series, covers } extraction + urls.ts URL parsing, canonicalisation, image URL normalisation + history/wayback.ts generic Wayback history provider (CDX + archived pages) + images.ts probe dimensions from header bytes, sha256 (WebCrypto), classify + covers.ts dedupe, merge observations, pick default + mangabaka/ + client.ts typed API client (public reads + submissions) + types.ts API types (subset of the OpenAPI spec we use) + hosting/ + host.ts ImageHost interface + worker.ts upload via our Worker → R2 (default) + direct.ts no rehosting: submit source URLs as-is + seed/covers.ts CoverSeed builder: existing + new images → submission body + web/ the static page (HTML + TS bundled by `bun build`) + worker/ Cloudflare Worker: serves web/ as static assets, /api/fetch, /api/upload + userscript/ entry + metadata banner, bundled by `bun build` into mbseed.user.js + cli/ later: Bun CLI on the same core +test/ + fixtures/ saved HTML / CDX / JSON responses + *.test.ts bun test +``` + +Dependency direction: `web/`, `userscript/`, `worker/` and `cli/` may import +`core/`; `core/` imports none of them. `worker/` imports only the small part of +`core/` it needs (host allowlist, image sniffing). + +### 4.1 The `Source` interface (extension point) + +```ts +export interface Source { + /** Stable id, e.g. "webtoons". Used in notes and output. */ + readonly id: string; + /** Human-readable name, e.g. "WEBTOON". */ + readonly name: string; + + /** True if this source handles the URL. */ + matches(url: URL): boolean; + + /** Fetches the series page and returns normalised series info. */ + resolve(url: URL, ctx: SourceContext): Promise<SourceSeries>; + + /** Covers currently shown by the source. */ + currentCovers(series: SourceSeries, ctx: SourceContext): Promise<CoverCandidate[]>; + + /** + * Optional: pure extraction of cover candidates from a series page's HTML. + * Sources that implement this get Wayback history for free. + */ + extractCovers?(html: string, pageUrl: URL): CoverCandidate[]; + + /** Optional source-specific history beyond Wayback (e.g. an API with old art). */ + historicalCovers?(series: SourceSeries, ctx: SourceContext): Promise<CoverCandidate[]>; + + /** Normalises an image URL to its largest, hotlinkable form. */ + normalizeImageUrl(url: string): string; + + /** True if a MangaBaka series link points at this source series. */ + isSameSeries(linkUrl: string, series: SourceSeries): boolean; + + /** Hosts this source fetches from; the Worker proxy allowlist is built from these. */ + readonly hosts: readonly string[]; +} + +export interface SourceSeries { + source: string; // Source.id + externalId: string; // e.g. Webtoons title_no "95" + title: string; + language: string; // MangaBaka language code, e.g. "en", "zh-Hant" + canonicalUrl: string; + archiveUrls: string[]; // URL prefixes to look up in Wayback + mature: boolean; // source flags the series as mature/age-gated (§4.7) + flags: Record<string, boolean>; // source-specific, e.g. { canvas: true } +} + +export interface CoverCandidate { + url: string; // normalised + role: "poster" | "square" | "banner" | "foreground" | "other"; + current: boolean; // shown on the live page now + seenAt?: string; // archive timestamp (YYYYMMDD…) if historical + archivedUrl?: string; // Wayback fallback for the image itself +} +``` + +**Adding a source** (e.g. Tapas): create `src/core/sources/tapas/` implementing +`Source` (HTML extraction in a pure `parse.ts` with fixture tests), add it to +`registry.ts`, and map its language codes to MangaBaka codes. Its `hosts` flow +into the Worker allowlist automatically. + +### 4.2 Transport (how each front end reaches the network) + +```ts +export interface Transport { + /** Fetch-like. Returns a standard Response. */ + fetch(url: string, init?: { method?: string; headers?: Record<string, string>; + body?: BodyInit; signal?: AbortSignal }): Promise<Response>; +} +``` + +| Front end | MangaBaka API | Webtoons / Wayback / pstatic | +|---|---|---| +| Static page | `fetch` directly (CORS open) | `fetch("/api/fetch?url=…")` on our Worker (same origin) | +| Userscript | `fetch` directly | `GM_xmlhttpRequest`, wrapped into a `Response` | +| CLI (later) | Bun `fetch` | Bun `fetch` | +| Tests | fake transport serving fixtures | fake transport serving fixtures | + +Shared behaviour lives in `core/transport.ts` and wraps whichever transport is +injected: + +- limits each host to 1 request per second, and Wayback to 1 every 1.5 s; +- times out after 20 s; +- retries twice with backoff on 429/5xx; +- never sends the MangaBaka token anywhere except `api.mangabaka.org`. In + particular the token never passes through the Worker. + +### 4.3 History + +`history/wayback.ts` knows nothing about any one source. For each prefix in +`series.archiveUrls` it: + +1. queries CDX with `collapse=timestamp:6` (one snapshot per month, capped by a + history limit, default 60); +2. fetches each snapshot with the `id_` modifier; +3. calls `source.extractCovers(html, originalUrl)`; +4. normalises the image URLs and tags each candidate with `seenAt` and + `archivedUrl`. + +Candidates that share a normalised URL are merged; the merged candidate records +the first and last time it was seen. Wayback failures (timeouts, 5xx, rate +limits) produce a warning and an empty history; they are never fatal. The user +can turn history off. + +### 4.4 Image inspection and classification + +`images.ts` downloads each candidate through the transport (live URL first, +then `archivedUrl`) and records `width`, `height`, `sha256` (WebCrypto +`crypto.subtle.digest`, available in browsers, Workers and Bun), `contentType` +and `live` status. Dimensions come from the image header bytes, so no DOM is +needed. Covers we've seen range from about 20 KB to 650 KB; downloads over +20 MB are refused. Candidates that cannot be downloaded at all are dropped with +a warning. + +Classification by aspect ratio (h / w): + +| Shape | Ratio | MangaBaka `type` | +|---|---|---| +| portrait | ≥ 1.15 | `volume` | +| square-ish | 0.85 – 1.15 | `other` | +| wide | < 0.5 | `banner` | +| otherwise | – | `other` | + +### 4.5 Series and dedupe (`links.ts`, `covers.ts`) + +- **Inputs.** `links.ts` accepts a MangaBaka series link or a bare id. Verified + forms: `https://mangabaka.org/9999` redirects (301) to the canonical + `https://mangabaka.org/manhwa/9999/Tower-of-God`; `/series/9999` is 404. So + take the first all-digits path segment. Plus a Webtoons series URL (`www.` + or `m.`). +- **Sanity check.** If none of the MangaBaka series' links passes + `source.isSameSeries` (same `title_no`), show a warning and require the user + to confirm. It is a warning, not an error: the link may simply be missing on + MangaBaka. +- **Duplicates.** Images with the same `sha256` are merged; the merged entry + keeps the live URL and the widest `seenAt` range. +- **Already on MangaBaka.** Candidates whose `sha256` matches an existing + MangaBaka image are dropped and shown as "already present". +- **Default selection.** The default is the current live poster, or the current + live square if there is none (canvas). The new default gets + `is_default: true` and every existing image gets `is_default: false`. The + reviewer can still reject that. +- **Notes.** Every new image gets a note, e.g. `"WEBTOON poster (current)"` or + `"WEBTOON poster, seen 2019-03 – 2021-01"`. + +### 4.6 Image hosting + +MangaBaka must be able to download each submitted image until a reviewer acts, +which may take weeks. Rehosted images expire after **30 days**; if the review +takes longer, the user reruns the tool. + +```ts +export interface ImageHost { + readonly id: string; // "worker" | "direct" | … + /** Rehosts one inspected image; returns a public URL and the sha256 the host stored. */ + upload(image: { sourceUrl: string; sha256: string; contentType: string }, + ctx: HostContext): Promise<{ url: string; sha256: string }>; +} +``` + +| Host | Notes | +|---|---| +| `worker` (default) | `POST /api/upload { url }` on our Worker. The Worker fetches the image itself from an allowlisted host, checks it is an image ≤ 20 MB, stores it in R2 at `covers/<sha256>.<ext>`, and returns its public URL and sha256. | +| `direct` | No upload: submits the normalised source URL. Use it if MangaBaka turns out to fetch `swebtoon-phinf.pstatic.net` fine. | + +Rules: + +- **Upload only what we checked.** The client sends the source URL, not bytes, + so the endpoint can't be used to host arbitrary files. The client compares + the returned `sha256` with the one it inspected and rejects a mismatch. +- **Re-uploads are idempotent.** The key is the hash, so rerunning overwrites + the same object (and restarts its 30 days). +- **No upload without an explicit user action.** +- **Failures are per image.** A failed upload is reported and the image left + out; if every upload fails, the action aborts. +- **Adding a host** is one file in `core/hosting/` plus one registry line. + +### 4.7 Content rating + +MangaBaka needs a `content_rating` per image. The tool can't judge it, so **the +user chooses it**, default `safe`: one selector for all new images, plus a +per-card override. If the source marks the series as mature (Webtoons +`ageGradeNotice : true`), the UI shows a warning to consider a rating above +`safe`. The default stays `safe`. Unattended runs (CLI `--yes`, later) must +pass a rating explicitly. + +### 4.8 Seed (`seed/covers.ts`) + +`buildCoverSeed(existing, newCovers, series, { contentRating })` returns: + +```ts +interface CoverSeed { + seriesId: number; + userNote: string; // "Add WEBTOON covers from <canonicalUrl>" + images: SubmissionImage[]; // existing (unchanged except is_default) + new + added: InspectedCover[]; // for display + skipped: { cover: InspectedCover; reason: string }[]; +} +``` + +Each new image becomes +`{ id: null, series_id, work_id: null, url, index: null, index_numeric: null, +language: series.language, type, note, content_rating, is_default }`. +`withHostedUrls(seed, urlMap)` swaps in the rehosted URLs after upload. + +### 4.9 Front ends + +**Static page** (`src/web/`, served by the Worker): + +1. Two inputs: MangaBaka series link, Webtoons series link. Optional: "include + history" (on), history limit. +2. Runs the pipeline and shows one card per new cover (thumbnail, dimensions, + suggested type, language, note, content-rating selector, checkbox), plus + "already present" and skipped covers. +3. **Rehost selected** uploads through the Worker and replaces the shown URLs. +4. **Without a token:** "Copy all selected" and a link to + `https://mangabaka.org/data/submissions/series/{id}/images` for manual + entry. +5. **With a token:** the user pastes a personal access token (kept only in + memory unless they tick "remember on this device", then `localStorage`). + The page fetches the snapshot, shows the preview diff, and on confirmation + rehosts and saves with `save_mode: "review"`. 409: refetch, rebuild, + re-preview, ask again (once). 403: explain the 25-approved-submissions rule + and fall back to manual entry. + +**Userscript** (`src/userscript/`, built to `mbseed.user.js`): + +- `@match https://mangabaka.org/*`. On a series page it adds a "Seed covers" + panel; the MangaBaka series id comes from the page URL and the user pastes + the Webtoons link. +- Fetches Webtoons/Wayback/pstatic with `GM_xmlhttpRequest` + (`@connect webtoons.com`, `pstatic.net`, `web.archive.org`, our Worker's + host). Same review cards as the page. +- Rehosts through the same Worker `/api/upload` (no R2 keys in the script). +- Submits through the API when a token is set (stored with `GM_setValue`). +- **Without a token**, on the cover form page it can fill the form fields + for the selected covers. Whether the form accepts programmatically filled + fields is unverified (§7). + +**Worker** (`src/worker/`): + +| Route | Behaviour | +|---|---| +| `GET /*` | Static assets of the page. | +| `GET /api/fetch?url=` | Fetches `url` only if its host is in the allowlist (union of `Source.hosts` plus `web.archive.org`), with a browser User-Agent; streams the response back. Anything else: 403. | +| `POST /api/upload` | §4.6. | + +Both API routes reject other origins (except the userscript, which is not bound +by CORS), cap response size at 20 MB, and are rate-limited per IP. The R2 +bucket has a lifecycle rule deleting objects after 30 days and is publicly +readable through a custom domain or `r2.dev`. + +--- + +## 5. Cross-cutting concerns + +- **Secrets.** The MangaBaka token stays in the user's browser (or userscript + storage) and is only sent to `api.mangabaka.org`. The Worker holds no + secrets; R2 is a binding. +- **Errors.** The UI shows a per-stage status; Wayback failure is a warning, + not a failure. +- **Cost.** Cloudflare free tiers cover expected use: a run is ~60–100 Worker + requests and ≤ ~6 MB stored for 30 days. Abuse of the proxy is the main cost + risk, hence the allowlist, the size cap and the per-IP rate limit. + +## 6. Testing + +- **`bun test`** against fixtures saved from real responses: Webtoons pages + (official, canvas, mature, a 2015 Wayback snapshot), CDX output, MangaBaka + series/links/images/snapshot/preview JSON. +- Pure functions under test: link parsing, Webtoons parsing, URL normalisation, + classification, dedupe, default selection, seed building, content-rating + resolution, language mapping, the Worker allowlist. +- I/O (`pipeline`, `mangabaka/client`, `history/wayback`, `hosting/worker`, + the Worker routes) with a fake transport. **No network in tests.** +- Manual smoke test: the page against Tower of God (MangaBaka `9999`, Webtoons + `title_no=95`), with "Rehost" and without submitting. + +## 7. Open questions and risks + +1. **Cloudflare egress.** Webtoons or Wayback may block or throttle Cloudflare + IPs. The userscript does not have this risk (it fetches from the user's IP). +2. **When MangaBaka copies images** (submission or approval) and whether it + accepts our R2 URLs or `swebtoon-phinf.pstatic.net` (`direct`). +3. **"Main cover" = `is_default: true`** is inferred from the API spec. +4. **Where users create a personal access token** in the MangaBaka UI. +5. **Whether the cover form accepts fields filled by a userscript.** +6. **Hosting details:** Cloudflare account, Worker domain, R2 public domain. + +## 8. Test data + +Tower of God historical `og:image`s: + +``` +2015 http://swebtoon.phinf.naver.net/20140710_28/1404979632969uKBA8_JPEG/01_EC8BA0EC9D98ED8391.jpg +2017 http://swebtoon.phinf.naver.net/20151006_141/1444125046528RTfRM_JPEG/_EB93BBEB80AF_EBA38CEABCA5_E29587EABCB1_EB9384EB84B0_ipa.jpg + (twitter:image …/20151006_243/1444125042491PuNza_JPEG/…_mob.jpg) +2019 https://swebtoon-phinf.pstatic.net/20190318_291/1552868599909GoVLY_JPEG/10_EC8DB8EB84A4EC9DBC_ipad.jpg + (twitter:image …/20190318_26/1552868584759vl9A1_JPEG/11_EC8DB8EB84A4EC9DBC_mobile.jpg) +2021 https://swebtoon-phinf.pstatic.net/20210127_106/1611705089085OYcUB_JPEG/1Tower-of-God-Mobile-Landing-Page.jpg +2023+ https://swebtoon-phinf.pstatic.net/20230324_114/1679612146968KhLLz_JPEG/6TowerOfGod_landingpage_mobile.jpg +live https://swebtoon-phinf.pstatic.net/20250204_46/17386458444547o1b2_JPEG/95.jpg (poster, 480×623) +``` + +The 2014, 2015 and 2019 images are still live on `swebtoon-phinf.pstatic.net`; +the 2021 one is gone from Wayback's own copy but live on pstatic. + +Other test inputs: + +- canvas: `https://www.webtoons.com/en/canvas/crow-time/list?title_no=693372` +- mature: title_no `6065` (filename contains spaces and `)`; URL-encode it) +- FR example: Violet Romance, `2Thumb_Poster_9782.jpg`. + +## 9. References + +- yambs (the model project, Go): `/home/user/yambs` on the maintainer's machine, with the same + sources → seed → actions shape. +- MangaBaka API docs: https://mangabaka.org/data/api; explorer: https://mangabaka.org/data/api/explorer; + support: https://mangabaka.org/discord. +- The maintainer is a regular MangaBaka contributor without API access (fewer than 25 approved + submissions as of 2026-10-14), so the manual path must be first-class. diff --git a/docs/specs/README.md b/docs/specs/README.md new file mode 100644 index 0000000..0a7a7d8 --- /dev/null +++ b/docs/specs/README.md @@ -0,0 +1,10 @@ +# Specs + +Approved specifications from the superpowers brainstorming workflow. Nothing non-trivial gets +implemented without one. + +- Naming: `NNN-short-slug.md`, zero-padded, sequential. +- A spec states **what** and **why**, plus acceptance criteria. Not **how**. +- Front matter: `status: draft | approved | implemented | superseded`. No link to a plan — plans are + removed before merge. +- Never delete an implemented spec; it is the record of intent.