diff --git a/src/core/history/wayback.ts b/src/core/history/wayback.ts new file mode 100644 index 0000000..45b5843 --- /dev/null +++ b/src/core/history/wayback.ts @@ -0,0 +1,119 @@ +import { formatMonth, mergeByUrl } from "../covers"; +import type { CoverCandidate, Granularity, SourceSeries } from "../model"; +import type { Source } from "../sources/source"; +import type { Transport } from "../transport"; + +export const WAYBACK = "https://web.archive.org"; + +export interface CdxRow { + timestamp: string; + original: string; +} + +export interface HistoryOptions { + every: Granularity; + transport: Transport; + warn(message: string): void; + onProgress?(done: number, total: number): void; + signal?: AbortSignal; +} + +/** Every month with a capture (collapse=timestamp:6) — one cheap request per prefix (spec §4.3). */ +export function cdxUrl(prefix: string): string { + const q = new URLSearchParams({ + url: prefix, matchType: "prefix", filter: "statuscode:200", collapse: "timestamp:6", fl: "timestamp,original", + }); + return `${WAYBACK}/cdx/search/cdx?${q}`; +} + +export function parseCdx(text: string): CdxRow[] { + const rows: CdxRow[] = []; + for (const line of text.split("\n")) { + const [timestamp, original] = line.trim().split(/\s+/); + if (timestamp && original && /^\d{14}$/.test(timestamp)) rows.push({ timestamp, original }); + } + return rows; +} + +const monthIndex = (ts: string) => Number(ts.slice(0, 4)) * 12 + Number(ts.slice(4, 6)) - 1; +const hostIs = (row: CdxRow, host: string) => { + try { + return new URL(row.original).hostname === host; + } catch { + return false; + } +}; + +/** Buckets of `every` months from the first capture; first capture per bucket, preferring `preferHost`. */ +export function pickSnapshots(rows: readonly CdxRow[], every: Granularity, preferHost: string): CdxRow[] { + const sorted = [...rows].sort((a, b) => a.timestamp.localeCompare(b.timestamp)); + if (!sorted.length) return []; + const first = monthIndex(sorted[0]!.timestamp); + const buckets = new Map(); + for (const row of sorted) { + const bucket = Math.floor((monthIndex(row.timestamp) - first) / every); + const current = buckets.get(bucket); + if (!current || (!hostIs(current, preferHost) && hostIs(row, preferHost))) buckets.set(bucket, row); + } + return [...buckets.values()].sort((a, b) => a.timestamp.localeCompare(b.timestamp)); +} + +export const snapshotUrl = (row: CdxRow): string => `${WAYBACK}/web/${row.timestamp}id_/${row.original}`; +export const archivedImageUrl = (timestamp: string, originalImageUrl: string): string => + `${WAYBACK}/web/${timestamp}id_/${originalImageUrl}`; + +const message = (err: unknown) => (err instanceof Error ? err.message : String(err)); + +export async function waybackHistory(source: Source, series: SourceSeries, opts: HistoryOptions): Promise { + if (!source.extractCovers) return []; + const rows: CdxRow[] = []; + for (const prefix of series.archiveUrls) { + try { + const res = await opts.transport.fetch(cdxUrl(prefix), { signal: opts.signal }); + if (!res.ok) { + await res.body?.cancel(); + opts.warn(`The Wayback index didn't answer (HTTP ${res.status}); no history from ${prefix}.`); + continue; + } + // matchType=prefix on title_no=95 also matches title_no=950 (spec §4.3). + rows.push(...parseCdx(await res.text()).filter((r) => source.isSameSeries(r.original, series))); + } catch (err) { + if (opts.signal?.aborted) return []; + opts.warn(`The Wayback index didn't answer (${message(err)}); no history from ${prefix}.`); + } + } + + const preferHost = series.archiveUrls[0]?.split("/")[0] ?? ""; + const picks = pickSnapshots(rows, opts.every, preferHost); + const found: CoverCandidate[] = []; + let done = 0; + for (const row of picks) { + if (opts.signal?.aborted) break; + let res: Response; + try { + res = await opts.transport.fetch(snapshotUrl(row), { signal: opts.signal }); + } catch (err) { + if (opts.signal?.aborted) break; + opts.warn(`Wayback stopped answering (${message(err)}); history stops at ${formatMonth(row.timestamp)}.`); + break; + } + if (res.status === 429 || res.status >= 500) { + await res.body?.cancel(); + opts.warn(`Wayback is refusing requests (HTTP ${res.status}); history stops at ${formatMonth(row.timestamp)}.`); + break; + } + if (res.ok) { + const pageUrl = new URL(row.original); + for (const c of source.extractCovers(await res.text(), pageUrl)) { + const candidate: CoverCandidate = { ...c, current: false, seenAt: row.timestamp, lastSeenAt: row.timestamp }; + if (c.originalUrl) candidate.archivedUrl = archivedImageUrl(row.timestamp, c.originalUrl); + found.push(candidate); + } + } else { + await res.body?.cancel(); + opts.warn(`Snapshot ${formatMonth(row.timestamp)} is unavailable (HTTP ${res.status}).`); + } + opts.onProgress?.(++done, picks.length); + } + return mergeByUrl(found); +} diff --git a/test/wayback.test.ts b/test/wayback.test.ts new file mode 100644 index 0000000..3da7556 --- /dev/null +++ b/test/wayback.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, test } from "bun:test"; +import { cdxUrl, parseCdx, pickSnapshots, waybackHistory } from "../src/core/history/wayback"; +import type { SourceSeries } from "../src/core/model"; +import { webtoons } from "../src/core/sources/webtoons"; +import { FakeTransport, text } from "./helpers/fake-transport"; +import { fixtureText } from "./helpers/fixtures"; + +const SERIES: SourceSeries = { + source: "webtoons", externalId: "95", title: "Tower of God", language: "en", + canonicalUrl: "https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95", + archiveUrls: ["www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95", "m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95"], + mature: false, flags: { canvas: false }, +}; +const W = "http://www.webtoons.com:80/en/fantasy/tower-of-god/list?title_no=95"; +const M = "http://m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95"; + +test("cdxUrl asks for one capture per month of the prefix", () => { + const u = new URL(cdxUrl(SERIES.archiveUrls[0]!)); + expect(u.origin + u.pathname).toBe("https://web.archive.org/cdx/search/cdx"); + expect(Object.fromEntries(u.searchParams)).toEqual({ + url: SERIES.archiveUrls[0]!, matchType: "prefix", filter: "statuscode:200", collapse: "timestamp:6", fl: "timestamp,original", + }); +}); + +test("parseCdx reads the captured index and skips junk lines", async () => { + const rows = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt")); + expect(rows.length).toBeGreaterThan(100); + expect(rows[0]!.timestamp).toMatch(/^2015\d{10}$/); + expect(parseCdx("garbage\n\n20150312154616 http://x/list?title_no=95\n")).toEqual([ + { timestamp: "20150312154616", original: "http://x/list?title_no=95" }, + ]); +}); + +describe("pickSnapshots", () => { + // Synthetic rows: they test the bucketing arithmetic, not a service response. + const rows = [ + { timestamp: "20150312000000", original: W }, + { timestamp: "20150401000000", original: M }, + { timestamp: "20150501000000", original: W }, + { timestamp: "20150901000000", original: M }, + { timestamp: "20151001000000", original: W }, + { timestamp: "20160301000000", original: W }, + ]; + test("one capture per bucket of N months from the first capture, www. preferred", () => { + expect(pickSnapshots(rows, 6, "www.webtoons.com").map((r) => r.timestamp)) + .toEqual(["20150312000000", "20151001000000", "20160301000000"]); + expect(pickSnapshots(rows, 12, "www.webtoons.com").map((r) => r.timestamp)) + .toEqual(["20150312000000", "20160301000000"]); + expect(pickSnapshots(rows, 1, "www.webtoons.com")).toHaveLength(6); + }); + test("falls back to m. when a bucket has no www. capture", () => { + expect(pickSnapshots([rows[1]!, rows[3]!], 3, "www.webtoons.com").map((r) => r.original)).toEqual([M, M]); + }); + test("real index, every 6 months: one per half-year bucket, in order", async () => { + const real = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt")); + const picks = pickSnapshots(real, 6, "www.webtoons.com"); + const months = new Set(real.map((r) => r.timestamp.slice(0, 6))).size; + expect(picks.length).toBeLessThanOrEqual(Math.ceil(months / 6) + 12); + expect(picks.length).toBeGreaterThanOrEqual(15); + expect([...picks].sort((a, b) => a.timestamp.localeCompare(b.timestamp))).toEqual(picks); + }); +}); + +function waybackFake(cdxWww: string, snapshot: (url: string) => Response) { + return new FakeTransport() + .on((u) => u.startsWith("https://web.archive.org/cdx/") && u.includes("www.webtoons.com"), () => text(cdxWww)) + .on((u) => u.startsWith("https://web.archive.org/cdx/"), () => text("")) + .on((u) => u.includes("id_/"), (u) => snapshot(u)); +} + +describe("waybackHistory", () => { + test("drops other title_no rows before picking snapshots", async () => { + // Review Focus 1: matchType=prefix on title_no=95 also returns title_no=950. + const cdx = `20150312154616 ${W}\n20160101000000 http://www.webtoons.com/en/fantasy/x/list?title_no=950\n`; + const page2015 = await fixtureText("wayback/tower-of-god-2015.html"); + const fake = waybackFake(cdx, () => text(page2015)); + await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {} }); + const snapshots = fake.urls().filter((u) => u.includes("id_/")); + expect(snapshots).toEqual([`https://web.archive.org/web/20150312154616id_/${W}`]); + }); + + test("turns snapshots into merged historical candidates with archive fallbacks", async () => { + const cdx = `20150312154616 ${W}\n20150901000000 ${W}\n20230401000000 ${W}\n`; + const [p2015, p2023] = [await fixtureText("wayback/tower-of-god-2015.html"), await fixtureText("wayback/tower-of-god-2023.html")]; + const fake = waybackFake(cdx, (u) => text(u.includes("/2023") ? p2023 : p2015)); + const progress: string[] = []; + const found = await waybackHistory(webtoons, SERIES, { + every: 1, transport: fake, warn: () => {}, onProgress: (d, t) => progress.push(`${d}/${t}`), + }); + expect(progress).toEqual(["1/3", "2/3", "3/3"]); + const poster2015 = found.find((c) => c.url.endsWith("01_EC8BA0EC9D98ED8391.jpg"))!; + expect(poster2015).toMatchObject({ + role: "poster", current: false, seenAt: "20150312154616", lastSeenAt: "20150901000000", + archivedUrl: "https://web.archive.org/web/20150312154616id_/http://swebtoon.phinf.naver.net/20140710_28/1404979632969uKBA8_JPEG/01_EC8BA0EC9D98ED8391.jpg", + }); + expect(found.some((c) => c.seenAt?.startsWith("2023"))).toBe(true); + }); + + test("a refused snapshot stops the history and keeps what was found", async () => { + const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n20170301000000 ${W}\n`; + const page2015 = await fixtureText("wayback/tower-of-god-2015.html"); + const fake = waybackFake(cdx, (u) => (u.includes("/2015") ? text(page2015) : text("blocked", 429))); + const warnings: string[] = []; + const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: (m) => warnings.push(m) }); + expect(found.length).toBeGreaterThan(0); + expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(2); + expect(warnings).toEqual([expect.stringContaining("history stops at 2016-03")]); + }); + + test("an unavailable index is a warning, not an error", async () => { + const fake = new FakeTransport().on(() => true, () => text("busy", 503)); + const warnings: string[] = []; + expect(await waybackHistory(webtoons, SERIES, { every: 6, transport: fake, warn: (m) => warnings.push(m) })).toEqual([]); + expect(warnings).toHaveLength(2); + }); + + test("cancelling keeps what was found so far", async () => { + const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n`; + const page2015 = await fixtureText("wayback/tower-of-god-2015.html"); + const controller = new AbortController(); + const fake = waybackFake(cdx, () => { + controller.abort(); + return text(page2015); + }); + const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {}, signal: controller.signal }); + expect(found.length).toBeGreaterThan(0); + expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(1); + }); +});