feat(core): Wayback history with 1/3/6/12-month granularity; stop on refusal

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 23:37:59 +02:00
1 parent 8f2109471c
commit 7247e096f7
@@ -0,0 +1,119 @@
+import { formatMonth, mergeByUrl } from "../covers";
+import type { CoverCandidate, Granularity, SourceSeries } from "../model";
+import type { Source } from "../sources/source";
+import type { Transport } from "../transport";
+
+export const WAYBACK = "https://web.archive.org";
+
+export interface CdxRow {
+  timestamp: string;
+  original: string;
+}
+
+export interface HistoryOptions {
+  every: Granularity;
+  transport: Transport;
+  warn(message: string): void;
+  onProgress?(done: number, total: number): void;
+  signal?: AbortSignal;
+}
+
+/** Every month with a capture (collapse=timestamp:6) — one cheap request per prefix (spec §4.3). */
+export function cdxUrl(prefix: string): string {
+  const q = new URLSearchParams({
+    url: prefix, matchType: "prefix", filter: "statuscode:200", collapse: "timestamp:6", fl: "timestamp,original",
+  });
+  return `${WAYBACK}/cdx/search/cdx?${q}`;
+}
+
+export function parseCdx(text: string): CdxRow[] {
+  const rows: CdxRow[] = [];
+  for (const line of text.split("\n")) {
+    const [timestamp, original] = line.trim().split(/\s+/);
+    if (timestamp && original && /^\d{14}$/.test(timestamp)) rows.push({ timestamp, original });
+  }
+  return rows;
+}
+
+const monthIndex = (ts: string) => Number(ts.slice(0, 4)) * 12 + Number(ts.slice(4, 6)) - 1;
+const hostIs = (row: CdxRow, host: string) => {
+  try {
+    return new URL(row.original).hostname === host;
+  } catch {
+    return false;
+  }
+};
+
+/** Buckets of `every` months from the first capture; first capture per bucket, preferring `preferHost`. */
+export function pickSnapshots(rows: readonly CdxRow[], every: Granularity, preferHost: string): CdxRow[] {
+  const sorted = [...rows].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+  if (!sorted.length) return [];
+  const first = monthIndex(sorted[0]!.timestamp);
+  const buckets = new Map<number, CdxRow>();
+  for (const row of sorted) {
+    const bucket = Math.floor((monthIndex(row.timestamp) - first) / every);
+    const current = buckets.get(bucket);
+    if (!current || (!hostIs(current, preferHost) && hostIs(row, preferHost))) buckets.set(bucket, row);
+  }
+  return [...buckets.values()].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+}
+
+export const snapshotUrl = (row: CdxRow): string => `${WAYBACK}/web/${row.timestamp}id_/${row.original}`;
+export const archivedImageUrl = (timestamp: string, originalImageUrl: string): string =>
+  `${WAYBACK}/web/${timestamp}id_/${originalImageUrl}`;
+
+const message = (err: unknown) => (err instanceof Error ? err.message : String(err));
+
+export async function waybackHistory(source: Source, series: SourceSeries, opts: HistoryOptions): Promise<CoverCandidate[]> {
+  if (!source.extractCovers) return [];
+  const rows: CdxRow[] = [];
+  for (const prefix of series.archiveUrls) {
+    try {
+      const res = await opts.transport.fetch(cdxUrl(prefix), { signal: opts.signal });
+      if (!res.ok) {
+        await res.body?.cancel();
+        opts.warn(`The Wayback index didn't answer (HTTP ${res.status}); no history from ${prefix}.`);
+        continue;
+      }
+      // matchType=prefix on title_no=95 also matches title_no=950 (spec §4.3).
+      rows.push(...parseCdx(await res.text()).filter((r) => source.isSameSeries(r.original, series)));
+    } catch (err) {
+      if (opts.signal?.aborted) return [];
+      opts.warn(`The Wayback index didn't answer (${message(err)}); no history from ${prefix}.`);
+    }
+  }
+
+  const preferHost = series.archiveUrls[0]?.split("/")[0] ?? "";
+  const picks = pickSnapshots(rows, opts.every, preferHost);
+  const found: CoverCandidate[] = [];
+  let done = 0;
+  for (const row of picks) {
+    if (opts.signal?.aborted) break;
+    let res: Response;
+    try {
+      res = await opts.transport.fetch(snapshotUrl(row), { signal: opts.signal });
+    } catch (err) {
+      if (opts.signal?.aborted) break;
+      opts.warn(`Wayback stopped answering (${message(err)}); history stops at ${formatMonth(row.timestamp)}.`);
+      break;
+    }
+    if (res.status === 429 || res.status >= 500) {
+      await res.body?.cancel();
+      opts.warn(`Wayback is refusing requests (HTTP ${res.status}); history stops at ${formatMonth(row.timestamp)}.`);
+      break;
+    }
+    if (res.ok) {
+      const pageUrl = new URL(row.original);
+      for (const c of source.extractCovers(await res.text(), pageUrl)) {
+        const candidate: CoverCandidate = { ...c, current: false, seenAt: row.timestamp, lastSeenAt: row.timestamp };
+        if (c.originalUrl) candidate.archivedUrl = archivedImageUrl(row.timestamp, c.originalUrl);
+        found.push(candidate);
+      }
+    } else {
+      await res.body?.cancel();
+      opts.warn(`Snapshot ${formatMonth(row.timestamp)} is unavailable (HTTP ${res.status}).`);
+    }
+    opts.onProgress?.(++done, picks.length);
+  }
+  return mergeByUrl(found);
+}
@@ -0,0 +1,129 @@
+import { describe, expect, test } from "bun:test";
+import { cdxUrl, parseCdx, pickSnapshots, waybackHistory } from "../src/core/history/wayback";
+import type { SourceSeries } from "../src/core/model";
+import { webtoons } from "../src/core/sources/webtoons";
+import { FakeTransport, text } from "./helpers/fake-transport";
+import { fixtureText } from "./helpers/fixtures";
+
+const SERIES: SourceSeries = {
+  source: "webtoons", externalId: "95", title: "Tower of God", language: "en",
+  canonicalUrl: "https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95",
+  archiveUrls: ["www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95", "m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95"],
+  mature: false, flags: { canvas: false },
+};
+const W = "http://www.webtoons.com:80/en/fantasy/tower-of-god/list?title_no=95";
+const M = "http://m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95";
+
+test("cdxUrl asks for one capture per month of the prefix", () => {
+  const u = new URL(cdxUrl(SERIES.archiveUrls[0]!));
+  expect(u.origin + u.pathname).toBe("https://web.archive.org/cdx/search/cdx");
+  expect(Object.fromEntries(u.searchParams)).toEqual({
+    url: SERIES.archiveUrls[0]!, matchType: "prefix", filter: "statuscode:200", collapse: "timestamp:6", fl: "timestamp,original",
+  });
+});
+
+test("parseCdx reads the captured index and skips junk lines", async () => {
+  const rows = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt"));
+  expect(rows.length).toBeGreaterThan(100);
+  expect(rows[0]!.timestamp).toMatch(/^2015\d{10}$/);
+  expect(parseCdx("garbage\n\n20150312154616 http://x/list?title_no=95\n")).toEqual([
+    { timestamp: "20150312154616", original: "http://x/list?title_no=95" },
+  ]);
+});
+
+describe("pickSnapshots", () => {
+  // Synthetic rows: they test the bucketing arithmetic, not a service response.
+  const rows = [
+    { timestamp: "20150312000000", original: W },
+    { timestamp: "20150401000000", original: M },
+    { timestamp: "20150501000000", original: W },
+    { timestamp: "20150901000000", original: M },
+    { timestamp: "20151001000000", original: W },
+    { timestamp: "20160301000000", original: W },
+  ];
+  test("one capture per bucket of N months from the first capture, www. preferred", () => {
+    expect(pickSnapshots(rows, 6, "www.webtoons.com").map((r) => r.timestamp))
+      .toEqual(["20150312000000", "20151001000000", "20160301000000"]);
+    expect(pickSnapshots(rows, 12, "www.webtoons.com").map((r) => r.timestamp))
+      .toEqual(["20150312000000", "20160301000000"]);
+    expect(pickSnapshots(rows, 1, "www.webtoons.com")).toHaveLength(6);
+  });
+  test("falls back to m. when a bucket has no www. capture", () => {
+    expect(pickSnapshots([rows[1]!, rows[3]!], 3, "www.webtoons.com").map((r) => r.original)).toEqual([M, M]);
+  });
+  test("real index, every 6 months: one per half-year bucket, in order", async () => {
+    const real = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt"));
+    const picks = pickSnapshots(real, 6, "www.webtoons.com");
+    const months = new Set(real.map((r) => r.timestamp.slice(0, 6))).size;
+    expect(picks.length).toBeLessThanOrEqual(Math.ceil(months / 6) + 12);
+    expect(picks.length).toBeGreaterThanOrEqual(15);
+    expect([...picks].sort((a, b) => a.timestamp.localeCompare(b.timestamp))).toEqual(picks);
+  });
+});
+
+function waybackFake(cdxWww: string, snapshot: (url: string) => Response) {
+  return new FakeTransport()
+    .on((u) => u.startsWith("https://web.archive.org/cdx/") && u.includes("www.webtoons.com"), () => text(cdxWww))
+    .on((u) => u.startsWith("https://web.archive.org/cdx/"), () => text(""))
+    .on((u) => u.includes("id_/"), (u) => snapshot(u));
+}
+
+describe("waybackHistory", () => {
+  test("drops other title_no rows before picking snapshots", async () => {
+    // Review Focus 1: matchType=prefix on title_no=95 also returns title_no=950.
+    const cdx = `20150312154616 ${W}\n20160101000000 http://www.webtoons.com/en/fantasy/x/list?title_no=950\n`;
+    const page2015 = await fixtureText("wayback/tower-of-god-2015.html");
+    const fake = waybackFake(cdx, () => text(page2015));
+    await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {} });
+    const snapshots = fake.urls().filter((u) => u.includes("id_/"));
+    expect(snapshots).toEqual([`https://web.archive.org/web/20150312154616id_/${W}`]);
+  });
+
+  test("turns snapshots into merged historical candidates with archive fallbacks", async () => {
+    const cdx = `20150312154616 ${W}\n20150901000000 ${W}\n20230401000000 ${W}\n`;
+    const [p2015, p2023] = [await fixtureText("wayback/tower-of-god-2015.html"), await fixtureText("wayback/tower-of-god-2023.html")];
+    const fake = waybackFake(cdx, (u) => text(u.includes("/2023") ? p2023 : p2015));
+    const progress: string[] = [];
+    const found = await waybackHistory(webtoons, SERIES, {
+      every: 1, transport: fake, warn: () => {}, onProgress: (d, t) => progress.push(`${d}/${t}`),
+    });
+    expect(progress).toEqual(["1/3", "2/3", "3/3"]);
+    const poster2015 = found.find((c) => c.url.endsWith("01_EC8BA0EC9D98ED8391.jpg"))!;
+    expect(poster2015).toMatchObject({
+      role: "poster", current: false, seenAt: "20150312154616", lastSeenAt: "20150901000000",
+      archivedUrl: "https://web.archive.org/web/20150312154616id_/http://swebtoon.phinf.naver.net/20140710_28/1404979632969uKBA8_JPEG/01_EC8BA0EC9D98ED8391.jpg",
+    });
+    expect(found.some((c) => c.seenAt?.startsWith("2023"))).toBe(true);
+  });
+
+  test("a refused snapshot stops the history and keeps what was found", async () => {
+    const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n20170301000000 ${W}\n`;
+    const page2015 = await fixtureText("wayback/tower-of-god-2015.html");
+    const fake = waybackFake(cdx, (u) => (u.includes("/2015") ? text(page2015) : text("blocked", 429)));
+    const warnings: string[] = [];
+    const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: (m) => warnings.push(m) });
+    expect(found.length).toBeGreaterThan(0);
+    expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(2);
+    expect(warnings).toEqual([expect.stringContaining("history stops at 2016-03")]);
+  });
+
+  test("an unavailable index is a warning, not an error", async () => {
+    const fake = new FakeTransport().on(() => true, () => text("busy", 503));
+    const warnings: string[] = [];
+    expect(await waybackHistory(webtoons, SERIES, { every: 6, transport: fake, warn: (m) => warnings.push(m) })).toEqual([]);
+    expect(warnings).toHaveLength(2);
+  });
+
+  test("cancelling keeps what was found so far", async () => {
+    const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n`;
+    const page2015 = await fixtureText("wayback/tower-of-god-2015.html");
+    const controller = new AbortController();
+    const fake = waybackFake(cdx, () => {
+      controller.abort();
+      return text(page2015);
+    });
+    const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {}, signal: controller.signal });
+    expect(found.length).toBeGreaterThan(0);
+    expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(1);
+  });
+});