import { describe, expect, test } from "bun:test"; import { cdxUrl, parseCdx, pickSnapshots, waybackHistory } from "../src/core/history/wayback"; import type { SourceSeries } from "../src/core/model"; import { webtoons } from "../src/core/sources/webtoons"; import { FakeTransport, text } from "./helpers/fake-transport"; import { fixtureText } from "./helpers/fixtures"; const SERIES: SourceSeries = { source: "webtoons", externalId: "95", title: "Tower of God", language: "en", canonicalUrl: "https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95", archiveUrls: ["www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95", "m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95"], mature: false, flags: { canvas: false }, }; const W = "http://www.webtoons.com:80/en/fantasy/tower-of-god/list?title_no=95"; const M = "http://m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95"; test("cdxUrl asks for one capture per month of the prefix", () => { const u = new URL(cdxUrl(SERIES.archiveUrls[0]!)); expect(u.origin + u.pathname).toBe("https://web.archive.org/cdx/search/cdx"); expect(Object.fromEntries(u.searchParams)).toEqual({ url: SERIES.archiveUrls[0]!, matchType: "prefix", filter: "statuscode:200", collapse: "timestamp:6", fl: "timestamp,original", }); }); test("parseCdx reads the captured index and skips junk lines", async () => { const rows = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt")); expect(rows.length).toBeGreaterThan(100); expect(rows[0]!.timestamp).toMatch(/^2015\d{10}$/); expect(parseCdx("garbage\n\n20150312154616 http://x/list?title_no=95\n")).toEqual([ { timestamp: "20150312154616", original: "http://x/list?title_no=95" }, ]); }); describe("pickSnapshots", () => { // Synthetic rows: they test the bucketing arithmetic, not a service response. const rows = [ { timestamp: "20150312000000", original: W }, { timestamp: "20150401000000", original: M }, { timestamp: "20150501000000", original: W }, { timestamp: "20150901000000", original: M }, { timestamp: "20151001000000", original: W }, { timestamp: "20160301000000", original: W }, ]; test("one capture per bucket of N months from the first capture, www. preferred", () => { expect(pickSnapshots(rows, 6, "www.webtoons.com").map((r) => r.timestamp)) .toEqual(["20150312000000", "20151001000000", "20160301000000"]); expect(pickSnapshots(rows, 12, "www.webtoons.com").map((r) => r.timestamp)) .toEqual(["20150312000000", "20160301000000"]); expect(pickSnapshots(rows, 1, "www.webtoons.com")).toHaveLength(6); }); test("falls back to m. when a bucket has no www. capture", () => { expect(pickSnapshots([rows[1]!, rows[3]!], 3, "www.webtoons.com").map((r) => r.original)).toEqual([M, M]); }); test("real index, every 6 months: one per half-year bucket, in order", async () => { const real = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt")); const picks = pickSnapshots(real, 6, "www.webtoons.com"); const months = new Set(real.map((r) => r.timestamp.slice(0, 6))).size; expect(picks.length).toBeLessThanOrEqual(Math.ceil(months / 6) + 12); expect(picks.length).toBeGreaterThanOrEqual(15); expect([...picks].sort((a, b) => a.timestamp.localeCompare(b.timestamp))).toEqual(picks); }); }); function waybackFake(cdxWww: string, snapshot: (url: string) => Response) { return new FakeTransport() .on((u) => u.startsWith("https://web.archive.org/cdx/") && u.includes("www.webtoons.com"), () => text(cdxWww)) .on((u) => u.startsWith("https://web.archive.org/cdx/"), () => text("")) .on((u) => u.includes("id_/"), (u) => snapshot(u)); } describe("waybackHistory", () => { test("drops other title_no rows before picking snapshots", async () => { // Review Focus 1: matchType=prefix on title_no=95 also returns title_no=950. const cdx = `20150312154616 ${W}\n20160101000000 http://www.webtoons.com/en/fantasy/x/list?title_no=950\n`; const page2015 = await fixtureText("wayback/tower-of-god-2015.html"); const fake = waybackFake(cdx, () => text(page2015)); await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {} }); const snapshots = fake.urls().filter((u) => u.includes("id_/")); expect(snapshots).toEqual([`https://web.archive.org/web/20150312154616id_/${W}`]); }); test("turns snapshots into merged historical candidates with archive fallbacks", async () => { const cdx = `20150312154616 ${W}\n20150901000000 ${W}\n20230401000000 ${W}\n`; const [p2015, p2023] = [await fixtureText("wayback/tower-of-god-2015.html"), await fixtureText("wayback/tower-of-god-2023.html")]; const fake = waybackFake(cdx, (u) => text(u.includes("/2023") ? p2023 : p2015)); const progress: string[] = []; const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {}, onProgress: (d, t) => progress.push(`${d}/${t}`), }); expect(progress).toEqual(["1/3", "2/3", "3/3"]); const poster2015 = found.find((c) => c.url.endsWith("01_EC8BA0EC9D98ED8391.jpg"))!; expect(poster2015).toMatchObject({ role: "poster", current: false, seenAt: "20150312154616", lastSeenAt: "20150901000000", archivedUrl: "https://web.archive.org/web/20150312154616id_/http://swebtoon.phinf.naver.net/20140710_28/1404979632969uKBA8_JPEG/01_EC8BA0EC9D98ED8391.jpg", }); expect(found.some((c) => c.seenAt?.startsWith("2023"))).toBe(true); }); test("a refused snapshot stops the history and keeps what was found", async () => { const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n20170301000000 ${W}\n`; const page2015 = await fixtureText("wayback/tower-of-god-2015.html"); const fake = waybackFake(cdx, (u) => (u.includes("/2015") ? text(page2015) : text("blocked", 429))); const warnings: string[] = []; const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: (m) => warnings.push(m) }); expect(found.length).toBeGreaterThan(0); expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(2); expect(warnings).toEqual([expect.stringContaining("history stops at 2016-03")]); }); test("an unavailable index is a warning, not an error", async () => { const fake = new FakeTransport().on(() => true, () => text("busy", 503)); const warnings: string[] = []; expect(await waybackHistory(webtoons, SERIES, { every: 6, transport: fake, warn: (m) => warnings.push(m) })).toEqual([]); expect(warnings).toHaveLength(2); }); test("cancelling keeps what was found so far", async () => { const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n`; const page2015 = await fixtureText("wayback/tower-of-god-2015.html"); const controller = new AbortController(); const fake = waybackFake(cdx, () => { controller.abort(); return text(page2015); }); const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {}, signal: controller.signal }); expect(found.length).toBeGreaterThan(0); expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(1); }); });