130 lines
7.0 KiB
TypeScript
130 lines
7.0 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { cdxUrl, parseCdx, pickSnapshots, waybackHistory } from "../src/core/history/wayback";
|
|
import type { SourceSeries } from "../src/core/model";
|
|
import { webtoons } from "../src/core/sources/webtoons";
|
|
import { FakeTransport, text } from "./helpers/fake-transport";
|
|
import { fixtureText } from "./helpers/fixtures";
|
|
|
|
const SERIES: SourceSeries = {
|
|
source: "webtoons", externalId: "95", title: "Tower of God", language: "en",
|
|
canonicalUrl: "https://www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95",
|
|
archiveUrls: ["www.webtoons.com/en/fantasy/tower-of-god/list?title_no=95", "m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95"],
|
|
mature: false, flags: { canvas: false },
|
|
};
|
|
const W = "http://www.webtoons.com:80/en/fantasy/tower-of-god/list?title_no=95";
|
|
const M = "http://m.webtoons.com/en/fantasy/tower-of-god/list?title_no=95";
|
|
|
|
test("cdxUrl asks for one capture per month of the prefix", () => {
|
|
const u = new URL(cdxUrl(SERIES.archiveUrls[0]!));
|
|
expect(u.origin + u.pathname).toBe("https://web.archive.org/cdx/search/cdx");
|
|
expect(Object.fromEntries(u.searchParams)).toEqual({
|
|
url: SERIES.archiveUrls[0]!, matchType: "prefix", filter: "statuscode:200", collapse: "timestamp:6", fl: "timestamp,original",
|
|
});
|
|
});
|
|
|
|
test("parseCdx reads the captured index and skips junk lines", async () => {
|
|
const rows = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt"));
|
|
expect(rows.length).toBeGreaterThan(100);
|
|
expect(rows[0]!.timestamp).toMatch(/^2015\d{10}$/);
|
|
expect(parseCdx("garbage\n\n20150312154616 http://x/list?title_no=95\n")).toEqual([
|
|
{ timestamp: "20150312154616", original: "http://x/list?title_no=95" },
|
|
]);
|
|
});
|
|
|
|
describe("pickSnapshots", () => {
|
|
// Synthetic rows: they test the bucketing arithmetic, not a service response.
|
|
const rows = [
|
|
{ timestamp: "20150312000000", original: W },
|
|
{ timestamp: "20150401000000", original: M },
|
|
{ timestamp: "20150501000000", original: W },
|
|
{ timestamp: "20150901000000", original: M },
|
|
{ timestamp: "20151001000000", original: W },
|
|
{ timestamp: "20160301000000", original: W },
|
|
];
|
|
test("one capture per bucket of N months from the first capture, www. preferred", () => {
|
|
expect(pickSnapshots(rows, 6, "www.webtoons.com").map((r) => r.timestamp))
|
|
.toEqual(["20150312000000", "20151001000000", "20160301000000"]);
|
|
expect(pickSnapshots(rows, 12, "www.webtoons.com").map((r) => r.timestamp))
|
|
.toEqual(["20150312000000", "20160301000000"]);
|
|
expect(pickSnapshots(rows, 1, "www.webtoons.com")).toHaveLength(6);
|
|
});
|
|
test("falls back to m. when a bucket has no www. capture", () => {
|
|
expect(pickSnapshots([rows[1]!, rows[3]!], 3, "www.webtoons.com").map((r) => r.original)).toEqual([M, M]);
|
|
});
|
|
test("real index, every 6 months: one per half-year bucket, in order", async () => {
|
|
const real = parseCdx(await fixtureText("wayback/tower-of-god-cdx-www.txt"));
|
|
const picks = pickSnapshots(real, 6, "www.webtoons.com");
|
|
const months = new Set(real.map((r) => r.timestamp.slice(0, 6))).size;
|
|
expect(picks.length).toBeLessThanOrEqual(Math.ceil(months / 6) + 12);
|
|
expect(picks.length).toBeGreaterThanOrEqual(15);
|
|
expect([...picks].sort((a, b) => a.timestamp.localeCompare(b.timestamp))).toEqual(picks);
|
|
});
|
|
});
|
|
|
|
function waybackFake(cdxWww: string, snapshot: (url: string) => Response) {
|
|
return new FakeTransport()
|
|
.on((u) => u.startsWith("https://web.archive.org/cdx/") && u.includes("www.webtoons.com"), () => text(cdxWww))
|
|
.on((u) => u.startsWith("https://web.archive.org/cdx/"), () => text(""))
|
|
.on((u) => u.includes("id_/"), (u) => snapshot(u));
|
|
}
|
|
|
|
describe("waybackHistory", () => {
|
|
test("drops other title_no rows before picking snapshots", async () => {
|
|
// Review Focus 1: matchType=prefix on title_no=95 also returns title_no=950.
|
|
const cdx = `20150312154616 ${W}\n20160101000000 http://www.webtoons.com/en/fantasy/x/list?title_no=950\n`;
|
|
const page2015 = await fixtureText("wayback/tower-of-god-2015.html");
|
|
const fake = waybackFake(cdx, () => text(page2015));
|
|
await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {} });
|
|
const snapshots = fake.urls().filter((u) => u.includes("id_/"));
|
|
expect(snapshots).toEqual([`https://web.archive.org/web/20150312154616id_/${W}`]);
|
|
});
|
|
|
|
test("turns snapshots into merged historical candidates with archive fallbacks", async () => {
|
|
const cdx = `20150312154616 ${W}\n20150901000000 ${W}\n20230401000000 ${W}\n`;
|
|
const [p2015, p2023] = [await fixtureText("wayback/tower-of-god-2015.html"), await fixtureText("wayback/tower-of-god-2023.html")];
|
|
const fake = waybackFake(cdx, (u) => text(u.includes("/2023") ? p2023 : p2015));
|
|
const progress: string[] = [];
|
|
const found = await waybackHistory(webtoons, SERIES, {
|
|
every: 1, transport: fake, warn: () => {}, onProgress: (d, t) => progress.push(`${d}/${t}`),
|
|
});
|
|
expect(progress).toEqual(["1/3", "2/3", "3/3"]);
|
|
const poster2015 = found.find((c) => c.url.endsWith("01_EC8BA0EC9D98ED8391.jpg"))!;
|
|
expect(poster2015).toMatchObject({
|
|
role: "poster", current: false, seenAt: "20150312154616", lastSeenAt: "20150901000000",
|
|
archivedUrl: "https://web.archive.org/web/20150312154616id_/http://swebtoon.phinf.naver.net/20140710_28/1404979632969uKBA8_JPEG/01_EC8BA0EC9D98ED8391.jpg",
|
|
});
|
|
expect(found.some((c) => c.seenAt?.startsWith("2023"))).toBe(true);
|
|
});
|
|
|
|
test("a refused snapshot stops the history and keeps what was found", async () => {
|
|
const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n20170301000000 ${W}\n`;
|
|
const page2015 = await fixtureText("wayback/tower-of-god-2015.html");
|
|
const fake = waybackFake(cdx, (u) => (u.includes("/2015") ? text(page2015) : text("blocked", 429)));
|
|
const warnings: string[] = [];
|
|
const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: (m) => warnings.push(m) });
|
|
expect(found.length).toBeGreaterThan(0);
|
|
expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(2);
|
|
expect(warnings).toEqual([expect.stringContaining("history stops at 2016-03")]);
|
|
});
|
|
|
|
test("an unavailable index is a warning, not an error", async () => {
|
|
const fake = new FakeTransport().on(() => true, () => text("busy", 503));
|
|
const warnings: string[] = [];
|
|
expect(await waybackHistory(webtoons, SERIES, { every: 6, transport: fake, warn: (m) => warnings.push(m) })).toEqual([]);
|
|
expect(warnings).toHaveLength(2);
|
|
});
|
|
|
|
test("cancelling keeps what was found so far", async () => {
|
|
const cdx = `20150312154616 ${W}\n20160301000000 ${W}\n`;
|
|
const page2015 = await fixtureText("wayback/tower-of-god-2015.html");
|
|
const controller = new AbortController();
|
|
const fake = waybackFake(cdx, () => {
|
|
controller.abort();
|
|
return text(page2015);
|
|
});
|
|
const found = await waybackHistory(webtoons, SERIES, { every: 1, transport: fake, warn: () => {}, signal: controller.signal });
|
|
expect(found.length).toBeGreaterThan(0);
|
|
expect(fake.urls().filter((u) => u.includes("id_/"))).toHaveLength(1);
|
|
});
|
|
});
|