import { afterEach, beforeEach, expect, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { refreshArticles } from "../scripts/refresh-articles.ts"; import { runRefresh, type RefreshOptions, type RobotsGate } from "../scripts/refresh-sources.ts"; import type { Adapter, ParseContext } from "../src/ingest/adapters/types.ts"; import { articleDue, articleFileName, ArticleStore, type ArticleMeta } from "../src/ingest/articleStore.ts"; import { SnapshotStore } from "../src/ingest/snapshots.ts"; import type { GachaEvent } from "../src/shared/schema.ts"; const NOW = new Date("2026-11-07T12:00:00.000Z"); const HOUR = 60 * 60 * 1000; const ago = (ms: number) => new Date(NOW.getTime() - ms).toISOString(); const TITLE = "Hour_of_Ruin_(Event)"; const WITH_TIME = await Bun.file("fixtures/genshin/articles/hour-of-ruin-2026-11-07.html").text(); const NO_TIME = JSON.stringify({ parse: { text: "

A stub.

" } }); const MISSING = JSON.stringify({ error: { code: "missingtitle", info: "The page does not exist." } }); let root: string; let articles: ArticleStore; beforeEach(async () => { root = await mkdtemp(join(tmpdir(), "event-clock-articles-")); articles = new ArticleStore(root); }); afterEach(async () => { await rm(root, { recursive: true, force: true }); }); const ALLOW_ALL: RobotsGate = { allows: async () => ({ allowed: true, reason: "ok" }) }; interface Call { url: string; userAgent: string | undefined } function run( respond: () => Response | Promise, over: Partial[0]> = {}, ) { const calls: Call[] = []; const naps: number[] = []; const logs: string[] = []; const done = refreshArticles({ titles: [TITLE], store: articles, robots: ALLOW_ALL, fetchImpl: async (url, init) => { calls.push({ url, userAgent: (init?.headers as Record)["User-Agent"] }); return respond(); }, userAgent: "test-agent", now: () => NOW, sleep: async (ms) => void naps.push(ms), dryRun: false, timeoutMs: 1000, log: (l) => void logs.push(l), requestedHosts: new Set(), ...over, }); return { done, calls, naps, logs }; } const meta = (over: Partial): ArticleMeta => ({ title: TITLE, lastAttemptAt: ago(30 * 24 * HOUR), lastReadAt: ago(30 * 24 * HOUR), statesTime: true, ...over, }); const at = NOW.getTime(); test("due: never read", () => { expect(articleDue(null, at)).toBe(true); }); test("due: no time found, only after 6 hours", () => { const m = (h: number) => meta({ lastAttemptAt: ago(h * HOUR), lastReadAt: ago(h * HOUR), statesTime: false }); expect(articleDue(m(5.9), at)).toBe(false); expect(articleDue(m(6), at)).toBe(true); }); test("due: time found, only 7 days after the last read that worked", () => { const m = (d: number) => meta({ lastAttemptAt: ago(d * 24 * HOUR), lastReadAt: ago(d * 24 * HOUR) }); expect(articleDue(m(6.9), at)).toBe(false); expect(articleDue(m(7), at)).toBe(true); }); test("due: a failed attempt inside 6 hours is not due, even past 7 days", () => { expect(articleDue(meta({ lastAttemptAt: ago(1 * HOUR) }), at)).toBe(false); expect(articleDue(meta({ lastAttemptAt: ago(6 * HOUR) }), at)).toBe(true); }); test("a percent-decoded title gets one stable file name that keeps titles apart", () => { const name = articleFileName("The_Forge_Realm's_Temper/2026-11-04"); expect(name).toMatch(/^[A-Za-z0-9._-]+$/); expect(name).toBe(articleFileName("The_Forge_Realm's_Temper/2026-11-04")); expect(articleFileName("a/b")).not.toBe(articleFileName("a:b")); }); test("a first read stores the article and its end", async () => { const { done } = run(() => new Response(WITH_TIME)); expect((await done).changed).toBe(1); expect((await articles.readEnds()).get(TITLE)?.day).toBe("2026-12-15"); expect(await articles.readMeta(TITLE)).toEqual({ title: TITLE, lastAttemptAt: NOW.toISOString(), lastReadAt: NOW.toISOString(), statesTime: true, }); }); /** A stored article whose 7-day re-read is due now. */ async function dueArticle() { await articles.saveRead(TITLE, WITH_TIME, ago(8 * 24 * HOUR)); } async function expectKept(result: { changed: number; warnings: string[] }) { expect(result.changed).toBe(0); expect(result.warnings.length).toBe(1); expect(await articles.readBody(TITLE)).toBe(WITH_TIME); expect((await articles.readEnds()).has(TITLE)).toBe(true); const m = await articles.readMeta(TITLE); expect(m?.lastReadAt).toBe(ago(8 * 24 * HOUR)); expect(m?.lastAttemptAt).toBe(NOW.toISOString()); } test.each([500, 403, 202])("HTTP %i keeps the body and the time, and warns", async (status) => { await dueArticle(); await expectKept(await run(() => new Response("nope", { status })).done); }); test("a network error keeps the body and the time, and warns", async () => { await dueArticle(); await expectKept(await run(() => { throw new Error("reset"); }).done); }); test("a body that fails mid-read keeps the body and the time, and warns", async () => { await dueArticle(); await expectKept(await run(() => ({ status: 200, arrayBuffer: () => Promise.reject(new Error("truncated")), text: () => Promise.reject(new Error("truncated")), }) as unknown as Response).done); }); test("a 200 that is not the API's answer keeps the body and the time, and warns", async () => { await dueArticle(); await expectKept(await run(() => new Response("Just a moment…")).done); }); test("a 200 API error other than a missing page keeps the body and the time", async () => { await dueArticle(); await expectKept(await run(() => new Response(JSON.stringify({ error: { code: "ratelimited" } }))).done); }); test("robots refusal makes no request, keeps the body, and warns", async () => { await dueArticle(); const robots: RobotsGate = { allows: async () => ({ allowed: false, reason: "disallowed" }) }; const { done, calls } = run(() => new Response(WITH_TIME), { robots }); await expectKept(await done); expect(calls.length).toBe(0); }); test.each([ ["404", () => new Response("gone", { status: 404 })], ["a missingtitle API error", () => new Response(MISSING)], ])("%s removes the stored article and warns", async (_name, respond) => { await dueArticle(); const result = await run(respond).done; expect(result.changed).toBe(1); expect(result.warnings.length).toBe(1); expect(await articles.readBody(TITLE)).toBeNull(); expect((await articles.readEnds()).has(TITLE)).toBe(false); }); test("a removed article still honours the 6-hour floor", async () => { await dueArticle(); await run(() => new Response("gone", { status: 404 })).done; const { done, calls } = run(() => new Response(WITH_TIME)); await done; expect(calls.length).toBe(0); }); test("a 200 with no usable End overwrites the body, sets statesTime false, and logs without warning", async () => { await dueArticle(); const { done, logs } = run(() => new Response(NO_TIME)); const result = await done; expect(result.changed).toBe(1); expect(result.warnings).toEqual([]); expect(logs.join("\n")).toContain(TITLE); expect(await articles.readBody(TITLE)).toBe(NO_TIME); expect((await articles.readMeta(TITLE))?.statesTime).toBe(false); expect((await articles.readEnds()).has(TITLE)).toBe(false); }); test("an unchanged re-read moves lastReadAt but changes no bytes", async () => { await dueArticle(); const result = await run(() => new Response(WITH_TIME)).done; expect(result.changed).toBe(0); expect((await articles.readMeta(TITLE))?.lastReadAt).toBe(NOW.toISOString()); }); test("an article not yet due makes no request", async () => { await articles.saveRead(TITLE, WITH_TIME, ago(24 * HOUR)); const { done, calls } = run(() => new Response(WITH_TIME)); await done; expect(calls.length).toBe(0); }); test("the requested URL carries the decoded title, section=0, and the runner User-Agent", async () => { const title = "The_Forge_Realm's_Temper/2026-11-04"; const { done, calls } = run(() => new Response(WITH_TIME), { titles: [title] }); await done; const url = new URL(calls[0]!.url); expect(url.host).toBe("genshin-impact.fandom.com"); expect(url.pathname).toBe("/api.php"); expect(url.searchParams.get("page")).toBe(title); expect(url.searchParams.get("action")).toBe("parse"); expect(url.searchParams.get("prop")).toBe("text"); expect(url.searchParams.get("section")).toBe("0"); expect(calls[0]!.userAgent).toBe("test-agent"); }); test("the same title twice is one request", async () => { const { done, calls } = run(() => new Response(WITH_TIME), { titles: [TITLE, TITLE] }); await done; expect(calls.length).toBe(1); }); test("per-host spacing applies when the host was already asked this cycle", async () => { const { done, naps } = run(() => new Response(WITH_TIME), { titles: ["A", "B"], requestedHosts: new Set(["genshin-impact.fandom.com"]), }); await done; expect(naps).toEqual([2000, 2000]); }); test("a dry run requests and writes nothing", async () => { await articles.saveRead("Old", NO_TIME, ago(24 * HOUR)); const { done, calls, logs } = run(() => new Response(WITH_TIME), { dryRun: true }); expect((await done).changed).toBe(0); expect(calls.length).toBe(0); expect(await articles.readBody(TITLE)).toBeNull(); expect(await articles.readBody("Old")).toBe(NO_TIME); expect(logs.join("\n")).toContain(TITLE); }); test("a stored article no longer listed is removed", async () => { await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); const result = await run(() => new Response(WITH_TIME)).done; expect(await articles.listed()).toEqual([TITLE]); expect(result.changed).toBe(2); }); test("no listed titles removes nothing", async () => { await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); expect((await run(() => new Response(WITH_TIME), { titles: [] }).done).changed).toBe(0); expect(await articles.listed()).toEqual(["Old"]); }); test("no directory reads as no ends", async () => { expect((await articles.readEnds()).size).toBe(0); expect(await articles.listed()).toEqual([]); }); // Through the runner. The stand-in parser reads `url` as listed rows. const FANDOM_ID = "genshin-fandom-events"; function fandomAdapter(): Adapter { return { id: FANDOM_ID, game: "genshin", url: "https://genshin-impact.fandom.com/api.php?action=parse&page=Event", parserId: "fandom", minIntervalMs: 6 * HOUR, priority: 0, parse(html: string, _ctx: ParseContext): GachaEvent[] { return [...html.matchAll(/(.*?)<\/row>/g)].map((m) => ({ sourceUrl: m[1] }) as GachaEvent); }, }; } const ROW = `https://genshin-impact.fandom.com/wiki/${TITLE}`; function runner(page: string, respondArticle: () => Response, over: Partial = {}) { const urls: string[] = []; const rebuilds = { count: 0 }; const store = new SnapshotStore(root); const opts: RefreshOptions = { adapters: [fandomAdapter()], store, robots: ALLOW_ALL, fetchImpl: async (url) => { urls.push(url); return url.includes("page=Event") ? new Response(page) : respondArticle(); }, userAgent: "test-agent", now: () => NOW, sleep: async () => {}, dryRun: false, only: null, force: false, timeoutMs: 1000, log: () => {}, rebuildFeed: async () => { rebuilds.count += 1; }, articleSourceId: FANDOM_ID, ...over, }; return { opts, urls, rebuilds, store }; } test("article reads run only with articleSourceId set; existing callers are unaffected", async () => { const { opts, urls } = runner(`${ROW}`, () => new Response(WITH_TIME)); const { articleSourceId: _omit, ...without } = opts; await runRefresh(without); expect(urls.length).toBe(1); }); test("a listed row's article is read and stored", async () => { const { opts, urls } = runner(`${ROW}`, () => new Response(WITH_TIME)); const summary = await runRefresh(opts); expect(urls.length).toBe(2); expect(summary.articlesChanged).toBe(1); expect((await articles.readEnds()).has(TITLE)).toBe(true); }); test("a row with no article link makes no article request", async () => { const { opts, urls } = runner(`${fandomAdapter().url}`, () => new Response(WITH_TIME)); await runRefresh(opts); expect(urls.length).toBe(1); }); test("articles are read when --only names another source not at all", async () => { const other: Adapter = { ...fandomAdapter(), id: "other", url: "https://example.test/x" }; const { opts, urls } = runner(`${ROW}`, () => new Response(WITH_TIME), { adapters: [fandomAdapter(), other], only: "other", }); await runRefresh(opts); expect(urls.some((u) => u.includes("Hour_of_Ruin"))).toBe(false); }); test("an empty source snapshot removes nothing and does not throw", async () => { await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); const { opts, store } = runner("", () => new Response(WITH_TIME)); await store.save(FANDOM_ID, { url: opts.adapters[0]!.url, body: "", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 0, }); const summary = await runRefresh({ ...opts, fetchImpl: async () => new Response("", { status: 304 }) }); expect(summary.hardFailure).toBeNull(); expect(await articles.listed()).toEqual(["Old"]); }); test("article failures leave broken, hardFailure and the source's streak untouched", async () => { const { opts, store } = runner(`${ROW}`, () => new Response("no", { status: 500 })); const summary = await runRefresh(opts); expect(summary.hardFailure).toBeNull(); expect(summary.broken).toEqual([]); expect(summary.warnings.some((w) => w.includes(TITLE))).toBe(true); expect((await store.readState(FANDOM_ID)).consecutiveFailures).toBe(0); }); test("a parser that throws on the snapshot is one warning, not a failed run", async () => { const adapter: Adapter = { ...fandomAdapter(), parse: (html) => { if (html.includes("")) throw new Error("template"); return [{ sourceUrl: ROW } as GachaEvent]; }, }; const { opts, store } = runner("x", () => new Response(WITH_TIME), { adapters: [adapter] }); await store.save(FANDOM_ID, { url: adapter.url, body: "", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1, }); const skipFetch = { ...opts, fetchImpl: async () => new Response("", { status: 304 }) }; const summary = await runRefresh(skipFetch); expect(summary.hardFailure).toBeNull(); expect(summary.warnings.some((w) => w.includes("article"))).toBe(true); }); test("a changed article triggers the rebuild even when no source page changed", async () => { const page = `${ROW}`; const { opts, rebuilds, store } = runner(page, () => new Response(WITH_TIME)); await store.save(FANDOM_ID, { url: opts.adapters[0]!.url, body: page, etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1, }); const summary = await runRefresh(opts); expect(summary.changed).toBe(0); expect(summary.articlesChanged).toBe(1); expect(rebuilds.count).toBe(1); }); test("a row kept alive by its stated end stays listed, so its article is not removed", async () => { // On a row's last day the parser only emits it because of the stored end. Read // the table without those ends and the article looks unlisted and gets deleted, // and the row vanishes from the feed hours early. await articles.saveRead(TITLE, WITH_TIME, ago(24 * HOUR)); const adapter: Adapter = { ...fandomAdapter(), parse: (_html, ctx) => (ctx.articleEnds?.has(TITLE) ? [{ sourceUrl: ROW } as GachaEvent] : []), }; const { opts } = runner("", () => new Response(WITH_TIME), { adapters: [adapter] }); await opts.store.save(FANDOM_ID, { url: adapter.url, body: "", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1, }); await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); await runRefresh({ ...opts, fetchImpl: async () => new Response("", { status: 304 }) }); expect(await articles.listed()).toEqual([TITLE]); });