diff --git a/.gitignore b/.gitignore index db3080c..7204257 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,7 @@ snapshots/*.state.json # the refresh workflow commits the whole snapshots/ directory, which would # otherwise pin a truncated page in git forever. snapshots/*.tmp-* +snapshots/articles/**/*.tmp-* # Impeccable design tooling. The local config only records this machine's hook # consent. (The design.json sidecar was dropped; see AGENTS.md, docs/DESIGN.md row.) diff --git a/scripts/refresh-articles.ts b/scripts/refresh-articles.ts new file mode 100644 index 0000000..f5fe4e3 --- /dev/null +++ b/scripts/refresh-articles.ts @@ -0,0 +1,127 @@ +/** + * Keep Genshin event articles beside the snapshots (spec decisions 5, 6 and 12). + * + * Nothing here can fail a refresh or touch a source's failure streak: an article + * only ever improves a row, and a row without one is exactly what it was. What a + * failed read does to a time already held depends on what the wiki actually said. + */ +import { parseArticleEnd } from "../src/ingest/articleEnds.ts"; +import { articleDue, type ArticleStore } from "../src/ingest/articleStore.ts"; +import type { FetchLike } from "../src/ingest/robots.ts"; +import type { RobotsGate } from "./refresh-sources.ts"; + +const HOST = "genshin-impact.fandom.com"; +const DEFAULT_HOST_GAP_MS = 2_000; + +export interface RefreshArticlesInput { + titles: readonly string[]; + store: ArticleStore; + robots: RobotsGate; + fetchImpl: FetchLike; + userAgent: string; + now: () => Date; + sleep: (ms: number) => Promise; + dryRun: boolean; + timeoutMs: number; + log: (line: string) => void; + /** Shared with the source loop, which has usually asked this host already. */ + requestedHosts: Set; +} + +export function articleUrl(title: string): string { + return ( + `https://${HOST}/api.php?action=parse&page=${encodeURIComponent(title)}` + + "&prop=text§ion=0&formatversion=2&format=json" + ); +} + +/** + * A 200 is not yet an answer: a challenge page is a 200 too, and reading one as + * "the article states no time" would throw away a good time on a passing blip. + */ +function answerOf(body: string): "article" | "gone" | "unusable" { + try { + const parsed = JSON.parse(body) as { + parse?: { text?: unknown }; + error?: { code?: unknown }; + } | null; + if (typeof parsed?.parse?.text === "string") return "article"; + if (parsed?.error?.code === "missingtitle") return "gone"; + } catch { + // Not JSON, so not the API's answer. + } + return "unusable"; +} + +export async function refreshArticles( + input: RefreshArticlesInput, +): Promise<{ changed: number; warnings: string[] }> { + const { store, log } = input; + const titles = [...new Set(input.titles)]; + const nowIso = input.now().toISOString(); + const nowMs = input.now().getTime(); + const warnings: string[] = []; + let changed = 0; + + for (const title of titles) { + const label = `article ${title}`; + if (!articleDue(await store.readMeta(title), nowMs)) continue; + if (input.dryRun) { + log(` ${label}: would read`); + continue; + } + + const url = articleUrl(title); + const failed = async (why: string) => { + await store.recordAttempt(title, nowIso); + warnings.push(`${label}: ${why}; keeping what is stored`); + }; + + const decision = await input.robots.allows(url); + if (!decision.allowed) { + await failed(decision.reason); + continue; + } + if (input.requestedHosts.has(HOST)) { + await input.sleep(decision.crawlDelayMs ?? DEFAULT_HOST_GAP_MS); + } + input.requestedHosts.add(HOST); + + let status: number; + let body: string; + try { + const response = await input.fetchImpl(url, { + headers: { "User-Agent": input.userAgent, Accept: "application/json" }, + signal: AbortSignal.timeout(input.timeoutMs), + redirect: "follow", + }); + status = response.status; + body = status === 200 ? await response.text() : ""; + } catch (error) { + await failed(`unreachable: ${String(error)}`); + continue; + } + + const answer = status === 200 ? answerOf(body) : status === 404 ? "gone" : "unusable"; + if (answer === "gone") { + if (await store.markGone(title, nowIso)) changed += 1; + warnings.push(`${label}: the wiki says it does not exist`); + } else if (answer === "unusable") { + await failed(status === 200 ? "the answer was not the API's" : `HTTP ${status}`); + } else { + if (await store.saveRead(title, body, nowIso)) changed += 1; + // A stub is ordinary and a warning per stub per cycle would bury real ones. + if (parseArticleEnd(body) === null) log(` ${label}: states no time yet`); + } + } + + // No titles means the table failed to parse, not that every event ended. + if (titles.length > 0 && !input.dryRun) { + const listed = new Set(titles); + for (const title of await store.listed()) { + if (!listed.has(title) && (await store.remove(title))) changed += 1; + } + } + + return { changed, warnings }; +} diff --git a/scripts/refresh-sources.ts b/scripts/refresh-sources.ts index bfc6803..3728731 100644 --- a/scripts/refresh-sources.ts +++ b/scripts/refresh-sources.ts @@ -48,10 +48,13 @@ import { appendFile } from "node:fs/promises"; import { ADAPTERS, } from "../src/ingest/adapters/index.ts"; -import { SIX_HOURS_MS } from "../src/ingest/adapters/types.ts"; +import { SIX_HOURS_MS, type Adapter } from "../src/ingest/adapters/types.ts"; import { CODE_SOURCES, type RefreshSource } from "../src/ingest/codes/registry.ts"; const REFRESH_SOURCES: readonly RefreshSource[] = [...ADAPTERS, ...CODE_SOURCES]; import { RobotsCache, type FetchLike } from "../src/ingest/robots.ts"; +import { articleTitle } from "../src/ingest/articleEnds.ts"; +import { ArticleStore } from "../src/ingest/articleStore.ts"; +import { refreshArticles } from "./refresh-articles.ts"; import { decodeBody, SnapshotStore } from "../src/ingest/snapshots.ts"; const DEFAULT_CONTACT = @@ -92,6 +95,8 @@ export interface RefreshSummary { outcomes: SourceOutcome[]; /** Sources whose stored bytes changed — the only reason to commit. */ changed: number; + /** Stored articles whose bytes changed. Rebuilds the feed; never a source result. */ + articlesChanged: number; /** Sources we actually sent a request to. */ attempted: number; /** Sources that answered (200 or 304). */ @@ -151,6 +156,11 @@ export interface RefreshOptions { force: boolean; timeoutMs: number; log: (line: string) => void; + /** + * The source whose rows have articles worth reading for an exact end. Unset + * reads none. `--force` deliberately does not reach these (spec decision 6). + */ + articleSourceId?: string; /** Called once when something changed. Null skips the rebuild (tests). */ rebuildFeed: (() => Promise) | null; } @@ -188,6 +198,7 @@ export async function runRefresh( const summary: RefreshSummary = { outcomes: [], changed: 0, + articlesChanged: 0, attempted: 0, confirmed: 0, warnings: [], @@ -288,6 +299,44 @@ export async function runRefresh( } } + // After every source had its turn, so an article can never cost a page one. + const articleSource = selected.find((a) => a.id === options.articleSourceId); + if (articleSource !== undefined) { + try { + const snapshot = await options.store.read(articleSource.id); + if (snapshot !== null) { + const articles = new ArticleStore(options.store.root); + // With the stored ends, as the feed build does: on a row's last day they + // are what keeps it listed, and an unlisted article gets deleted. + const rows = (articleSource.parse as Adapter["parse"])(snapshot.html, { + now: options.now().toISOString(), + sourceUrl: articleSource.url, + sourceId: articleSource.id, + game: articleSource.game, + articleEnds: await articles.readEnds(), + }); + const titles = rows.flatMap((row) => articleTitle(row.sourceUrl) ?? []); + const result = await refreshArticles({ + titles, + store: articles, + robots: options.robots, + fetchImpl: options.fetchImpl, + userAgent: options.userAgent, + now: options.now, + sleep: options.sleep, + dryRun: options.dryRun, + timeoutMs: options.timeoutMs, + log: options.log, + requestedHosts: cycle.requestedHosts, + }); + summary.articlesChanged = result.changed; + summary.warnings.push(...result.warnings); + } + } catch (error) { + summary.warnings.push(`articles for ${articleSource.id}: ${String(error)}`); + } + } + // Named in the summary rather than only in the per-source log, so it survives // into the job summary and cannot be scrolled past. summary.forced = [...cycle.forced].sort(); @@ -321,7 +370,10 @@ export async function runRefresh( return summary; } - if (summary.changed > 0 && options.rebuildFeed !== null) { + if ( + (summary.changed > 0 || summary.articlesChanged > 0) && + options.rebuildFeed !== null + ) { try { await options.rebuildFeed(); } catch (error) { @@ -977,6 +1029,7 @@ async function main(): Promise { timeoutMs: 20_000, log: (line) => console.log(line), rebuildFeed: args.dryRun || !args.rebuild ? null : rebuildFeedViaScript, + articleSourceId: "genshin-fandom-events", }); console.log( diff --git a/snapshots/README.md b/snapshots/README.md index 4c677e9..8d99ad2 100644 --- a/snapshots/README.md +++ b/snapshots/README.md @@ -20,6 +20,12 @@ interrupted run leaves a stray temp file rather than a truncated snapshot or met bytes that were never stored. Those temp files are gitignored: `refresh.yml` commits the whole directory, so otherwise a run killed mid-write would pin a half-page in git forever. +`articles/genshin-fandom/` holds the lead section of each Genshin event's wiki article, as the API +served it, for the end time its infobox states (`docs/INGESTION.md` § Article ends). `.html` is +the response and `.meta.json` its read history. There is no `.state.json`: the due rule has to +survive a fresh checkout, and a failed article read is never a source's failure streak. The refresh +removes an article once its row leaves the table. + Three reasons this is committed rather than cached: - **Re-parsing never re-fetches.** Iterating on a parser reads these files, not the wikis diff --git a/src/ingest/articleStore.ts b/src/ingest/articleStore.ts new file mode 100644 index 0000000..b6fae37 --- /dev/null +++ b/src/ingest/articleStore.ts @@ -0,0 +1,145 @@ +import { mkdir, readdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { parseArticleEnd, type StatedEnd } from "./articleEnds.ts"; +import { hashBody, writeAtomic } from "./snapshots.ts"; + +/** + * Stored lead sections of Genshin event articles, under the snapshot root. + * + * Tracked like every other snapshot, so re-parsing never re-fetches and a moved + * end time is a reviewable diff. There is no `.state.json`: an article's read + * history lives in its metadata, because the due rule has to survive a fresh + * checkout and a failed read here is never a source's failure streak. + */ +export interface ArticleMeta { + title: string; + lastAttemptAt: string; + /** The last read that worked. A failed attempt never moves this. */ + lastReadAt: string | null; + statesTime: boolean; +} + +const SIX_HOURS_MS = 6 * 60 * 60 * 1000; +const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000; + +/** + * Titles hold `/`, `:` and `'`. The hash keeps two titles that flatten alike apart. + */ +export function articleFileName(title: string): string { + return `${title.replace(/[^A-Za-z0-9._-]/g, "_")}-${hashBody(title).slice(0, 8)}`; +} + +/** Spec decision 6: wiki editors correct these, so a kept time still gets re-read. */ +export function articleDue(meta: ArticleMeta | null, nowMs: number): boolean { + if (meta === null) return true; + if (nowMs - Date.parse(meta.lastAttemptAt) < SIX_HOURS_MS) return false; + if (!meta.statesTime || meta.lastReadAt === null) return true; + return nowMs - Date.parse(meta.lastReadAt) >= SEVEN_DAYS_MS; +} + +export class ArticleStore { + private readonly dir: string; + + constructor(root: string) { + this.dir = join(root, "articles", "genshin-fandom"); + } + + private bodyPath(title: string): string { + return join(this.dir, `${articleFileName(title)}.html`); + } + + private metaPath(title: string): string { + return join(this.dir, `${articleFileName(title)}.meta.json`); + } + + private async writeMeta(meta: ArticleMeta): Promise { + await mkdir(this.dir, { recursive: true }); + await writeAtomic(this.metaPath(meta.title), `${JSON.stringify(meta, null, 2)}\n`); + } + + async readMeta(title: string): Promise { + const file = Bun.file(this.metaPath(title)); + return (await file.exists()) ? ((await file.json()) as ArticleMeta) : null; + } + + async readBody(title: string): Promise { + const file = Bun.file(this.bodyPath(title)); + return (await file.exists()) ? await file.text() : null; + } + + /** A read that worked. Body before metadata. True when the stored bytes moved. */ + async saveRead(title: string, body: string, at: string): Promise { + const changed = (await this.readBody(title)) !== body; + await mkdir(this.dir, { recursive: true }); + if (changed) await writeAtomic(this.bodyPath(title), body); + await this.writeMeta({ + title, + lastAttemptAt: at, + lastReadAt: at, + statesTime: parseArticleEnd(body) !== null, + }); + return changed; + } + + /** A request that told us nothing about the article: keep what we hold. */ + async recordAttempt(title: string, at: string): Promise { + const meta = await this.readMeta(title); + await this.writeMeta({ + title, + lastAttemptAt: at, + lastReadAt: meta?.lastReadAt ?? null, + statesTime: meta?.statesTime ?? false, + }); + } + + /** + * The wiki says the article is gone. The metadata stays while the row is + * listed, so the 6-hour floor still holds for the retries. True if bytes went. + */ + async markGone(title: string, at: string): Promise { + const held = (await this.readBody(title)) !== null; + await rm(this.bodyPath(title), { force: true }); + await this.writeMeta({ + title, + lastAttemptAt: at, + lastReadAt: (await this.readMeta(title))?.lastReadAt ?? null, + statesTime: false, + }); + return held; + } + + /** The row left the table. True if bytes went. */ + async remove(title: string): Promise { + const held = (await this.readBody(title)) !== null; + await rm(this.bodyPath(title), { force: true }); + await rm(this.metaPath(title), { force: true }); + return held; + } + + /** Every stored title. Empty, not an error, on a checkout with no directory. */ + async listed(): Promise { + let names: string[]; + try { + names = await readdir(this.dir); + } catch { + return []; + } + const titles: string[] = []; + for (const name of names.filter((n) => n.endsWith(".meta.json"))) { + const meta = (await Bun.file(join(this.dir, name)).json()) as ArticleMeta; + titles.push(meta.title); + } + return titles.sort(); + } + + /** What the feed build reads. An article stating no time contributes nothing. */ + async readEnds(): Promise> { + const ends = new Map(); + for (const title of await this.listed()) { + const body = await this.readBody(title); + const end = body === null ? null : parseArticleEnd(body); + if (end !== null) ends.set(title, end); + } + return ends; + } +} diff --git a/src/ingest/snapshots.ts b/src/ingest/snapshots.ts index 9a16979..66f0879 100644 --- a/src/ingest/snapshots.ts +++ b/src/ingest/snapshots.ts @@ -163,7 +163,7 @@ function decodeStored(bytes: Uint8Array, charset: string | undefined): string { * the next refresh — sees either the whole old file or the whole new one, and * a crash mid-write leaves a stray temp file rather than a truncated snapshot. */ -async function writeAtomic(path: string, data: Uint8Array | string): Promise { +export async function writeAtomic(path: string, data: Uint8Array | string): Promise { const temp = `${path}.tmp-${process.pid.toString(36)}-${Math.random().toString(36).slice(2, 10)}`; try { await writeFile(temp, data); diff --git a/test/article-refresh.test.ts b/test/article-refresh.test.ts new file mode 100644 index 0000000..90fc051 --- /dev/null +++ b/test/article-refresh.test.ts @@ -0,0 +1,409 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { refreshArticles } from "../scripts/refresh-articles.ts"; +import { runRefresh, type RefreshOptions, type RobotsGate } from "../scripts/refresh-sources.ts"; +import type { Adapter, ParseContext } from "../src/ingest/adapters/types.ts"; +import { articleDue, articleFileName, ArticleStore, type ArticleMeta } from "../src/ingest/articleStore.ts"; +import { SnapshotStore } from "../src/ingest/snapshots.ts"; +import type { GachaEvent } from "../src/shared/schema.ts"; + +const NOW = new Date("2026-11-07T12:00:00.000Z"); +const HOUR = 60 * 60 * 1000; +const ago = (ms: number) => new Date(NOW.getTime() - ms).toISOString(); +const TITLE = "Hour_of_Ruin_(Event)"; +const WITH_TIME = await Bun.file("fixtures/genshin/articles/hour-of-ruin-2026-11-07.html").text(); +const NO_TIME = JSON.stringify({ parse: { text: "

A stub.

" } }); +const MISSING = JSON.stringify({ error: { code: "missingtitle", info: "The page does not exist." } }); + +let root: string; +let articles: ArticleStore; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "event-clock-articles-")); + articles = new ArticleStore(root); +}); +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +const ALLOW_ALL: RobotsGate = { allows: async () => ({ allowed: true, reason: "ok" }) }; + +interface Call { url: string; userAgent: string | undefined } + +function run( + respond: () => Response | Promise, + over: Partial[0]> = {}, +) { + const calls: Call[] = []; + const naps: number[] = []; + const logs: string[] = []; + const done = refreshArticles({ + titles: [TITLE], + store: articles, + robots: ALLOW_ALL, + fetchImpl: async (url, init) => { + calls.push({ url, userAgent: (init?.headers as Record)["User-Agent"] }); + return respond(); + }, + userAgent: "test-agent", + now: () => NOW, + sleep: async (ms) => void naps.push(ms), + dryRun: false, + timeoutMs: 1000, + log: (l) => void logs.push(l), + requestedHosts: new Set(), + ...over, + }); + return { done, calls, naps, logs }; +} + +const meta = (over: Partial): ArticleMeta => ({ + title: TITLE, + lastAttemptAt: ago(30 * 24 * HOUR), + lastReadAt: ago(30 * 24 * HOUR), + statesTime: true, + ...over, +}); +const at = NOW.getTime(); + +test("due: never read", () => { + expect(articleDue(null, at)).toBe(true); +}); + +test("due: no time found, only after 6 hours", () => { + const m = (h: number) => meta({ lastAttemptAt: ago(h * HOUR), lastReadAt: ago(h * HOUR), statesTime: false }); + expect(articleDue(m(5.9), at)).toBe(false); + expect(articleDue(m(6), at)).toBe(true); +}); + +test("due: time found, only 7 days after the last read that worked", () => { + const m = (d: number) => meta({ lastAttemptAt: ago(d * 24 * HOUR), lastReadAt: ago(d * 24 * HOUR) }); + expect(articleDue(m(6.9), at)).toBe(false); + expect(articleDue(m(7), at)).toBe(true); +}); + +test("due: a failed attempt inside 6 hours is not due, even past 7 days", () => { + expect(articleDue(meta({ lastAttemptAt: ago(1 * HOUR) }), at)).toBe(false); + expect(articleDue(meta({ lastAttemptAt: ago(6 * HOUR) }), at)).toBe(true); +}); + +test("a percent-decoded title gets one stable file name that keeps titles apart", () => { + const name = articleFileName("The_Forge_Realm's_Temper/2026-11-04"); + expect(name).toMatch(/^[A-Za-z0-9._-]+$/); + expect(name).toBe(articleFileName("The_Forge_Realm's_Temper/2026-11-04")); + expect(articleFileName("a/b")).not.toBe(articleFileName("a:b")); +}); + +test("a first read stores the article and its end", async () => { + const { done } = run(() => new Response(WITH_TIME)); + expect((await done).changed).toBe(1); + expect((await articles.readEnds()).get(TITLE)?.day).toBe("2026-12-15"); + expect(await articles.readMeta(TITLE)).toEqual({ + title: TITLE, + lastAttemptAt: NOW.toISOString(), + lastReadAt: NOW.toISOString(), + statesTime: true, + }); +}); + +/** A stored article whose 7-day re-read is due now. */ +async function dueArticle() { + await articles.saveRead(TITLE, WITH_TIME, ago(8 * 24 * HOUR)); +} + +async function expectKept(result: { changed: number; warnings: string[] }) { + expect(result.changed).toBe(0); + expect(result.warnings.length).toBe(1); + expect(await articles.readBody(TITLE)).toBe(WITH_TIME); + expect((await articles.readEnds()).has(TITLE)).toBe(true); + const m = await articles.readMeta(TITLE); + expect(m?.lastReadAt).toBe(ago(8 * 24 * HOUR)); + expect(m?.lastAttemptAt).toBe(NOW.toISOString()); +} + +test.each([500, 403, 202])("HTTP %i keeps the body and the time, and warns", async (status) => { + await dueArticle(); + await expectKept(await run(() => new Response("nope", { status })).done); +}); + +test("a network error keeps the body and the time, and warns", async () => { + await dueArticle(); + await expectKept(await run(() => { throw new Error("reset"); }).done); +}); + +test("a body that fails mid-read keeps the body and the time, and warns", async () => { + await dueArticle(); + await expectKept(await run(() => ({ + status: 200, + arrayBuffer: () => Promise.reject(new Error("truncated")), + text: () => Promise.reject(new Error("truncated")), + }) as unknown as Response).done); +}); + +test("a 200 that is not the API's answer keeps the body and the time, and warns", async () => { + await dueArticle(); + await expectKept(await run(() => new Response("Just a moment…")).done); +}); + +test("a 200 API error other than a missing page keeps the body and the time", async () => { + await dueArticle(); + await expectKept(await run(() => new Response(JSON.stringify({ error: { code: "ratelimited" } }))).done); +}); + +test("robots refusal makes no request, keeps the body, and warns", async () => { + await dueArticle(); + const robots: RobotsGate = { allows: async () => ({ allowed: false, reason: "disallowed" }) }; + const { done, calls } = run(() => new Response(WITH_TIME), { robots }); + await expectKept(await done); + expect(calls.length).toBe(0); +}); + +test.each([ + ["404", () => new Response("gone", { status: 404 })], + ["a missingtitle API error", () => new Response(MISSING)], +])("%s removes the stored article and warns", async (_name, respond) => { + await dueArticle(); + const result = await run(respond).done; + expect(result.changed).toBe(1); + expect(result.warnings.length).toBe(1); + expect(await articles.readBody(TITLE)).toBeNull(); + expect((await articles.readEnds()).has(TITLE)).toBe(false); +}); + +test("a removed article still honours the 6-hour floor", async () => { + await dueArticle(); + await run(() => new Response("gone", { status: 404 })).done; + const { done, calls } = run(() => new Response(WITH_TIME)); + await done; + expect(calls.length).toBe(0); +}); + +test("a 200 with no usable End overwrites the body, sets statesTime false, and logs without warning", async () => { + await dueArticle(); + const { done, logs } = run(() => new Response(NO_TIME)); + const result = await done; + expect(result.changed).toBe(1); + expect(result.warnings).toEqual([]); + expect(logs.join("\n")).toContain(TITLE); + expect(await articles.readBody(TITLE)).toBe(NO_TIME); + expect((await articles.readMeta(TITLE))?.statesTime).toBe(false); + expect((await articles.readEnds()).has(TITLE)).toBe(false); +}); + +test("an unchanged re-read moves lastReadAt but changes no bytes", async () => { + await dueArticle(); + const result = await run(() => new Response(WITH_TIME)).done; + expect(result.changed).toBe(0); + expect((await articles.readMeta(TITLE))?.lastReadAt).toBe(NOW.toISOString()); +}); + +test("an article not yet due makes no request", async () => { + await articles.saveRead(TITLE, WITH_TIME, ago(24 * HOUR)); + const { done, calls } = run(() => new Response(WITH_TIME)); + await done; + expect(calls.length).toBe(0); +}); + +test("the requested URL carries the decoded title, section=0, and the runner User-Agent", async () => { + const title = "The_Forge_Realm's_Temper/2026-11-04"; + const { done, calls } = run(() => new Response(WITH_TIME), { titles: [title] }); + await done; + const url = new URL(calls[0]!.url); + expect(url.host).toBe("genshin-impact.fandom.com"); + expect(url.pathname).toBe("/api.php"); + expect(url.searchParams.get("page")).toBe(title); + expect(url.searchParams.get("action")).toBe("parse"); + expect(url.searchParams.get("prop")).toBe("text"); + expect(url.searchParams.get("section")).toBe("0"); + expect(calls[0]!.userAgent).toBe("test-agent"); +}); + +test("the same title twice is one request", async () => { + const { done, calls } = run(() => new Response(WITH_TIME), { titles: [TITLE, TITLE] }); + await done; + expect(calls.length).toBe(1); +}); + +test("per-host spacing applies when the host was already asked this cycle", async () => { + const { done, naps } = run(() => new Response(WITH_TIME), { + titles: ["A", "B"], + requestedHosts: new Set(["genshin-impact.fandom.com"]), + }); + await done; + expect(naps).toEqual([2000, 2000]); +}); + +test("a dry run requests and writes nothing", async () => { + await articles.saveRead("Old", NO_TIME, ago(24 * HOUR)); + const { done, calls, logs } = run(() => new Response(WITH_TIME), { dryRun: true }); + expect((await done).changed).toBe(0); + expect(calls.length).toBe(0); + expect(await articles.readBody(TITLE)).toBeNull(); + expect(await articles.readBody("Old")).toBe(NO_TIME); + expect(logs.join("\n")).toContain(TITLE); +}); + +test("a stored article no longer listed is removed", async () => { + await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); + const result = await run(() => new Response(WITH_TIME)).done; + expect(await articles.listed()).toEqual([TITLE]); + expect(result.changed).toBe(2); +}); + +test("no listed titles removes nothing", async () => { + await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); + expect((await run(() => new Response(WITH_TIME), { titles: [] }).done).changed).toBe(0); + expect(await articles.listed()).toEqual(["Old"]); +}); + +test("no directory reads as no ends", async () => { + expect((await articles.readEnds()).size).toBe(0); + expect(await articles.listed()).toEqual([]); +}); + +// Through the runner. The stand-in parser reads `url` as listed rows. +const FANDOM_ID = "genshin-fandom-events"; +function fandomAdapter(): Adapter { + return { + id: FANDOM_ID, + game: "genshin", + url: "https://genshin-impact.fandom.com/api.php?action=parse&page=Event", + parserId: "fandom", + minIntervalMs: 6 * HOUR, + priority: 0, + parse(html: string, _ctx: ParseContext): GachaEvent[] { + return [...html.matchAll(/(.*?)<\/row>/g)].map((m) => ({ sourceUrl: m[1] }) as GachaEvent); + }, + }; +} +const ROW = `https://genshin-impact.fandom.com/wiki/${TITLE}`; + +function runner(page: string, respondArticle: () => Response, over: Partial = {}) { + const urls: string[] = []; + const rebuilds = { count: 0 }; + const store = new SnapshotStore(root); + const opts: RefreshOptions = { + adapters: [fandomAdapter()], + store, + robots: ALLOW_ALL, + fetchImpl: async (url) => { + urls.push(url); + return url.includes("page=Event") ? new Response(page) : respondArticle(); + }, + userAgent: "test-agent", + now: () => NOW, + sleep: async () => {}, + dryRun: false, + only: null, + force: false, + timeoutMs: 1000, + log: () => {}, + rebuildFeed: async () => { + rebuilds.count += 1; + }, + articleSourceId: FANDOM_ID, + ...over, + }; + return { opts, urls, rebuilds, store }; +} + +test("article reads run only with articleSourceId set; existing callers are unaffected", async () => { + const { opts, urls } = runner(`${ROW}`, () => new Response(WITH_TIME)); + const { articleSourceId: _omit, ...without } = opts; + await runRefresh(without); + expect(urls.length).toBe(1); +}); + +test("a listed row's article is read and stored", async () => { + const { opts, urls } = runner(`${ROW}`, () => new Response(WITH_TIME)); + const summary = await runRefresh(opts); + expect(urls.length).toBe(2); + expect(summary.articlesChanged).toBe(1); + expect((await articles.readEnds()).has(TITLE)).toBe(true); +}); + +test("a row with no article link makes no article request", async () => { + const { opts, urls } = runner(`${fandomAdapter().url}`, () => new Response(WITH_TIME)); + await runRefresh(opts); + expect(urls.length).toBe(1); +}); + +test("articles are read when --only names another source not at all", async () => { + const other: Adapter = { ...fandomAdapter(), id: "other", url: "https://example.test/x" }; + const { opts, urls } = runner(`${ROW}`, () => new Response(WITH_TIME), { + adapters: [fandomAdapter(), other], + only: "other", + }); + await runRefresh(opts); + expect(urls.some((u) => u.includes("Hour_of_Ruin"))).toBe(false); +}); + +test("an empty source snapshot removes nothing and does not throw", async () => { + await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); + const { opts, store } = runner("", () => new Response(WITH_TIME)); + await store.save(FANDOM_ID, { + url: opts.adapters[0]!.url, body: "", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 0, + }); + const summary = await runRefresh({ ...opts, fetchImpl: async () => new Response("", { status: 304 }) }); + expect(summary.hardFailure).toBeNull(); + expect(await articles.listed()).toEqual(["Old"]); +}); + +test("article failures leave broken, hardFailure and the source's streak untouched", async () => { + const { opts, store } = runner(`${ROW}`, () => new Response("no", { status: 500 })); + const summary = await runRefresh(opts); + expect(summary.hardFailure).toBeNull(); + expect(summary.broken).toEqual([]); + expect(summary.warnings.some((w) => w.includes(TITLE))).toBe(true); + expect((await store.readState(FANDOM_ID)).consecutiveFailures).toBe(0); +}); + +test("a parser that throws on the snapshot is one warning, not a failed run", async () => { + const adapter: Adapter = { + ...fandomAdapter(), + parse: (html) => { + if (html.includes("")) throw new Error("template"); + return [{ sourceUrl: ROW } as GachaEvent]; + }, + }; + const { opts, store } = runner("x", () => new Response(WITH_TIME), { adapters: [adapter] }); + await store.save(FANDOM_ID, { + url: adapter.url, body: "", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1, + }); + const skipFetch = { ...opts, fetchImpl: async () => new Response("", { status: 304 }) }; + const summary = await runRefresh(skipFetch); + expect(summary.hardFailure).toBeNull(); + expect(summary.warnings.some((w) => w.includes("article"))).toBe(true); +}); + +test("a changed article triggers the rebuild even when no source page changed", async () => { + const page = `${ROW}`; + const { opts, rebuilds, store } = runner(page, () => new Response(WITH_TIME)); + await store.save(FANDOM_ID, { + url: opts.adapters[0]!.url, body: page, etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1, + }); + const summary = await runRefresh(opts); + expect(summary.changed).toBe(0); + expect(summary.articlesChanged).toBe(1); + expect(rebuilds.count).toBe(1); +}); + +test("a row kept alive by its stated end stays listed, so its article is not removed", async () => { + // On a row's last day the parser only emits it because of the stored end. Read + // the table without those ends and the article looks unlisted and gets deleted, + // and the row vanishes from the feed hours early. + await articles.saveRead(TITLE, WITH_TIME, ago(24 * HOUR)); + const adapter: Adapter = { + ...fandomAdapter(), + parse: (_html, ctx) => (ctx.articleEnds?.has(TITLE) ? [{ sourceUrl: ROW } as GachaEvent] : []), + }; + const { opts } = runner("", () => new Response(WITH_TIME), { adapters: [adapter] }); + await opts.store.save(FANDOM_ID, { + url: adapter.url, body: "", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1, + }); + await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR)); + await runRefresh({ ...opts, fetchImpl: async () => new Response("", { status: 304 }) }); + expect(await articles.listed()).toEqual([TITLE]); +}); diff --git a/test/refresh.test.ts b/test/refresh.test.ts index 02b17e0..fcf2e71 100644 --- a/test/refresh.test.ts +++ b/test/refresh.test.ts @@ -973,6 +973,7 @@ describe("what the runner reports to the runner", () => { }, ], changed: 0, + articlesChanged: 0, attempted: 1, confirmed: 0, warnings: ["genshin-game8-events: HTTP 403 (cloudflare, cf-ray)"],