feat(refresh): keep Genshin event articles beside the snapshots
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
7f086ece36
commit
43530fc222
@@ -132,6 +132,7 @@ snapshots/*.state.json
|
||||
# the refresh workflow commits the whole snapshots/ directory, which would
|
||||
# otherwise pin a truncated page in git forever.
|
||||
snapshots/*.tmp-*
|
||||
snapshots/articles/**/*.tmp-*
|
||||
|
||||
# Impeccable design tooling. The local config only records this machine's hook
|
||||
# consent. (The design.json sidecar was dropped; see AGENTS.md, docs/DESIGN.md row.)
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Keep Genshin event articles beside the snapshots (spec decisions 5, 6 and 12).
|
||||
*
|
||||
* Nothing here can fail a refresh or touch a source's failure streak: an article
|
||||
* only ever improves a row, and a row without one is exactly what it was. What a
|
||||
* failed read does to a time already held depends on what the wiki actually said.
|
||||
*/
|
||||
import { parseArticleEnd } from "../src/ingest/articleEnds.ts";
|
||||
import { articleDue, type ArticleStore } from "../src/ingest/articleStore.ts";
|
||||
import type { FetchLike } from "../src/ingest/robots.ts";
|
||||
import type { RobotsGate } from "./refresh-sources.ts";
|
||||
|
||||
const HOST = "genshin-impact.fandom.com";
|
||||
const DEFAULT_HOST_GAP_MS = 2_000;
|
||||
|
||||
export interface RefreshArticlesInput {
|
||||
titles: readonly string[];
|
||||
store: ArticleStore;
|
||||
robots: RobotsGate;
|
||||
fetchImpl: FetchLike;
|
||||
userAgent: string;
|
||||
now: () => Date;
|
||||
sleep: (ms: number) => Promise<void>;
|
||||
dryRun: boolean;
|
||||
timeoutMs: number;
|
||||
log: (line: string) => void;
|
||||
/** Shared with the source loop, which has usually asked this host already. */
|
||||
requestedHosts: Set<string>;
|
||||
}
|
||||
|
||||
export function articleUrl(title: string): string {
|
||||
return (
|
||||
`https://${HOST}/api.php?action=parse&page=${encodeURIComponent(title)}` +
|
||||
"&prop=text§ion=0&formatversion=2&format=json"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A 200 is not yet an answer: a challenge page is a 200 too, and reading one as
|
||||
* "the article states no time" would throw away a good time on a passing blip.
|
||||
*/
|
||||
function answerOf(body: string): "article" | "gone" | "unusable" {
|
||||
try {
|
||||
const parsed = JSON.parse(body) as {
|
||||
parse?: { text?: unknown };
|
||||
error?: { code?: unknown };
|
||||
} | null;
|
||||
if (typeof parsed?.parse?.text === "string") return "article";
|
||||
if (parsed?.error?.code === "missingtitle") return "gone";
|
||||
} catch {
|
||||
// Not JSON, so not the API's answer.
|
||||
}
|
||||
return "unusable";
|
||||
}
|
||||
|
||||
export async function refreshArticles(
|
||||
input: RefreshArticlesInput,
|
||||
): Promise<{ changed: number; warnings: string[] }> {
|
||||
const { store, log } = input;
|
||||
const titles = [...new Set(input.titles)];
|
||||
const nowIso = input.now().toISOString();
|
||||
const nowMs = input.now().getTime();
|
||||
const warnings: string[] = [];
|
||||
let changed = 0;
|
||||
|
||||
for (const title of titles) {
|
||||
const label = `article ${title}`;
|
||||
if (!articleDue(await store.readMeta(title), nowMs)) continue;
|
||||
if (input.dryRun) {
|
||||
log(` ${label}: would read`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = articleUrl(title);
|
||||
const failed = async (why: string) => {
|
||||
await store.recordAttempt(title, nowIso);
|
||||
warnings.push(`${label}: ${why}; keeping what is stored`);
|
||||
};
|
||||
|
||||
const decision = await input.robots.allows(url);
|
||||
if (!decision.allowed) {
|
||||
await failed(decision.reason);
|
||||
continue;
|
||||
}
|
||||
if (input.requestedHosts.has(HOST)) {
|
||||
await input.sleep(decision.crawlDelayMs ?? DEFAULT_HOST_GAP_MS);
|
||||
}
|
||||
input.requestedHosts.add(HOST);
|
||||
|
||||
let status: number;
|
||||
let body: string;
|
||||
try {
|
||||
const response = await input.fetchImpl(url, {
|
||||
headers: { "User-Agent": input.userAgent, Accept: "application/json" },
|
||||
signal: AbortSignal.timeout(input.timeoutMs),
|
||||
redirect: "follow",
|
||||
});
|
||||
status = response.status;
|
||||
body = status === 200 ? await response.text() : "";
|
||||
} catch (error) {
|
||||
await failed(`unreachable: ${String(error)}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const answer = status === 200 ? answerOf(body) : status === 404 ? "gone" : "unusable";
|
||||
if (answer === "gone") {
|
||||
if (await store.markGone(title, nowIso)) changed += 1;
|
||||
warnings.push(`${label}: the wiki says it does not exist`);
|
||||
} else if (answer === "unusable") {
|
||||
await failed(status === 200 ? "the answer was not the API's" : `HTTP ${status}`);
|
||||
} else {
|
||||
if (await store.saveRead(title, body, nowIso)) changed += 1;
|
||||
// A stub is ordinary and a warning per stub per cycle would bury real ones.
|
||||
if (parseArticleEnd(body) === null) log(` ${label}: states no time yet`);
|
||||
}
|
||||
}
|
||||
|
||||
// No titles means the table failed to parse, not that every event ended.
|
||||
if (titles.length > 0 && !input.dryRun) {
|
||||
const listed = new Set(titles);
|
||||
for (const title of await store.listed()) {
|
||||
if (!listed.has(title) && (await store.remove(title))) changed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { changed, warnings };
|
||||
}
|
||||
@@ -48,10 +48,13 @@ import { appendFile } from "node:fs/promises";
|
||||
import {
|
||||
ADAPTERS,
|
||||
} from "../src/ingest/adapters/index.ts";
|
||||
import { SIX_HOURS_MS } from "../src/ingest/adapters/types.ts";
|
||||
import { SIX_HOURS_MS, type Adapter } from "../src/ingest/adapters/types.ts";
|
||||
import { CODE_SOURCES, type RefreshSource } from "../src/ingest/codes/registry.ts";
|
||||
const REFRESH_SOURCES: readonly RefreshSource[] = [...ADAPTERS, ...CODE_SOURCES];
|
||||
import { RobotsCache, type FetchLike } from "../src/ingest/robots.ts";
|
||||
import { articleTitle } from "../src/ingest/articleEnds.ts";
|
||||
import { ArticleStore } from "../src/ingest/articleStore.ts";
|
||||
import { refreshArticles } from "./refresh-articles.ts";
|
||||
import { decodeBody, SnapshotStore } from "../src/ingest/snapshots.ts";
|
||||
|
||||
const DEFAULT_CONTACT =
|
||||
@@ -92,6 +95,8 @@ export interface RefreshSummary {
|
||||
outcomes: SourceOutcome[];
|
||||
/** Sources whose stored bytes changed — the only reason to commit. */
|
||||
changed: number;
|
||||
/** Stored articles whose bytes changed. Rebuilds the feed; never a source result. */
|
||||
articlesChanged: number;
|
||||
/** Sources we actually sent a request to. */
|
||||
attempted: number;
|
||||
/** Sources that answered (200 or 304). */
|
||||
@@ -151,6 +156,11 @@ export interface RefreshOptions {
|
||||
force: boolean;
|
||||
timeoutMs: number;
|
||||
log: (line: string) => void;
|
||||
/**
|
||||
* The source whose rows have articles worth reading for an exact end. Unset
|
||||
* reads none. `--force` deliberately does not reach these (spec decision 6).
|
||||
*/
|
||||
articleSourceId?: string;
|
||||
/** Called once when something changed. Null skips the rebuild (tests). */
|
||||
rebuildFeed: (() => Promise<void>) | null;
|
||||
}
|
||||
@@ -188,6 +198,7 @@ export async function runRefresh(
|
||||
const summary: RefreshSummary = {
|
||||
outcomes: [],
|
||||
changed: 0,
|
||||
articlesChanged: 0,
|
||||
attempted: 0,
|
||||
confirmed: 0,
|
||||
warnings: [],
|
||||
@@ -288,6 +299,44 @@ export async function runRefresh(
|
||||
}
|
||||
}
|
||||
|
||||
// After every source had its turn, so an article can never cost a page one.
|
||||
const articleSource = selected.find((a) => a.id === options.articleSourceId);
|
||||
if (articleSource !== undefined) {
|
||||
try {
|
||||
const snapshot = await options.store.read(articleSource.id);
|
||||
if (snapshot !== null) {
|
||||
const articles = new ArticleStore(options.store.root);
|
||||
// With the stored ends, as the feed build does: on a row's last day they
|
||||
// are what keeps it listed, and an unlisted article gets deleted.
|
||||
const rows = (articleSource.parse as Adapter["parse"])(snapshot.html, {
|
||||
now: options.now().toISOString(),
|
||||
sourceUrl: articleSource.url,
|
||||
sourceId: articleSource.id,
|
||||
game: articleSource.game,
|
||||
articleEnds: await articles.readEnds(),
|
||||
});
|
||||
const titles = rows.flatMap((row) => articleTitle(row.sourceUrl) ?? []);
|
||||
const result = await refreshArticles({
|
||||
titles,
|
||||
store: articles,
|
||||
robots: options.robots,
|
||||
fetchImpl: options.fetchImpl,
|
||||
userAgent: options.userAgent,
|
||||
now: options.now,
|
||||
sleep: options.sleep,
|
||||
dryRun: options.dryRun,
|
||||
timeoutMs: options.timeoutMs,
|
||||
log: options.log,
|
||||
requestedHosts: cycle.requestedHosts,
|
||||
});
|
||||
summary.articlesChanged = result.changed;
|
||||
summary.warnings.push(...result.warnings);
|
||||
}
|
||||
} catch (error) {
|
||||
summary.warnings.push(`articles for ${articleSource.id}: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Named in the summary rather than only in the per-source log, so it survives
|
||||
// into the job summary and cannot be scrolled past.
|
||||
summary.forced = [...cycle.forced].sort();
|
||||
@@ -321,7 +370,10 @@ export async function runRefresh(
|
||||
return summary;
|
||||
}
|
||||
|
||||
if (summary.changed > 0 && options.rebuildFeed !== null) {
|
||||
if (
|
||||
(summary.changed > 0 || summary.articlesChanged > 0) &&
|
||||
options.rebuildFeed !== null
|
||||
) {
|
||||
try {
|
||||
await options.rebuildFeed();
|
||||
} catch (error) {
|
||||
@@ -977,6 +1029,7 @@ async function main(): Promise<number> {
|
||||
timeoutMs: 20_000,
|
||||
log: (line) => console.log(line),
|
||||
rebuildFeed: args.dryRun || !args.rebuild ? null : rebuildFeedViaScript,
|
||||
articleSourceId: "genshin-fandom-events",
|
||||
});
|
||||
|
||||
console.log(
|
||||
|
||||
@@ -20,6 +20,12 @@ interrupted run leaves a stray temp file rather than a truncated snapshot or met
|
||||
bytes that were never stored. Those temp files are gitignored: `refresh.yml` commits the whole
|
||||
directory, so otherwise a run killed mid-write would pin a half-page in git forever.
|
||||
|
||||
`articles/genshin-fandom/` holds the lead section of each Genshin event's wiki article, as the API
|
||||
served it, for the end time its infobox states (`docs/INGESTION.md` § Article ends). `<name>.html` is
|
||||
the response and `<name>.meta.json` its read history. There is no `.state.json`: the due rule has to
|
||||
survive a fresh checkout, and a failed article read is never a source's failure streak. The refresh
|
||||
removes an article once its row leaves the table.
|
||||
|
||||
Three reasons this is committed rather than cached:
|
||||
|
||||
- **Re-parsing never re-fetches.** Iterating on a parser reads these files, not the wikis
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import { mkdir, readdir, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { parseArticleEnd, type StatedEnd } from "./articleEnds.ts";
|
||||
import { hashBody, writeAtomic } from "./snapshots.ts";
|
||||
|
||||
/**
|
||||
* Stored lead sections of Genshin event articles, under the snapshot root.
|
||||
*
|
||||
* Tracked like every other snapshot, so re-parsing never re-fetches and a moved
|
||||
* end time is a reviewable diff. There is no `.state.json`: an article's read
|
||||
* history lives in its metadata, because the due rule has to survive a fresh
|
||||
* checkout and a failed read here is never a source's failure streak.
|
||||
*/
|
||||
export interface ArticleMeta {
|
||||
title: string;
|
||||
lastAttemptAt: string;
|
||||
/** The last read that worked. A failed attempt never moves this. */
|
||||
lastReadAt: string | null;
|
||||
statesTime: boolean;
|
||||
}
|
||||
|
||||
const SIX_HOURS_MS = 6 * 60 * 60 * 1000;
|
||||
const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Titles hold `/`, `:` and `'`. The hash keeps two titles that flatten alike apart.
|
||||
*/
|
||||
export function articleFileName(title: string): string {
|
||||
return `${title.replace(/[^A-Za-z0-9._-]/g, "_")}-${hashBody(title).slice(0, 8)}`;
|
||||
}
|
||||
|
||||
/** Spec decision 6: wiki editors correct these, so a kept time still gets re-read. */
|
||||
export function articleDue(meta: ArticleMeta | null, nowMs: number): boolean {
|
||||
if (meta === null) return true;
|
||||
if (nowMs - Date.parse(meta.lastAttemptAt) < SIX_HOURS_MS) return false;
|
||||
if (!meta.statesTime || meta.lastReadAt === null) return true;
|
||||
return nowMs - Date.parse(meta.lastReadAt) >= SEVEN_DAYS_MS;
|
||||
}
|
||||
|
||||
export class ArticleStore {
|
||||
private readonly dir: string;
|
||||
|
||||
constructor(root: string) {
|
||||
this.dir = join(root, "articles", "genshin-fandom");
|
||||
}
|
||||
|
||||
private bodyPath(title: string): string {
|
||||
return join(this.dir, `${articleFileName(title)}.html`);
|
||||
}
|
||||
|
||||
private metaPath(title: string): string {
|
||||
return join(this.dir, `${articleFileName(title)}.meta.json`);
|
||||
}
|
||||
|
||||
private async writeMeta(meta: ArticleMeta): Promise<void> {
|
||||
await mkdir(this.dir, { recursive: true });
|
||||
await writeAtomic(this.metaPath(meta.title), `${JSON.stringify(meta, null, 2)}\n`);
|
||||
}
|
||||
|
||||
async readMeta(title: string): Promise<ArticleMeta | null> {
|
||||
const file = Bun.file(this.metaPath(title));
|
||||
return (await file.exists()) ? ((await file.json()) as ArticleMeta) : null;
|
||||
}
|
||||
|
||||
async readBody(title: string): Promise<string | null> {
|
||||
const file = Bun.file(this.bodyPath(title));
|
||||
return (await file.exists()) ? await file.text() : null;
|
||||
}
|
||||
|
||||
/** A read that worked. Body before metadata. True when the stored bytes moved. */
|
||||
async saveRead(title: string, body: string, at: string): Promise<boolean> {
|
||||
const changed = (await this.readBody(title)) !== body;
|
||||
await mkdir(this.dir, { recursive: true });
|
||||
if (changed) await writeAtomic(this.bodyPath(title), body);
|
||||
await this.writeMeta({
|
||||
title,
|
||||
lastAttemptAt: at,
|
||||
lastReadAt: at,
|
||||
statesTime: parseArticleEnd(body) !== null,
|
||||
});
|
||||
return changed;
|
||||
}
|
||||
|
||||
/** A request that told us nothing about the article: keep what we hold. */
|
||||
async recordAttempt(title: string, at: string): Promise<void> {
|
||||
const meta = await this.readMeta(title);
|
||||
await this.writeMeta({
|
||||
title,
|
||||
lastAttemptAt: at,
|
||||
lastReadAt: meta?.lastReadAt ?? null,
|
||||
statesTime: meta?.statesTime ?? false,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The wiki says the article is gone. The metadata stays while the row is
|
||||
* listed, so the 6-hour floor still holds for the retries. True if bytes went.
|
||||
*/
|
||||
async markGone(title: string, at: string): Promise<boolean> {
|
||||
const held = (await this.readBody(title)) !== null;
|
||||
await rm(this.bodyPath(title), { force: true });
|
||||
await this.writeMeta({
|
||||
title,
|
||||
lastAttemptAt: at,
|
||||
lastReadAt: (await this.readMeta(title))?.lastReadAt ?? null,
|
||||
statesTime: false,
|
||||
});
|
||||
return held;
|
||||
}
|
||||
|
||||
/** The row left the table. True if bytes went. */
|
||||
async remove(title: string): Promise<boolean> {
|
||||
const held = (await this.readBody(title)) !== null;
|
||||
await rm(this.bodyPath(title), { force: true });
|
||||
await rm(this.metaPath(title), { force: true });
|
||||
return held;
|
||||
}
|
||||
|
||||
/** Every stored title. Empty, not an error, on a checkout with no directory. */
|
||||
async listed(): Promise<string[]> {
|
||||
let names: string[];
|
||||
try {
|
||||
names = await readdir(this.dir);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const titles: string[] = [];
|
||||
for (const name of names.filter((n) => n.endsWith(".meta.json"))) {
|
||||
const meta = (await Bun.file(join(this.dir, name)).json()) as ArticleMeta;
|
||||
titles.push(meta.title);
|
||||
}
|
||||
return titles.sort();
|
||||
}
|
||||
|
||||
/** What the feed build reads. An article stating no time contributes nothing. */
|
||||
async readEnds(): Promise<Map<string, StatedEnd>> {
|
||||
const ends = new Map<string, StatedEnd>();
|
||||
for (const title of await this.listed()) {
|
||||
const body = await this.readBody(title);
|
||||
const end = body === null ? null : parseArticleEnd(body);
|
||||
if (end !== null) ends.set(title, end);
|
||||
}
|
||||
return ends;
|
||||
}
|
||||
}
|
||||
@@ -163,7 +163,7 @@ function decodeStored(bytes: Uint8Array, charset: string | undefined): string {
|
||||
* the next refresh — sees either the whole old file or the whole new one, and
|
||||
* a crash mid-write leaves a stray temp file rather than a truncated snapshot.
|
||||
*/
|
||||
async function writeAtomic(path: string, data: Uint8Array | string): Promise<void> {
|
||||
export async function writeAtomic(path: string, data: Uint8Array | string): Promise<void> {
|
||||
const temp = `${path}.tmp-${process.pid.toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
try {
|
||||
await writeFile(temp, data);
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
import { afterEach, beforeEach, expect, test } from "bun:test";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { refreshArticles } from "../scripts/refresh-articles.ts";
|
||||
import { runRefresh, type RefreshOptions, type RobotsGate } from "../scripts/refresh-sources.ts";
|
||||
import type { Adapter, ParseContext } from "../src/ingest/adapters/types.ts";
|
||||
import { articleDue, articleFileName, ArticleStore, type ArticleMeta } from "../src/ingest/articleStore.ts";
|
||||
import { SnapshotStore } from "../src/ingest/snapshots.ts";
|
||||
import type { GachaEvent } from "../src/shared/schema.ts";
|
||||
|
||||
const NOW = new Date("2026-11-07T12:00:00.000Z");
|
||||
const HOUR = 60 * 60 * 1000;
|
||||
const ago = (ms: number) => new Date(NOW.getTime() - ms).toISOString();
|
||||
const TITLE = "Hour_of_Ruin_(Event)";
|
||||
const WITH_TIME = await Bun.file("fixtures/genshin/articles/hour-of-ruin-2026-11-07.html").text();
|
||||
const NO_TIME = JSON.stringify({ parse: { text: "<p>A stub.</p>" } });
|
||||
const MISSING = JSON.stringify({ error: { code: "missingtitle", info: "The page does not exist." } });
|
||||
|
||||
let root: string;
|
||||
let articles: ArticleStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "event-clock-articles-"));
|
||||
articles = new ArticleStore(root);
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const ALLOW_ALL: RobotsGate = { allows: async () => ({ allowed: true, reason: "ok" }) };
|
||||
|
||||
interface Call { url: string; userAgent: string | undefined }
|
||||
|
||||
function run(
|
||||
respond: () => Response | Promise<Response>,
|
||||
over: Partial<Parameters<typeof refreshArticles>[0]> = {},
|
||||
) {
|
||||
const calls: Call[] = [];
|
||||
const naps: number[] = [];
|
||||
const logs: string[] = [];
|
||||
const done = refreshArticles({
|
||||
titles: [TITLE],
|
||||
store: articles,
|
||||
robots: ALLOW_ALL,
|
||||
fetchImpl: async (url, init) => {
|
||||
calls.push({ url, userAgent: (init?.headers as Record<string, string>)["User-Agent"] });
|
||||
return respond();
|
||||
},
|
||||
userAgent: "test-agent",
|
||||
now: () => NOW,
|
||||
sleep: async (ms) => void naps.push(ms),
|
||||
dryRun: false,
|
||||
timeoutMs: 1000,
|
||||
log: (l) => void logs.push(l),
|
||||
requestedHosts: new Set(),
|
||||
...over,
|
||||
});
|
||||
return { done, calls, naps, logs };
|
||||
}
|
||||
|
||||
const meta = (over: Partial<ArticleMeta>): ArticleMeta => ({
|
||||
title: TITLE,
|
||||
lastAttemptAt: ago(30 * 24 * HOUR),
|
||||
lastReadAt: ago(30 * 24 * HOUR),
|
||||
statesTime: true,
|
||||
...over,
|
||||
});
|
||||
const at = NOW.getTime();
|
||||
|
||||
test("due: never read", () => {
|
||||
expect(articleDue(null, at)).toBe(true);
|
||||
});
|
||||
|
||||
test("due: no time found, only after 6 hours", () => {
|
||||
const m = (h: number) => meta({ lastAttemptAt: ago(h * HOUR), lastReadAt: ago(h * HOUR), statesTime: false });
|
||||
expect(articleDue(m(5.9), at)).toBe(false);
|
||||
expect(articleDue(m(6), at)).toBe(true);
|
||||
});
|
||||
|
||||
test("due: time found, only 7 days after the last read that worked", () => {
|
||||
const m = (d: number) => meta({ lastAttemptAt: ago(d * 24 * HOUR), lastReadAt: ago(d * 24 * HOUR) });
|
||||
expect(articleDue(m(6.9), at)).toBe(false);
|
||||
expect(articleDue(m(7), at)).toBe(true);
|
||||
});
|
||||
|
||||
test("due: a failed attempt inside 6 hours is not due, even past 7 days", () => {
|
||||
expect(articleDue(meta({ lastAttemptAt: ago(1 * HOUR) }), at)).toBe(false);
|
||||
expect(articleDue(meta({ lastAttemptAt: ago(6 * HOUR) }), at)).toBe(true);
|
||||
});
|
||||
|
||||
test("a percent-decoded title gets one stable file name that keeps titles apart", () => {
|
||||
const name = articleFileName("The_Forge_Realm's_Temper/2026-11-04");
|
||||
expect(name).toMatch(/^[A-Za-z0-9._-]+$/);
|
||||
expect(name).toBe(articleFileName("The_Forge_Realm's_Temper/2026-11-04"));
|
||||
expect(articleFileName("a/b")).not.toBe(articleFileName("a:b"));
|
||||
});
|
||||
|
||||
test("a first read stores the article and its end", async () => {
|
||||
const { done } = run(() => new Response(WITH_TIME));
|
||||
expect((await done).changed).toBe(1);
|
||||
expect((await articles.readEnds()).get(TITLE)?.day).toBe("2026-12-15");
|
||||
expect(await articles.readMeta(TITLE)).toEqual({
|
||||
title: TITLE,
|
||||
lastAttemptAt: NOW.toISOString(),
|
||||
lastReadAt: NOW.toISOString(),
|
||||
statesTime: true,
|
||||
});
|
||||
});
|
||||
|
||||
/** A stored article whose 7-day re-read is due now. */
|
||||
async function dueArticle() {
|
||||
await articles.saveRead(TITLE, WITH_TIME, ago(8 * 24 * HOUR));
|
||||
}
|
||||
|
||||
async function expectKept(result: { changed: number; warnings: string[] }) {
|
||||
expect(result.changed).toBe(0);
|
||||
expect(result.warnings.length).toBe(1);
|
||||
expect(await articles.readBody(TITLE)).toBe(WITH_TIME);
|
||||
expect((await articles.readEnds()).has(TITLE)).toBe(true);
|
||||
const m = await articles.readMeta(TITLE);
|
||||
expect(m?.lastReadAt).toBe(ago(8 * 24 * HOUR));
|
||||
expect(m?.lastAttemptAt).toBe(NOW.toISOString());
|
||||
}
|
||||
|
||||
test.each([500, 403, 202])("HTTP %i keeps the body and the time, and warns", async (status) => {
|
||||
await dueArticle();
|
||||
await expectKept(await run(() => new Response("nope", { status })).done);
|
||||
});
|
||||
|
||||
test("a network error keeps the body and the time, and warns", async () => {
|
||||
await dueArticle();
|
||||
await expectKept(await run(() => { throw new Error("reset"); }).done);
|
||||
});
|
||||
|
||||
test("a body that fails mid-read keeps the body and the time, and warns", async () => {
|
||||
await dueArticle();
|
||||
await expectKept(await run(() => ({
|
||||
status: 200,
|
||||
arrayBuffer: () => Promise.reject(new Error("truncated")),
|
||||
text: () => Promise.reject(new Error("truncated")),
|
||||
}) as unknown as Response).done);
|
||||
});
|
||||
|
||||
test("a 200 that is not the API's answer keeps the body and the time, and warns", async () => {
|
||||
await dueArticle();
|
||||
await expectKept(await run(() => new Response("<html>Just a moment…</html>")).done);
|
||||
});
|
||||
|
||||
test("a 200 API error other than a missing page keeps the body and the time", async () => {
|
||||
await dueArticle();
|
||||
await expectKept(await run(() => new Response(JSON.stringify({ error: { code: "ratelimited" } }))).done);
|
||||
});
|
||||
|
||||
test("robots refusal makes no request, keeps the body, and warns", async () => {
|
||||
await dueArticle();
|
||||
const robots: RobotsGate = { allows: async () => ({ allowed: false, reason: "disallowed" }) };
|
||||
const { done, calls } = run(() => new Response(WITH_TIME), { robots });
|
||||
await expectKept(await done);
|
||||
expect(calls.length).toBe(0);
|
||||
});
|
||||
|
||||
test.each([
|
||||
["404", () => new Response("gone", { status: 404 })],
|
||||
["a missingtitle API error", () => new Response(MISSING)],
|
||||
])("%s removes the stored article and warns", async (_name, respond) => {
|
||||
await dueArticle();
|
||||
const result = await run(respond).done;
|
||||
expect(result.changed).toBe(1);
|
||||
expect(result.warnings.length).toBe(1);
|
||||
expect(await articles.readBody(TITLE)).toBeNull();
|
||||
expect((await articles.readEnds()).has(TITLE)).toBe(false);
|
||||
});
|
||||
|
||||
test("a removed article still honours the 6-hour floor", async () => {
|
||||
await dueArticle();
|
||||
await run(() => new Response("gone", { status: 404 })).done;
|
||||
const { done, calls } = run(() => new Response(WITH_TIME));
|
||||
await done;
|
||||
expect(calls.length).toBe(0);
|
||||
});
|
||||
|
||||
test("a 200 with no usable End overwrites the body, sets statesTime false, and logs without warning", async () => {
|
||||
await dueArticle();
|
||||
const { done, logs } = run(() => new Response(NO_TIME));
|
||||
const result = await done;
|
||||
expect(result.changed).toBe(1);
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(logs.join("\n")).toContain(TITLE);
|
||||
expect(await articles.readBody(TITLE)).toBe(NO_TIME);
|
||||
expect((await articles.readMeta(TITLE))?.statesTime).toBe(false);
|
||||
expect((await articles.readEnds()).has(TITLE)).toBe(false);
|
||||
});
|
||||
|
||||
test("an unchanged re-read moves lastReadAt but changes no bytes", async () => {
|
||||
await dueArticle();
|
||||
const result = await run(() => new Response(WITH_TIME)).done;
|
||||
expect(result.changed).toBe(0);
|
||||
expect((await articles.readMeta(TITLE))?.lastReadAt).toBe(NOW.toISOString());
|
||||
});
|
||||
|
||||
test("an article not yet due makes no request", async () => {
|
||||
await articles.saveRead(TITLE, WITH_TIME, ago(24 * HOUR));
|
||||
const { done, calls } = run(() => new Response(WITH_TIME));
|
||||
await done;
|
||||
expect(calls.length).toBe(0);
|
||||
});
|
||||
|
||||
test("the requested URL carries the decoded title, section=0, and the runner User-Agent", async () => {
|
||||
const title = "The_Forge_Realm's_Temper/2026-11-04";
|
||||
const { done, calls } = run(() => new Response(WITH_TIME), { titles: [title] });
|
||||
await done;
|
||||
const url = new URL(calls[0]!.url);
|
||||
expect(url.host).toBe("genshin-impact.fandom.com");
|
||||
expect(url.pathname).toBe("/api.php");
|
||||
expect(url.searchParams.get("page")).toBe(title);
|
||||
expect(url.searchParams.get("action")).toBe("parse");
|
||||
expect(url.searchParams.get("prop")).toBe("text");
|
||||
expect(url.searchParams.get("section")).toBe("0");
|
||||
expect(calls[0]!.userAgent).toBe("test-agent");
|
||||
});
|
||||
|
||||
test("the same title twice is one request", async () => {
|
||||
const { done, calls } = run(() => new Response(WITH_TIME), { titles: [TITLE, TITLE] });
|
||||
await done;
|
||||
expect(calls.length).toBe(1);
|
||||
});
|
||||
|
||||
test("per-host spacing applies when the host was already asked this cycle", async () => {
|
||||
const { done, naps } = run(() => new Response(WITH_TIME), {
|
||||
titles: ["A", "B"],
|
||||
requestedHosts: new Set(["genshin-impact.fandom.com"]),
|
||||
});
|
||||
await done;
|
||||
expect(naps).toEqual([2000, 2000]);
|
||||
});
|
||||
|
||||
test("a dry run requests and writes nothing", async () => {
|
||||
await articles.saveRead("Old", NO_TIME, ago(24 * HOUR));
|
||||
const { done, calls, logs } = run(() => new Response(WITH_TIME), { dryRun: true });
|
||||
expect((await done).changed).toBe(0);
|
||||
expect(calls.length).toBe(0);
|
||||
expect(await articles.readBody(TITLE)).toBeNull();
|
||||
expect(await articles.readBody("Old")).toBe(NO_TIME);
|
||||
expect(logs.join("\n")).toContain(TITLE);
|
||||
});
|
||||
|
||||
test("a stored article no longer listed is removed", async () => {
|
||||
await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR));
|
||||
const result = await run(() => new Response(WITH_TIME)).done;
|
||||
expect(await articles.listed()).toEqual([TITLE]);
|
||||
expect(result.changed).toBe(2);
|
||||
});
|
||||
|
||||
test("no listed titles removes nothing", async () => {
|
||||
await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR));
|
||||
expect((await run(() => new Response(WITH_TIME), { titles: [] }).done).changed).toBe(0);
|
||||
expect(await articles.listed()).toEqual(["Old"]);
|
||||
});
|
||||
|
||||
test("no directory reads as no ends", async () => {
|
||||
expect((await articles.readEnds()).size).toBe(0);
|
||||
expect(await articles.listed()).toEqual([]);
|
||||
});
|
||||
|
||||
// Through the runner. The stand-in parser reads `<row>url</row>` as listed rows.
|
||||
const FANDOM_ID = "genshin-fandom-events";
|
||||
function fandomAdapter(): Adapter {
|
||||
return {
|
||||
id: FANDOM_ID,
|
||||
game: "genshin",
|
||||
url: "https://genshin-impact.fandom.com/api.php?action=parse&page=Event",
|
||||
parserId: "fandom",
|
||||
minIntervalMs: 6 * HOUR,
|
||||
priority: 0,
|
||||
parse(html: string, _ctx: ParseContext): GachaEvent[] {
|
||||
return [...html.matchAll(/<row>(.*?)<\/row>/g)].map((m) => ({ sourceUrl: m[1] }) as GachaEvent);
|
||||
},
|
||||
};
|
||||
}
|
||||
const ROW = `https://genshin-impact.fandom.com/wiki/${TITLE}`;
|
||||
|
||||
function runner(page: string, respondArticle: () => Response, over: Partial<RefreshOptions> = {}) {
|
||||
const urls: string[] = [];
|
||||
const rebuilds = { count: 0 };
|
||||
const store = new SnapshotStore(root);
|
||||
const opts: RefreshOptions = {
|
||||
adapters: [fandomAdapter()],
|
||||
store,
|
||||
robots: ALLOW_ALL,
|
||||
fetchImpl: async (url) => {
|
||||
urls.push(url);
|
||||
return url.includes("page=Event") ? new Response(page) : respondArticle();
|
||||
},
|
||||
userAgent: "test-agent",
|
||||
now: () => NOW,
|
||||
sleep: async () => {},
|
||||
dryRun: false,
|
||||
only: null,
|
||||
force: false,
|
||||
timeoutMs: 1000,
|
||||
log: () => {},
|
||||
rebuildFeed: async () => {
|
||||
rebuilds.count += 1;
|
||||
},
|
||||
articleSourceId: FANDOM_ID,
|
||||
...over,
|
||||
};
|
||||
return { opts, urls, rebuilds, store };
|
||||
}
|
||||
|
||||
test("article reads run only with articleSourceId set; existing callers are unaffected", async () => {
|
||||
const { opts, urls } = runner(`<row>${ROW}</row>`, () => new Response(WITH_TIME));
|
||||
const { articleSourceId: _omit, ...without } = opts;
|
||||
await runRefresh(without);
|
||||
expect(urls.length).toBe(1);
|
||||
});
|
||||
|
||||
test("a listed row's article is read and stored", async () => {
|
||||
const { opts, urls } = runner(`<row>${ROW}</row>`, () => new Response(WITH_TIME));
|
||||
const summary = await runRefresh(opts);
|
||||
expect(urls.length).toBe(2);
|
||||
expect(summary.articlesChanged).toBe(1);
|
||||
expect((await articles.readEnds()).has(TITLE)).toBe(true);
|
||||
});
|
||||
|
||||
test("a row with no article link makes no article request", async () => {
|
||||
const { opts, urls } = runner(`<row>${fandomAdapter().url}</row>`, () => new Response(WITH_TIME));
|
||||
await runRefresh(opts);
|
||||
expect(urls.length).toBe(1);
|
||||
});
|
||||
|
||||
test("articles are read when --only names another source not at all", async () => {
|
||||
const other: Adapter = { ...fandomAdapter(), id: "other", url: "https://example.test/x" };
|
||||
const { opts, urls } = runner(`<row>${ROW}</row>`, () => new Response(WITH_TIME), {
|
||||
adapters: [fandomAdapter(), other],
|
||||
only: "other",
|
||||
});
|
||||
await runRefresh(opts);
|
||||
expect(urls.some((u) => u.includes("Hour_of_Ruin"))).toBe(false);
|
||||
});
|
||||
|
||||
test("an empty source snapshot removes nothing and does not throw", async () => {
|
||||
await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR));
|
||||
const { opts, store } = runner("<html></html>", () => new Response(WITH_TIME));
|
||||
await store.save(FANDOM_ID, {
|
||||
url: opts.adapters[0]!.url, body: "<html></html>", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 0,
|
||||
});
|
||||
const summary = await runRefresh({ ...opts, fetchImpl: async () => new Response("", { status: 304 }) });
|
||||
expect(summary.hardFailure).toBeNull();
|
||||
expect(await articles.listed()).toEqual(["Old"]);
|
||||
});
|
||||
|
||||
test("article failures leave broken, hardFailure and the source's streak untouched", async () => {
|
||||
const { opts, store } = runner(`<row>${ROW}</row>`, () => new Response("no", { status: 500 }));
|
||||
const summary = await runRefresh(opts);
|
||||
expect(summary.hardFailure).toBeNull();
|
||||
expect(summary.broken).toEqual([]);
|
||||
expect(summary.warnings.some((w) => w.includes(TITLE))).toBe(true);
|
||||
expect((await store.readState(FANDOM_ID)).consecutiveFailures).toBe(0);
|
||||
});
|
||||
|
||||
test("a parser that throws on the snapshot is one warning, not a failed run", async () => {
|
||||
const adapter: Adapter = {
|
||||
...fandomAdapter(),
|
||||
parse: (html) => {
|
||||
if (html.includes("<boom>")) throw new Error("template");
|
||||
return [{ sourceUrl: ROW } as GachaEvent];
|
||||
},
|
||||
};
|
||||
const { opts, store } = runner("<row>x</row>", () => new Response(WITH_TIME), { adapters: [adapter] });
|
||||
await store.save(FANDOM_ID, {
|
||||
url: adapter.url, body: "<boom>", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1,
|
||||
});
|
||||
const skipFetch = { ...opts, fetchImpl: async () => new Response("", { status: 304 }) };
|
||||
const summary = await runRefresh(skipFetch);
|
||||
expect(summary.hardFailure).toBeNull();
|
||||
expect(summary.warnings.some((w) => w.includes("article"))).toBe(true);
|
||||
});
|
||||
|
||||
test("a changed article triggers the rebuild even when no source page changed", async () => {
|
||||
const page = `<row>${ROW}</row>`;
|
||||
const { opts, rebuilds, store } = runner(page, () => new Response(WITH_TIME));
|
||||
await store.save(FANDOM_ID, {
|
||||
url: opts.adapters[0]!.url, body: page, etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1,
|
||||
});
|
||||
const summary = await runRefresh(opts);
|
||||
expect(summary.changed).toBe(0);
|
||||
expect(summary.articlesChanged).toBe(1);
|
||||
expect(rebuilds.count).toBe(1);
|
||||
});
|
||||
|
||||
test("a row kept alive by its stated end stays listed, so its article is not removed", async () => {
|
||||
// On a row's last day the parser only emits it because of the stored end. Read
|
||||
// the table without those ends and the article looks unlisted and gets deleted,
|
||||
// and the row vanishes from the feed hours early.
|
||||
await articles.saveRead(TITLE, WITH_TIME, ago(24 * HOUR));
|
||||
const adapter: Adapter = {
|
||||
...fandomAdapter(),
|
||||
parse: (_html, ctx) => (ctx.articleEnds?.has(TITLE) ? [{ sourceUrl: ROW } as GachaEvent] : []),
|
||||
};
|
||||
const { opts } = runner("<html></html>", () => new Response(WITH_TIME), { adapters: [adapter] });
|
||||
await opts.store.save(FANDOM_ID, {
|
||||
url: adapter.url, body: "<row>", etag: null, lastModified: null, at: ago(24 * HOUR), eventCount: 1,
|
||||
});
|
||||
await articles.saveRead("Old", WITH_TIME, ago(24 * HOUR));
|
||||
await runRefresh({ ...opts, fetchImpl: async () => new Response("", { status: 304 }) });
|
||||
expect(await articles.listed()).toEqual([TITLE]);
|
||||
});
|
||||
@@ -973,6 +973,7 @@ describe("what the runner reports to the runner", () => {
|
||||
},
|
||||
],
|
||||
changed: 0,
|
||||
articlesChanged: 0,
|
||||
attempted: 1,
|
||||
confirmed: 0,
|
||||
warnings: ["genshin-game8-events: HTTP 403 (cloudflare, cf-ray)"],
|
||||
|
||||
Reference in New Issue
Block a user