feat(refresh): keep Genshin event articles beside the snapshots

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/refresh-articles.ts

@@ -0,0 +1,127 @@
+/**
+ * Keep Genshin event articles beside the snapshots (spec decisions 5, 6 and 12).
+ *
+ * Nothing here can fail a refresh or touch a source's failure streak: an article
+ * only ever improves a row, and a row without one is exactly what it was. What a
+ * failed read does to a time already held depends on what the wiki actually said.
+ */
+import { parseArticleEnd } from "../src/ingest/articleEnds.ts";
+import { articleDue, type ArticleStore } from "../src/ingest/articleStore.ts";
+import type { FetchLike } from "../src/ingest/robots.ts";
+import type { RobotsGate } from "./refresh-sources.ts";
+
+const HOST = "genshin-impact.fandom.com";
+const DEFAULT_HOST_GAP_MS = 2_000;
+
+export interface RefreshArticlesInput {
+  titles: readonly string[];
+  store: ArticleStore;
+  robots: RobotsGate;
+  fetchImpl: FetchLike;
+  userAgent: string;
+  now: () => Date;
+  sleep: (ms: number) => Promise<void>;
+  dryRun: boolean;
+  timeoutMs: number;
+  log: (line: string) => void;
+  /** Shared with the source loop, which has usually asked this host already. */
+  requestedHosts: Set<string>;
+}
+
+export function articleUrl(title: string): string {
+  return (
+    `https://${HOST}/api.php?action=parse&page=${encodeURIComponent(title)}` +
+    "&prop=text&section=0&formatversion=2&format=json"
+  );
+}
+
+/**
+ * A 200 is not yet an answer: a challenge page is a 200 too, and reading one as
+ * "the article states no time" would throw away a good time on a passing blip.
+ */
+function answerOf(body: string): "article" | "gone" | "unusable" {
+  try {
+    const parsed = JSON.parse(body) as {
+      parse?: { text?: unknown };
+      error?: { code?: unknown };
+    } | null;
+    if (typeof parsed?.parse?.text === "string") return "article";
+    if (parsed?.error?.code === "missingtitle") return "gone";
+  } catch {
+    // Not JSON, so not the API's answer.
+  }
+  return "unusable";
+}
+
+export async function refreshArticles(
+  input: RefreshArticlesInput,
+): Promise<{ changed: number; warnings: string[] }> {
+  const { store, log } = input;
+  const titles = [...new Set(input.titles)];
+  const nowIso = input.now().toISOString();
+  const nowMs = input.now().getTime();
+  const warnings: string[] = [];
+  let changed = 0;
+
+  for (const title of titles) {
+    const label = `article ${title}`;
+    if (!articleDue(await store.readMeta(title), nowMs)) continue;
+    if (input.dryRun) {
+      log(`  ${label}: would read`);
+      continue;
+    }
+
+    const url = articleUrl(title);
+    const failed = async (why: string) => {
+      await store.recordAttempt(title, nowIso);
+      warnings.push(`${label}: ${why}; keeping what is stored`);
+    };
+
+    const decision = await input.robots.allows(url);
+    if (!decision.allowed) {
+      await failed(decision.reason);
+      continue;
+    }
+    if (input.requestedHosts.has(HOST)) {
+      await input.sleep(decision.crawlDelayMs ?? DEFAULT_HOST_GAP_MS);
+    }
+    input.requestedHosts.add(HOST);
+
+    let status: number;
+    let body: string;
+    try {
+      const response = await input.fetchImpl(url, {
+        headers: { "User-Agent": input.userAgent, Accept: "application/json" },
+        signal: AbortSignal.timeout(input.timeoutMs),
+        redirect: "follow",
+      });
+      status = response.status;
+      body = status === 200 ? await response.text() : "";
+    } catch (error) {
+      await failed(`unreachable: ${String(error)}`);
+      continue;
+    }
+
+    const answer = status === 200 ? answerOf(body) : status === 404 ? "gone" : "unusable";
+    if (answer === "gone") {
+      if (await store.markGone(title, nowIso)) changed += 1;
+      warnings.push(`${label}: the wiki says it does not exist`);
+    } else if (answer === "unusable") {
+      await failed(status === 200 ? "the answer was not the API's" : `HTTP ${status}`);
+    } else {
+      if (await store.saveRead(title, body, nowIso)) changed += 1;
+      // A stub is ordinary and a warning per stub per cycle would bury real ones.
+      if (parseArticleEnd(body) === null) log(`  ${label}: states no time yet`);
+    }
+  }
+
+  // No titles means the table failed to parse, not that every event ended.
+  if (titles.length > 0 && !input.dryRun) {
+    const listed = new Set(titles);
+    for (const title of await store.listed()) {
+      if (!listed.has(title) && (await store.remove(title))) changed += 1;
+    }
+  }
+
+  return { changed, warnings };
+}

scripts/refresh-sources.ts

@@ -48,10 +48,13 @@ import { appendFile } from "node:fs/promises";
 import {
   ADAPTERS,
 } from "../src/ingest/adapters/index.ts";
-import { SIX_HOURS_MS } from "../src/ingest/adapters/types.ts";
+import { SIX_HOURS_MS, type Adapter } from "../src/ingest/adapters/types.ts";
 import { CODE_SOURCES, type RefreshSource } from "../src/ingest/codes/registry.ts";
 const REFRESH_SOURCES: readonly RefreshSource[] = [...ADAPTERS, ...CODE_SOURCES];
 import { RobotsCache, type FetchLike } from "../src/ingest/robots.ts";
+import { articleTitle } from "../src/ingest/articleEnds.ts";
+import { ArticleStore } from "../src/ingest/articleStore.ts";
+import { refreshArticles } from "./refresh-articles.ts";
 import { decodeBody, SnapshotStore } from "../src/ingest/snapshots.ts";
 
 const DEFAULT_CONTACT =
@@ -92,6 +95,8 @@ export interface RefreshSummary {
   outcomes: SourceOutcome[];
   /** Sources whose stored bytes changed — the only reason to commit. */
   changed: number;
+  /** Stored articles whose bytes changed. Rebuilds the feed; never a source result. */
+  articlesChanged: number;
   /** Sources we actually sent a request to. */
   attempted: number;
   /** Sources that answered (200 or 304). */
@@ -151,6 +156,11 @@ export interface RefreshOptions {
   force: boolean;
   timeoutMs: number;
   log: (line: string) => void;
+  /**
+   * The source whose rows have articles worth reading for an exact end. Unset
+   * reads none. `--force` deliberately does not reach these (spec decision 6).
+   */
+  articleSourceId?: string;
   /** Called once when something changed. Null skips the rebuild (tests). */
   rebuildFeed: (() => Promise<void>) | null;
 }
@@ -188,6 +198,7 @@ export async function runRefresh(
   const summary: RefreshSummary = {
     outcomes: [],
     changed: 0,
+    articlesChanged: 0,
     attempted: 0,
     confirmed: 0,
     warnings: [],
@@ -288,6 +299,44 @@ export async function runRefresh(
     }
   }
 
+  // After every source had its turn, so an article can never cost a page one.
+  const articleSource = selected.find((a) => a.id === options.articleSourceId);
+  if (articleSource !== undefined) {
+    try {
+      const snapshot = await options.store.read(articleSource.id);
+      if (snapshot !== null) {
+        const articles = new ArticleStore(options.store.root);
+        // With the stored ends, as the feed build does: on a row's last day they
+        // are what keeps it listed, and an unlisted article gets deleted.
+        const rows = (articleSource.parse as Adapter["parse"])(snapshot.html, {
+          now: options.now().toISOString(),
+          sourceUrl: articleSource.url,
+          sourceId: articleSource.id,
+          game: articleSource.game,
+          articleEnds: await articles.readEnds(),
+        });
+        const titles = rows.flatMap((row) => articleTitle(row.sourceUrl) ?? []);
+        const result = await refreshArticles({
+          titles,
+          store: articles,
+          robots: options.robots,
+          fetchImpl: options.fetchImpl,
+          userAgent: options.userAgent,
+          now: options.now,
+          sleep: options.sleep,
+          dryRun: options.dryRun,
+          timeoutMs: options.timeoutMs,
+          log: options.log,
+          requestedHosts: cycle.requestedHosts,
+        });
+        summary.articlesChanged = result.changed;
+        summary.warnings.push(...result.warnings);
+      }
+    } catch (error) {
+      summary.warnings.push(`articles for ${articleSource.id}: ${String(error)}`);
+    }
+  }
+
   // Named in the summary rather than only in the per-source log, so it survives
   // into the job summary and cannot be scrolled past.
   summary.forced = [...cycle.forced].sort();
@@ -321,7 +370,10 @@ export async function runRefresh(
     return summary;
   }
 
-  if (summary.changed > 0 && options.rebuildFeed !== null) {
+  if (
+    (summary.changed > 0 || summary.articlesChanged > 0) &&
+    options.rebuildFeed !== null
+  ) {
     try {
       await options.rebuildFeed();
     } catch (error) {
@@ -977,6 +1029,7 @@ async function main(): Promise<number> {
     timeoutMs: 20_000,
     log: (line) => console.log(line),
     rebuildFeed: args.dryRun || !args.rebuild ? null : rebuildFeedViaScript,
+    articleSourceId: "genshin-fandom-events",
   });
 
   console.log(