diff --git a/AGENTS.md b/AGENTS.md index 2ac8b00..84ecdd3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -314,7 +314,8 @@ These come from how gacha games actually schedule things, and they cause most bu page and an `h4` on Wuthering Waves'. `dedupe` blends it onto the better-dated copy and never blends a date, so reading both shapes adds information and can never contradict any. - **Prefer a source that states machine-readable times.** wiki.gg emits ISO timestamps with a timer - per server region, which is the only reason `regionEnds` carries real data anywhere. + per server region. Genshin's articles are the one other place `regionEnds` carries real data, and + those state a server clock rather than a timer (§ Domain rules). - **Silent drops are the dangerous failure.** A date format the parser does not recognise makes events vanish with no error. Abbreviated months (`Apr. 29 - May 13, 2026`) and slash-dated open ends (`09/02/2026 - TBA`) are supported for exactly this reason. When adding a source, compare the diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f61f5d8..50855af 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -134,9 +134,11 @@ scripts/ build-feed.ts fixtures → public/data/events.v1.json ✓ built build-static.ts shell + worker into public/, build-stamped ✓ built parse-fixture.ts run one adapter offline ✓ built - refresh-sources.ts fetch, cache, rebuild — the only network ✓ built + refresh-sources.ts fetch, cache, rebuild ✓ built + refresh-articles.ts Genshin event articles, for exact ends ✓ built fixtures// checked-in raw HTML + expected parse output snapshots/ the current page per source, rewritten by refresh + articles/ Genshin event articles' lead sections, rewritten by refresh ``` ## Request paths @@ -214,8 +216,8 @@ changing. Reads are confined to `public/` by resolving the path and checking it — string-matching `..` is not enough, because encodings and URL normalisation both change what the string looks like. -`scripts/refresh-sources.ts` (`bun run refresh`) is what fills `snapshots/`, and it is the only code -here that touches the network. `.gitea/workflows/refresh.yml` runs it at 05:27 and 17:27 UTC and +`scripts/refresh-sources.ts` (`bun run refresh`) is what fills `snapshots/`, and with +`scripts/refresh-articles.ts`, which it calls, it is the only code here that touches the network. `.gitea/workflows/refresh.yml` runs it at 05:27 and 17:27 UTC and commits only when a page actually changed. It stands in for the unbuilt scheduler and enforces the same conduct in code — the 6h floor, one request, no retries, conditional headers, per-host spacing, robots failing closed. A source that has failed `BROKEN_AFTER_FAILURES` (3) cycles running is diff --git a/docs/INGESTION.md b/docs/INGESTION.md index a864ed3..7b5059f 100644 --- a/docs/INGESTION.md +++ b/docs/INGESTION.md @@ -754,11 +754,13 @@ keeps it listed, and an unlisted article is deleted. | Result of a read | Stored article | Row shows | |---|---|---| -| Network error, non-200 other than 404, robots refusal, or a 200 that is not the API's answer | Kept | The last good time, if any | -| 404 or `missingtitle` | Body deleted; metadata kept for the 6-hour floor | Fallback | +| Network error, any non-200 including 404, robots refusal, or a 200 that is not the API's answer | Kept | The last good time, if any | +| The API's `missingtitle`, which it sends with a 200 | Body deleted; metadata kept for the 6-hour floor | Fallback | | A real article with no usable End | Overwritten | Fallback, logged but not warned: stubs are ordinary | -None of these touches a source's failure streak, health or broken count. +None of these touches a source's failure streak, health or broken count. A read on an assumed robots +permission names its host in the run's warning like any page does, even when the table itself was +skipped for its interval. **Parse.** `parseArticleEnd` reads only the `` cell, in two shapes: `December 15, 2026 14:59 (Server Time)` gives `regionEnds` at each server's clock, with `endsAt` the diff --git a/docs/superpowers/specs/2026-11-07-exact-end-times-design.md b/docs/superpowers/specs/2026-11-07-exact-end-times-design.md index 05ca034..3a9ca01 100644 --- a/docs/superpowers/specs/2026-11-07-exact-end-times-design.md +++ b/docs/superpowers/specs/2026-11-07-exact-end-times-design.md @@ -121,8 +121,8 @@ Approved with this document unless marked as the agent's call. | Result of a read | Stored article | Row shows | |---|---|---| - | Network error, 5xx, refused, or robots-blocked: the wiki told us nothing about the article | Kept untouched | The last good time, if any. Due again after the 6-hour floor | - | 404 or a missing-page API error: the article is gone or renamed | Deleted | Fallback | + | Network error, any non-200 including 404, refused, or robots-blocked: the wiki told us nothing about the article | Kept untouched | The last good time, if any. Due again after the 6-hour floor | + | The API's `missingtitle`, sent with a 200: the article is gone or renamed. Changed from "404" in final review: a bare 404 is an edge or outage, and one would wipe every row's time | Deleted | Fallback | | `200` whose End value is absent or unusable: the article no longer states a time | Overwritten | Fallback. Due again under the no-time rule | `.meta.json` records the last attempt separately from the last successful read, so the diff --git a/scripts/refresh-articles.ts b/scripts/refresh-articles.ts index f5fe4e3..f5f8472 100644 --- a/scripts/refresh-articles.ts +++ b/scripts/refresh-articles.ts @@ -26,6 +26,12 @@ export interface RefreshArticlesInput { log: (line: string) => void; /** Shared with the source loop, which has usually asked this host already. */ requestedHosts: Set; + /** + * Hosts fetched on a recorded robots permission, shared with the source loop. + * The page may have been skipped this cycle while an article is still read, and + * that warning must not become conditional (AGENTS.md § Scraping conduct). + */ + assumedRobots: Set; } export function articleUrl(title: string): string { @@ -82,6 +88,7 @@ export async function refreshArticles( await failed(decision.reason); continue; } + if (decision.assumedOnForbidden === true) input.assumedRobots.add(HOST); if (input.requestedHosts.has(HOST)) { await input.sleep(decision.crawlDelayMs ?? DEFAULT_HOST_GAP_MS); } @@ -102,7 +109,9 @@ export async function refreshArticles( continue; } - const answer = status === 200 ? answerOf(body) : status === 404 ? "gone" : "unusable"; + // Only the API's own `missingtitle` means gone. It says so with a 200, so a + // bare 404 is an edge or an outage, and must not cost every row its time. + const answer = status === 200 ? answerOf(body) : "unusable"; if (answer === "gone") { if (await store.markGone(title, nowIso)) changed += 1; warnings.push(`${label}: the wiki says it does not exist`); diff --git a/scripts/refresh-sources.ts b/scripts/refresh-sources.ts index 3728731..36dab33 100644 --- a/scripts/refresh-sources.ts +++ b/scripts/refresh-sources.ts @@ -328,6 +328,7 @@ export async function runRefresh( timeoutMs: options.timeoutMs, log: options.log, requestedHosts: cycle.requestedHosts, + assumedRobots: cycle.assumedRobots, }); summary.articlesChanged = result.changed; summary.warnings.push(...result.warnings); diff --git a/src/shared/time.ts b/src/shared/time.ts index 5377f1b..a9a293c 100644 --- a/src/shared/time.ts +++ b/src/shared/time.ts @@ -126,9 +126,9 @@ export type EndBearing = Pick< * precision that boundary was actually stated to. * * A `regionEnds` value is read as `exact` whatever `endPrecision` says about the - * fallback: the map only exists because the source published a timer per server - * (wiki.gg does; see AGENTS.md § Working on parsers), so there is a real instant - * in it and nothing left to resolve. + * fallback: the map only exists because the source stated a time per server + * (wiki.gg's timers, Genshin's articles; see AGENTS.md § Working on parsers), so + * there is a real instant in it and nothing left to resolve. */ function endBoundary( event: EndBearing, diff --git a/test/article-refresh.test.ts b/test/article-refresh.test.ts index 90fc051..0b7167b 100644 --- a/test/article-refresh.test.ts +++ b/test/article-refresh.test.ts @@ -54,6 +54,7 @@ function run( timeoutMs: 1000, log: (l) => void logs.push(l), requestedHosts: new Set(), + assumedRobots: new Set(), ...over, }); return { done, calls, naps, logs }; @@ -123,7 +124,7 @@ async function expectKept(result: { changed: number; warnings: string[] }) { expect(m?.lastAttemptAt).toBe(NOW.toISOString()); } -test.each([500, 403, 202])("HTTP %i keeps the body and the time, and warns", async (status) => { +test.each([500, 403, 202, 404])("HTTP %i keeps the body and the time, and warns", async (status) => { await dueArticle(); await expectKept(await run(() => new Response("nope", { status })).done); }); @@ -160,12 +161,9 @@ test("robots refusal makes no request, keeps the body, and warns", async () => { expect(calls.length).toBe(0); }); -test.each([ - ["404", () => new Response("gone", { status: 404 })], - ["a missingtitle API error", () => new Response(MISSING)], -])("%s removes the stored article and warns", async (_name, respond) => { +test("a missingtitle API error removes the stored article and warns", async () => { await dueArticle(); - const result = await run(respond).done; + const result = await run(() => new Response(MISSING)).done; expect(result.changed).toBe(1); expect(result.warnings.length).toBe(1); expect(await articles.readBody(TITLE)).toBeNull(); @@ -174,7 +172,7 @@ test.each([ test("a removed article still honours the 6-hour floor", async () => { await dueArticle(); - await run(() => new Response("gone", { status: 404 })).done; + await run(() => new Response(MISSING)).done; const { done, calls } = run(() => new Response(WITH_TIME)); await done; expect(calls.length).toBe(0); @@ -407,3 +405,20 @@ test("a row kept alive by its stated end stays listed, so its article is not rem await runRefresh({ ...opts, fetchImpl: async () => new Response("", { status: 304 }) }); expect(await articles.listed()).toEqual([TITLE]); }); + +test("an article read on an assumed robots permission names the host, even when the page was skipped", async () => { + // AGENTS.md § Scraping conduct: that warning must not become conditional. + const page = `${ROW}`; + const robots: RobotsGate = { + allows: async () => ({ allowed: true, reason: "assumed", assumedOnForbidden: true }), + }; + const { opts, store } = runner(page, () => new Response(WITH_TIME), { robots }); + await store.save(FANDOM_ID, { + url: opts.adapters[0]!.url, body: page, etag: null, lastModified: null, at: NOW.toISOString(), eventCount: 1, + }); + await store.recordCheck(FANDOM_ID, { at: NOW.toISOString(), status: 200, ok: true }); + const summary = await runRefresh(opts); + expect(summary.outcomes[0]?.result).toBe("skipped_interval"); + expect(summary.articlesChanged).toBe(1); + expect(summary.assumedRobots).toEqual(["genshin-impact.fandom.com"]); +});