From 3ea7286f56477aa2cc6f4f65237f0f9d07dd88de Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Sat, 15 Aug 2026 00:11:08 +0200 Subject: [PATCH] docs: drop the LLM extraction layer, document multi-source ingestion Event data is parsed deterministically; there is no model call, API key or per-run cost anywhere in the pipeline. A source that cannot be parsed deterministically gets no adapter rather than an inference fallback. Documents the parser/adapter/merge split, records that Game8 uses three page templates, and adds Neverness to Everness. Co-Authored-By: Claude Opus 5 (1M context) --- docs/ARCHITECTURE.md | 42 +++--- docs/DATA-MODEL.md | 36 ++--- docs/INGESTION.md | 322 +++++++++++++++++++++++------------------ docs/LLM-EXTRACTION.md | 291 ------------------------------------- docs/PRD.md | 12 +- 5 files changed, 215 insertions(+), 488 deletions(-) delete mode 100644 docs/LLM-EXTRACTION.md diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 88d3564..a582e39 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -14,7 +14,7 @@ are no users. │ │ │ │ ▼ │ │ ingest pipeline │ - │ fetch → clean → parse|extract → validate │ + │ fetch → parse → validate │ │ → reconcile → gate → publish │ │ │ │ │ │ │ └──► quarantine │ @@ -35,9 +35,8 @@ are no users. filters, region ← never leaves the device ``` -Anthropic API calls happen only inside the ingest pipeline. **No request path — not `/api/events`, -not a page load — ever calls the model.** If a feature seems to need live inference, it needs a -precomputed field instead. +The pipeline makes no third-party API calls beyond fetching source pages. There is no inference +anywhere, at ingest time or in a request path. ## Layout @@ -55,24 +54,19 @@ src/ migrations/ NNN-name.sql, applied in order at boot queries.ts all SQL lives here — no SQL in route handlers ingest/ - scheduler.ts timer + jitter + per-source lock - pipeline.ts the 7 stages, orchestration only - clean.ts HTML → text reduction before extraction - extract.ts Anthropic client, prompts, batch submission - validate.ts zod parse + calendar sanity rules - reconcile.ts diff vs published, confidence, conflict detection + scheduler.ts timer + jitter + per-source lock [not built] + pipeline.ts the 6 stages, orchestration only [not built] + html.ts flat-table HTML reader (no dependency) ✓ built + dates.ts deterministic date parsing ✓ built + validate.ts zod parse + calendar sanity rules [not built] + reconcile.ts diff vs published, confidence, conflicts [not built] adapters/ - index.ts registry: GameId → Adapter - genshin.ts - hsr.ts - zzz.ts - wuwa.ts - arknights.ts - endfield.ts + types.ts Adapter interface, ParseContext ✓ built + game8.ts shared Game8 parser (2 table shapes) ✓ built + index.ts registry: adapter id → Adapter ✓ built shared/ - schema.ts zod schemas — the contract, imported by both sides - types.ts z.infer types only - time.ts region reset math, duration formatting + schema.ts zod schemas — the contract, both sides ✓ built + time.ts region reset math, duration formatting [not built] client/ main.tsx App.tsx @@ -131,21 +125,19 @@ in that area needs an explicit auth story first. data — worst case, the game's lane goes stale and gets a warning badge (F7). - Three consecutive failures for one source raises its `health` to `failing` in `/api/health`. It does not stop the schedule; a wiki being down for a day is normal. -- Extraction results are written to `extraction_log` with the input hash, so a prompt change can be - evaluated against previously-seen inputs without re-fetching or re-paying. +- Raw snapshots are cached by content hash, so a parser change is always evaluated offline against + stored pages rather than by re-fetching. ## Deployment -Single process, single SQLite file, no external services beyond the Anthropic API. +Single process, single SQLite file, no external services at all. ``` PORT=3000 ADMIN_PORT=3001 # bound to 127.0.0.1 DATABASE_PATH=./data/events.sqlite -ANTHROPIC_API_KEY=sk-ant-... INGEST_INTERVAL_MS=21600000 INGEST_ENABLED=true # false for local UI work — never hits the network or the API -EXTRACTION_MODE=batch # batch | sync CONFIDENCE_THRESHOLD=0.8 ``` diff --git a/docs/DATA-MODEL.md b/docs/DATA-MODEL.md index cdf4192..0da8729 100644 --- a/docs/DATA-MODEL.md +++ b/docs/DATA-MODEL.md @@ -9,7 +9,7 @@ import { z } from "zod"; export const GameId = z.enum([ - "genshin", "hsr", "zzz", "wuwa", "arknights", "endfield", + "genshin", "hsr", "zzz", "wuwa", "arknights", "endfield", "nte", ]); export const EventType = z.enum([ @@ -54,7 +54,7 @@ export const GachaEvent = z.object({ status: z.enum(["published", "delisted"]), confidence: z.number().min(0).max(1), - extractionMethod: z.enum(["parser", "llm", "manual"]), + extractionMethod: z.enum(["parser", "manual"]), version: z.number().int().positive(), firstSeenAt: z.string().datetime(), @@ -78,8 +78,8 @@ true`, with `regionEnds` carrying the three resolved UTC instants. The client pi user's stored region (PRD F5). Collapsing these into a single timestamp loses up to 13 hours of accuracy and will make the countdown wrong for two thirds of users. -**`confidence`** is assigned during reconciliation, not by the model's self-report. See -`docs/LLM-EXTRACTION.md` § Scoring — a model asserting "I am 0.95 confident" is not evidence. +**`confidence`** is assigned by the parser and adjusted during merge and reconciliation — see +`docs/INGESTION.md` § Scoring. It records how firmly the sources pinned the event down. **`status: "delisted"`** means the event stopped appearing at its source. It is never deleted, because a source outage would otherwise silently empty the calendar. Delisted events are excluded @@ -122,7 +122,7 @@ CREATE TABLE events ( source_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'published', confidence REAL NOT NULL, - extraction_method TEXT NOT NULL, + extraction_method TEXT NOT NULL, -- 'parser' | 'manual' version INTEGER NOT NULL DEFAULT 1, first_seen_at TEXT NOT NULL, updated_at TEXT NOT NULL @@ -147,10 +147,11 @@ CREATE INDEX idx_quarantine_open ON events_quarantine (created_at) WHERE resolve -- One row per configured source. CREATE TABLE sources ( - id TEXT PRIMARY KEY, -- 'genshin-wiki-events' + id TEXT PRIMARY KEY, -- '--', e.g. 'genshin-game8-events' game TEXT NOT NULL, url TEXT NOT NULL, - strategy TEXT NOT NULL, -- 'parser' | 'llm' | 'parser_then_llm' + parser_id TEXT NOT NULL, -- parser template id, e.g. 'game8' + priority INTEGER NOT NULL DEFAULT 0, min_interval_ms INTEGER NOT NULL DEFAULT 21600000, etag TEXT, last_modified TEXT, @@ -177,26 +178,7 @@ CREATE TABLE ingest_runs ( events_held INTEGER DEFAULT 0 ); --- One row per LLM call. Enables replaying prompt changes against past inputs. -CREATE TABLE extraction_log ( - id TEXT PRIMARY KEY, - run_id TEXT NOT NULL REFERENCES ingest_runs(id), - source_id TEXT NOT NULL, - model TEXT NOT NULL, - prompt_version TEXT NOT NULL, - input_hash TEXT NOT NULL, -- of the cleaned text sent to the model - input_tokens INTEGER, - output_tokens INTEGER, - cache_read_tokens INTEGER, - cache_write_tokens INTEGER, - stop_reason TEXT, - refusal_category TEXT, - duration_ms INTEGER, - created_at TEXT NOT NULL -); -CREATE INDEX idx_extraction_input ON extraction_log (input_hash); - --- Cached raw + cleaned snapshots so re-extraction never re-fetches. +-- Cached raw snapshots so re-parsing never re-fetches. CREATE TABLE snapshots ( content_hash TEXT PRIMARY KEY, source_id TEXT NOT NULL, diff --git a/docs/INGESTION.md b/docs/INGESTION.md index fba8df0..d3a58ca 100644 --- a/docs/INGESTION.md +++ b/docs/INGESTION.md @@ -1,108 +1,156 @@ # Ingestion Pipeline -Seven stages, run per source. Every stage writes its outcome to `ingest_runs` so a failure two days -ago can be diagnosed without re-running or re-paying. +Six stages, run per source. Every stage writes its outcome to `ingest_runs` so a failure two days +ago can be diagnosed without re-running. ``` -fetch → clean → parse|extract → validate → reconcile → gate → publish - │ - └──► quarantine +fetch → parse → merge → validate → reconcile → gate → publish + │ + └──► quarantine ``` -## The adapter contract +## No LLM -An adapter is the only per-game code. Everything downstream of `parse` is shared. +Event data is extracted by deterministic code only. There is no model call anywhere in this +pipeline, no API key, and no per-run cost. + +This is a deliberate constraint, not an omission: + +- A source that cannot be parsed deterministically **does not get an adapter.** Report it rather + than reaching for inference. +- Parser output is reproducible — the same fixture always yields the same events, which is what + makes the fixture tests meaningful. +- Iterating is free and offline: `bun run parse `. + +If a source's markup is too unstable to parse, the answer is a different source, not a model. + +## Three layers: parsers, adapters, merge + +The layering is what makes a second, third, or tenth source cheap. + +| Layer | Answers | Lives in | Scope | +|---|---|---|---| +| **Parser** | "How is this *site* laid out?" | `src/ingest/parsers/` | One site template, many games | +| **Adapter** | "Which URL, for which game, via which parser?" | `src/ingest/adapters/index.ts` | One page | +| **Merge** | "These sources disagree — now what?" | `src/ingest/merge.ts` | One game, many sources | + +Consequences worth internalising: + +- Adding a source for a site already parsed = **one entry in `SOURCES`**. No new parsing code. +- Adding a new *site* = one parser module + its `PARSERS` entry, then adapters as above. +- A game may have any number of sources. `parseGame(game, documents, now)` runs them all and + merges. + +### The parser interface ```ts -export interface Adapter { - id: string; // 'genshin-wiki-events' - game: GameId; - url: string; - strategy: "parser" | "llm" | "parser_then_llm"; - minIntervalMs?: number; // default 6h - - /** Narrow the cleaned document to just the region containing event data. */ - select?(cleaned: string): string; - - /** - * Deterministic parse. Return null to fall through to LLM extraction - * (only meaningful when strategy is 'parser_then_llm'). - * Pure over its input — no network, no clock, no randomness. This is what - * makes fixture tests possible. - */ - parse?(cleaned: string, ctx: ParseContext): RawEvent[] | null; - - /** Extra instructions appended to the shared extraction prompt. */ - extractionHints?: string; - - /** Game-specific normalization: reset times, region offsets, patch cadence. */ - normalize(raw: RawEvent, ctx: ParseContext): GachaEvent; -} - -export interface ParseContext { - now: string; // injected, never Date.now() — keeps parse pure and testable - sourceUrl: string; - sourceId: string; - game: GameId; +export interface SourceParser { + id: string; // "game8" + label: string; // "Game8" + canParse(html: string): boolean; // structural sanity check + parse(html: string, ctx: ParseContext): GachaEvent[]; } ``` -**`parse` must not read the clock.** It takes `now` from `ctx`. This is what lets a fixture test -assert exact output for a page captured last March. +`canParse` is the redesign tripwire. Without it, a site rewrite makes every selector miss and the +parser returns zero events — which reads downstream as "this game has no events" rather than as a +failure. The adapter throws when `canParse` is false, so the run fails loudly and the previously +published events stay put. -### Choosing a strategy +Keep `canParse` structural, not content-based, and **do not over-fit it**. Game8's own pages differ +in attribute quote style (`class="a-table"` on Genshin, `class='a-table'` on NTE), which is exactly +the kind of variation a naive check gets wrong. Every regex in `html.ts` is attribute-agnostic for +the same reason. -| Source shape | Strategy | +### The adapter registry + +```ts +const SOURCES: SourceSpec[] = [ + { id: "genshin-game8-events", game: "genshin", + url: "https://game8.co/games/Genshin-Impact/archives/301601", parserId: "game8" }, + { id: "nte-game8-events", game: "nte", + url: "https://game8.co/games/Neverness-to-Everness/archives/592073", parserId: "game8" }, +]; +``` + +`priority` (default 0) breaks ties when two sources disagree and neither is clearly better — give +official feeds a higher number than community wikis. Adapter ids are `"--"` and +are recorded on every event as `sourceId`, so any row in the feed traces back to the source that +produced it. + +### Assessing a new source + +| Source shape | Verdict | |---|---| -| JSON API, or a stable HTML table with consistent headers | `parser` | -| Free-form patch notes, announcement prose, inconsistent markup | `llm` | -| Mostly-stable markup that occasionally changes | `parser_then_llm` | +| JSON API, or an HTML table with consistent headers | Good — write the adapter | +| Label/value or column tables with full dates including a year | Good — an existing parser may already handle it | +| Dates without a year, or no end date at all | **Unsupportable** — yields nothing rather than guessing | +| Free-form prose with no table structure | Find a different source | -Prefer `parser`. It is free, deterministic, and instantly testable. The LLM exists for sources that -genuinely cannot be parsed reliably, not as the default. A source with a clean API that goes through -the model is a bug. +Game8 uses at least three page templates and a game's page may use any of them: + +1. **Label/value detail tables** — `Event Start` / `Event End` rows under a per-event `h3`, full + dates with year. *(Genshin Impact)* +2. **Column tables** — `Event | Duration | Event Details | Rewards`, one row per event, under a + section heading. *(Neverness to Everness)* +3. **Image-grid schedules** — a bare `MM/DD`, no year, no end date. **Unsupportable.** + *(Arknights: Endfield)* + +Shapes 1 and 2 are handled. Before assuming a new Game8 page will work, dump its heading/table +structure and check which shape it uses. ## Stage 1 — fetch - Send `If-None-Match` / `If-Modified-Since` from `sources.etag` / `last_modified`. A `304` ends - the run as `skipped_unchanged` with zero further cost. + the run as `skipped_unchanged`. - `User-Agent: gacha-event-tracker/1.0 (+https://github.com//gacha-event-tracker)`. -- Honor `robots.txt`. Cache the parsed robots per host for 24h. -- 20s timeout; retry twice with exponential backoff on 5xx and network errors; never retry 4xx. -- Store the raw bytes in `snapshots`. +- Honor `robots.txt`; cache parsed robots per host for 24h. +- 20s timeout; retry twice with backoff on 5xx and network errors; never retry 4xx. +- Store raw bytes in `snapshots`. -On failure: increment `consecutive_failures`, leave published events untouched, end the run as -`failed`. A source being down never mutates the feed. +On failure: increment `consecutive_failures`, leave published events untouched, end as `failed`. A +source being down never mutates the feed. -## Stage 2 — clean +## Stage 2 — parse -Reduce the document before it costs anything. This stage is the second-biggest cost lever after the -content-hash skip. +Hash the raw body (sha256) → `content_hash`. **If it matches `sources.content_hash`, end as +`skipped_unchanged`** and do no further work. -- Drop `