From caefe6b0d735ae80c26fdc921868558879e34923 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Sat, 15 Aug 2026 00:10:53 +0200 Subject: [PATCH] feat: add HTML table reader and date parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every date function returns null rather than inferring a missing year, month or end date. A source that does not state an end yields endsAt: null, never a plausible guess — a confidently wrong end date is the failure this product exists to prevent. Handles the three formats seen on Game8: "August 12, 2026", "August 12 - September 21, 2026" (year applies to both ends, rolling the start back when the range crosses New Year), and a slash datetime range. Abbreviated months parse too, so a source switching to "Apr. 29" does not silently drop events. Co-Authored-By: Claude Opus 5 (1M context) --- src/ingest/dates.ts | 110 ++++++++++++++++++++++++++++++++++++++++ src/ingest/html.ts | 121 ++++++++++++++++++++++++++++++++++++++++++++ test/dates.test.ts | 86 +++++++++++++++++++++++++++++++ 3 files changed, 317 insertions(+) create mode 100644 src/ingest/dates.ts create mode 100644 src/ingest/html.ts create mode 100644 test/dates.test.ts diff --git a/src/ingest/dates.ts b/src/ingest/dates.ts new file mode 100644 index 0000000..520c3c6 --- /dev/null +++ b/src/ingest/dates.ts @@ -0,0 +1,110 @@ +/** + * Deterministic date parsing for adapter sources. + * + * Every function here returns null rather than guessing. A source that does not + * state a year, a month, or an end does not get one invented — see + * docs/PRD.md § Quality bar. Returning null is a correct outcome. + */ + +import type { Precision } from "../shared/schema.ts"; + +export interface ParsedInstant { + /** UTC ISO 8601. */ + iso: string; + precision: Extract; +} + +const MONTHS: Record = { + january: 1, february: 2, march: 3, april: 4, may: 5, june: 6, + july: 7, august: 8, september: 9, october: 10, november: 11, december: 12, + // Game8 abbreviates in some tables ("Apr. 29 - May 13, 2026"). Without these + // such rows parse as null and the events vanish silently, which is a worse + // failure than a wrong date because nothing surfaces it. + jan: 1, feb: 2, mar: 3, apr: 4, jun: 6, jul: 7, + aug: 8, sep: 9, sept: 9, oct: 10, nov: 11, dec: 12, +}; + +function monthNumber(name: string): number | null { + return MONTHS[name.toLowerCase().replace(/\.$/, "")] ?? null; +} + +function iso( + y: number, + m: number, + d: number, + hh = 0, + mm = 0, +): string | null { + if (m < 1 || m > 12 || d < 1 || d > 31 || hh > 23 || mm > 59) return null; + const date = new Date(Date.UTC(y, m - 1, d, hh, mm, 0, 0)); + // Rejects impossible calendar dates such as February 30. + if (date.getUTCMonth() !== m - 1 || date.getUTCDate() !== d) return null; + return date.toISOString(); +} + +/** "August 12, 2026" → 2026-08-12T00:00:00.000Z, day precision. */ +export function parseMonthDayYear(input: string): ParsedInstant | null { + const m = /([A-Za-z]+)\.?\s+(\d{1,2}),\s*(\d{4})/.exec(input); + if (!m) return null; + const month = monthNumber(m[1] ?? ""); + if (month === null) return null; + const value = iso(Number(m[3]), month, Number(m[2])); + return value === null ? null : { iso: value, precision: "day" }; +} + +/** + * "August 12 - September 21, 2026" → both instants, year taken from the end. + * A range whose end carries no year is unresolvable and returns null. + */ +export function parseMonthDayRange( + input: string, +): { start: ParsedInstant; end: ParsedInstant } | null { + const m = + /([A-Za-z]+)\.?\s+(\d{1,2})\s*[-–—]\s*([A-Za-z]+)\.?\s+(\d{1,2}),\s*(\d{4})/.exec( + input, + ); + if (!m) return null; + const startMonth = monthNumber(m[1] ?? ""); + const endMonth = monthNumber(m[3] ?? ""); + if (startMonth === null || endMonth === null) return null; + + const year = Number(m[5]); + // A range that crosses New Year renders as "December 28 - January 4, 2027", + // where the stated year belongs to the end. Roll the start back a year. + const startYear = startMonth > endMonth ? year - 1 : year; + + const startIso = iso(startYear, startMonth, Number(m[2])); + const endIso = iso(year, endMonth, Number(m[4])); + if (startIso === null || endIso === null) return null; + + return { + start: { iso: startIso, precision: "day" }, + end: { iso: endIso, precision: "day" }, + }; +} + +/** + * "2021/01/16 04:00 - 2021/01/31 03:59" → both instants, exact precision. + * Trailing prose after the range (e.g. "Currently Unavailable") is ignored. + * + * NOTE: the source states a wall-clock time but not a timezone. These are read + * as UTC. See the timezone caveat in the Genshin adapter. + */ +export function parseSlashDateTimeRange( + input: string, +): { start: ParsedInstant; end: ParsedInstant } | null { + const re = + /(\d{4})\/(\d{1,2})\/(\d{1,2})\s+(\d{1,2}):(\d{2})\s*[-–—]\s*(\d{4})\/(\d{1,2})\/(\d{1,2})\s+(\d{1,2}):(\d{2})/; + const m = re.exec(input); + if (!m) return null; + + const n = (i: number) => Number(m[i]); + const startIso = iso(n(1), n(2), n(3), n(4), n(5)); + const endIso = iso(n(6), n(7), n(8), n(9), n(10)); + if (startIso === null || endIso === null) return null; + + return { + start: { iso: startIso, precision: "exact" }, + end: { iso: endIso, precision: "exact" }, + }; +} diff --git a/src/ingest/html.ts b/src/ingest/html.ts new file mode 100644 index 0000000..66e53d4 --- /dev/null +++ b/src/ingest/html.ts @@ -0,0 +1,121 @@ +/** + * Minimal, dependency-free HTML reading for adapters. + * + * Deliberately not a general HTML parser. It handles the one shape adapters + * need — a linear walk of headings and flat tables — and is only safe because + * adapters assert their fixtures have no nested tables. If a source ever needs + * more than this, use Bun's built-in HTMLRewriter rather than growing regexes. + */ + +const ENTITIES: Record = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", + ndash: "–", + mdash: "—", + hellip: "…", + rsquo: "’", + lsquo: "‘", + ldquo: "“", + rdquo: "”", +}; + +export function decodeEntities(input: string): string { + return input + .replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) => + String.fromCodePoint(parseInt(hex, 16)), + ) + .replace(/&#(\d+);/g, (_, dec: string) => + String.fromCodePoint(parseInt(dec, 10)), + ) + .replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole); +} + +/** Strip tags, decode entities, collapse whitespace. */ +export function text(html: string): string { + return decodeEntities(html.replace(/<[^>]*>/g, " ")) + .replace(/\s+/g, " ") + .trim(); +} + +export interface TableNode { + kind: "table"; + /** Every → next-sibling pair, text-only. Empty for column tables. */ + pairs: Array<{ label: string; value: string }>; + /** All texts, in order — lets a caller recognise a header-row table. */ + headers: string[]; + /** Every as its cell texts, header row included. For column tables. */ + rows: string[][]; +} + +export type DocNode = + | { kind: "h2"; text: string } + | { kind: "h3"; text: string } + | TableNode; + +/** + * Walk a document in source order, yielding h2/h3 headings and tables. + * + * Pure: no network, no clock. Given identical input it always yields identical + * output, which is what makes fixture tests meaningful. + */ +export function scanDocument(rawHtml: string): DocNode[] { + const html = rawHtml.replace(//g, "").replace(/\s+/g, " "); + const nodes: DocNode[] = []; + + const re = /]*>([\s\S]*?)<\/h2>|]*>([\s\S]*?)<\/h3>|]*>([\s\S]*?)<\/table>/gi; + + for (const m of html.matchAll(re)) { + const [, h2, h3, table] = m; + if (h2 !== undefined) { + nodes.push({ kind: "h2", text: text(h2) }); + } else if (h3 !== undefined) { + nodes.push({ kind: "h3", text: text(h3) }); + } else if (table !== undefined) { + nodes.push(readTable(table)); + } + } + + return nodes; +} + +function readTable(body: string): TableNode { + const headers: string[] = []; + for (const m of body.matchAll(/]*>([\s\S]*?)<\/th>/gi)) { + headers.push(text(m[1] ?? "")); + } + + // Label/value rows: a immediately followed by a . + const pairs: Array<{ label: string; value: string }> = []; + const pairRe = /]*>([\s\S]*?)<\/th>\s*]*>([\s\S]*?)<\/td>/gi; + for (const m of body.matchAll(pairRe)) { + pairs.push({ label: text(m[1] ?? ""), value: text(m[2] ?? "") }); + } + + // Row/cell grid, for column-oriented tables. + const rows: string[][] = []; + for (const tr of body.matchAll(/]*>([\s\S]*?)<\/tr>/gi)) { + const cells: string[] = []; + for (const cell of (tr[1] ?? "").matchAll( + /]*>([\s\S]*?)<\/t\1>/gi, + )) { + cells.push(text(cell[2] ?? "")); + } + if (cells.length > 0) rows.push(cells); + } + + return { kind: "table", pairs, headers, rows }; +} + +/** True when the document contains no nested elements. */ +export function assertFlatTables(rawHtml: string): boolean { + let depth = 0; + for (const m of rawHtml.matchAll(/]*>|<\/table>/gi)) { + depth += m[0].startsWith(" 1) return false; + } + return depth === 0; +} diff --git a/test/dates.test.ts b/test/dates.test.ts new file mode 100644 index 0000000..58b4333 --- /dev/null +++ b/test/dates.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, test } from "bun:test"; +import { + parseMonthDayRange, + parseMonthDayYear, + parseSlashDateTimeRange, +} from "../src/ingest/dates.ts"; + +describe("parseMonthDayYear", () => { + test("parses a full date at day precision", () => { + expect(parseMonthDayYear("August 12, 2026")).toEqual({ + iso: "2026-08-12T00:00:00.000Z", + precision: "day", + }); + }); + + test("accepts abbreviated months with and without a period", () => { + expect(parseMonthDayYear("Apr. 29, 2026")?.iso).toBe( + "2026-04-29T00:00:00.000Z", + ); + expect(parseMonthDayYear("Sept 3, 2026")?.iso).toBe( + "2026-09-03T00:00:00.000Z", + ); + }); + + test("returns null rather than guessing a missing year", () => { + expect(parseMonthDayYear("August 12")).toBeNull(); + }); + + test("rejects impossible calendar dates", () => { + expect(parseMonthDayYear("February 30, 2026")).toBeNull(); + }); + + test("rejects an unknown month name", () => { + expect(parseMonthDayYear("Smarch 3, 2026")).toBeNull(); + }); +}); + +describe("parseMonthDayRange", () => { + test("applies the stated year to both ends", () => { + expect(parseMonthDayRange("August 12 - September 21, 2026")).toEqual({ + start: { iso: "2026-08-12T00:00:00.000Z", precision: "day" }, + end: { iso: "2026-09-21T00:00:00.000Z", precision: "day" }, + }); + }); + + test("rolls the start back a year when the range crosses New Year", () => { + // "December 28 - January 4, 2027" — the stated year belongs to the end. + const r = parseMonthDayRange("December 28 - January 4, 2027"); + expect(r?.start.iso).toBe("2026-12-28T00:00:00.000Z"); + expect(r?.end.iso).toBe("2027-01-04T00:00:00.000Z"); + }); + + test("handles en dash and abbreviated months", () => { + const r = parseMonthDayRange("Apr. 29 – May 13, 2026"); + expect(r?.start.iso).toBe("2026-04-29T00:00:00.000Z"); + expect(r?.end.iso).toBe("2026-05-13T00:00:00.000Z"); + }); + + test("returns null for a year-less range", () => { + // Game8 summary tables render "08/12 - 08/24". Guessing the year here is + // exactly the failure the product exists to prevent. + expect(parseMonthDayRange("08/12 - 08/24")).toBeNull(); + }); +}); + +describe("parseSlashDateTimeRange", () => { + test("parses a timed range at exact precision", () => { + expect( + parseSlashDateTimeRange("2021/01/16 04:00 - 2021/01/31 03:59"), + ).toEqual({ + start: { iso: "2021-01-16T04:00:00.000Z", precision: "exact" }, + end: { iso: "2021-01-31T03:59:00.000Z", precision: "exact" }, + }); + }); + + test("ignores trailing prose after the range", () => { + const r = parseSlashDateTimeRange( + "2021/01/16 04:00 - 2021/01/31 03:59 Currently Unavailable", + ); + expect(r?.end.iso).toBe("2021-01-31T03:59:00.000Z"); + }); + + test("returns null for non-date prose", () => { + expect(parseSlashDateTimeRange("Permanently Available")).toBeNull(); + }); +});