feat: add HTML table reader and date parsing
Every date function returns null rather than inferring a missing year, month or end date. A source that does not state an end yields endsAt: null, never a plausible guess — a confidently wrong end date is the failure this product exists to prevent. Handles the three formats seen on Game8: "August 12, 2026", "August 12 - September 21, 2026" (year applies to both ends, rolling the start back when the range crosses New Year), and a slash datetime range. Abbreviated months parse too, so a source switching to "Apr. 29" does not silently drop events. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5
Claude Opus 5
parent
e8293c7e8c
commit
caefe6b0d7
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* Deterministic date parsing for adapter sources.
|
||||
*
|
||||
* Every function here returns null rather than guessing. A source that does not
|
||||
* state a year, a month, or an end does not get one invented — see
|
||||
* docs/PRD.md § Quality bar. Returning null is a correct outcome.
|
||||
*/
|
||||
|
||||
import type { Precision } from "../shared/schema.ts";
|
||||
|
||||
export interface ParsedInstant {
|
||||
/** UTC ISO 8601. */
|
||||
iso: string;
|
||||
precision: Extract<Precision, "exact" | "day">;
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
january: 1, february: 2, march: 3, april: 4, may: 5, june: 6,
|
||||
july: 7, august: 8, september: 9, october: 10, november: 11, december: 12,
|
||||
// Game8 abbreviates in some tables ("Apr. 29 - May 13, 2026"). Without these
|
||||
// such rows parse as null and the events vanish silently, which is a worse
|
||||
// failure than a wrong date because nothing surfaces it.
|
||||
jan: 1, feb: 2, mar: 3, apr: 4, jun: 6, jul: 7,
|
||||
aug: 8, sep: 9, sept: 9, oct: 10, nov: 11, dec: 12,
|
||||
};
|
||||
|
||||
function monthNumber(name: string): number | null {
|
||||
return MONTHS[name.toLowerCase().replace(/\.$/, "")] ?? null;
|
||||
}
|
||||
|
||||
function iso(
|
||||
y: number,
|
||||
m: number,
|
||||
d: number,
|
||||
hh = 0,
|
||||
mm = 0,
|
||||
): string | null {
|
||||
if (m < 1 || m > 12 || d < 1 || d > 31 || hh > 23 || mm > 59) return null;
|
||||
const date = new Date(Date.UTC(y, m - 1, d, hh, mm, 0, 0));
|
||||
// Rejects impossible calendar dates such as February 30.
|
||||
if (date.getUTCMonth() !== m - 1 || date.getUTCDate() !== d) return null;
|
||||
return date.toISOString();
|
||||
}
|
||||
|
||||
/** "August 12, 2026" → 2026-08-12T00:00:00.000Z, day precision. */
|
||||
export function parseMonthDayYear(input: string): ParsedInstant | null {
|
||||
const m = /([A-Za-z]+)\.?\s+(\d{1,2}),\s*(\d{4})/.exec(input);
|
||||
if (!m) return null;
|
||||
const month = monthNumber(m[1] ?? "");
|
||||
if (month === null) return null;
|
||||
const value = iso(Number(m[3]), month, Number(m[2]));
|
||||
return value === null ? null : { iso: value, precision: "day" };
|
||||
}
|
||||
|
||||
/**
|
||||
* "August 12 - September 21, 2026" → both instants, year taken from the end.
|
||||
* A range whose end carries no year is unresolvable and returns null.
|
||||
*/
|
||||
export function parseMonthDayRange(
|
||||
input: string,
|
||||
): { start: ParsedInstant; end: ParsedInstant } | null {
|
||||
const m =
|
||||
/([A-Za-z]+)\.?\s+(\d{1,2})\s*[-–—]\s*([A-Za-z]+)\.?\s+(\d{1,2}),\s*(\d{4})/.exec(
|
||||
input,
|
||||
);
|
||||
if (!m) return null;
|
||||
const startMonth = monthNumber(m[1] ?? "");
|
||||
const endMonth = monthNumber(m[3] ?? "");
|
||||
if (startMonth === null || endMonth === null) return null;
|
||||
|
||||
const year = Number(m[5]);
|
||||
// A range that crosses New Year renders as "December 28 - January 4, 2027",
|
||||
// where the stated year belongs to the end. Roll the start back a year.
|
||||
const startYear = startMonth > endMonth ? year - 1 : year;
|
||||
|
||||
const startIso = iso(startYear, startMonth, Number(m[2]));
|
||||
const endIso = iso(year, endMonth, Number(m[4]));
|
||||
if (startIso === null || endIso === null) return null;
|
||||
|
||||
return {
|
||||
start: { iso: startIso, precision: "day" },
|
||||
end: { iso: endIso, precision: "day" },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* "2021/01/16 04:00 - 2021/01/31 03:59" → both instants, exact precision.
|
||||
* Trailing prose after the range (e.g. "Currently Unavailable") is ignored.
|
||||
*
|
||||
* NOTE: the source states a wall-clock time but not a timezone. These are read
|
||||
* as UTC. See the timezone caveat in the Genshin adapter.
|
||||
*/
|
||||
export function parseSlashDateTimeRange(
|
||||
input: string,
|
||||
): { start: ParsedInstant; end: ParsedInstant } | null {
|
||||
const re =
|
||||
/(\d{4})\/(\d{1,2})\/(\d{1,2})\s+(\d{1,2}):(\d{2})\s*[-–—]\s*(\d{4})\/(\d{1,2})\/(\d{1,2})\s+(\d{1,2}):(\d{2})/;
|
||||
const m = re.exec(input);
|
||||
if (!m) return null;
|
||||
|
||||
const n = (i: number) => Number(m[i]);
|
||||
const startIso = iso(n(1), n(2), n(3), n(4), n(5));
|
||||
const endIso = iso(n(6), n(7), n(8), n(9), n(10));
|
||||
if (startIso === null || endIso === null) return null;
|
||||
|
||||
return {
|
||||
start: { iso: startIso, precision: "exact" },
|
||||
end: { iso: endIso, precision: "exact" },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Minimal, dependency-free HTML reading for adapters.
|
||||
*
|
||||
* Deliberately not a general HTML parser. It handles the one shape adapters
|
||||
* need — a linear walk of headings and flat tables — and is only safe because
|
||||
* adapters assert their fixtures have no nested tables. If a source ever needs
|
||||
* more than this, use Bun's built-in HTMLRewriter rather than growing regexes.
|
||||
*/
|
||||
|
||||
const ENTITIES: Record<string, string> = {
|
||||
amp: "&",
|
||||
lt: "<",
|
||||
gt: ">",
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: " ",
|
||||
ndash: "–",
|
||||
mdash: "—",
|
||||
hellip: "…",
|
||||
rsquo: "’",
|
||||
lsquo: "‘",
|
||||
ldquo: "“",
|
||||
rdquo: "”",
|
||||
};
|
||||
|
||||
export function decodeEntities(input: string): string {
|
||||
return input
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) =>
|
||||
String.fromCodePoint(parseInt(hex, 16)),
|
||||
)
|
||||
.replace(/&#(\d+);/g, (_, dec: string) =>
|
||||
String.fromCodePoint(parseInt(dec, 10)),
|
||||
)
|
||||
.replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole);
|
||||
}
|
||||
|
||||
/** Strip tags, decode entities, collapse whitespace. */
|
||||
export function text(html: string): string {
|
||||
return decodeEntities(html.replace(/<[^>]*>/g, " "))
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export interface TableNode {
|
||||
kind: "table";
|
||||
/** Every <th> → next-sibling <td> pair, text-only. Empty for column tables. */
|
||||
pairs: Array<{ label: string; value: string }>;
|
||||
/** All <th> texts, in order — lets a caller recognise a header-row table. */
|
||||
headers: string[];
|
||||
/** Every <tr> as its cell texts, header row included. For column tables. */
|
||||
rows: string[][];
|
||||
}
|
||||
|
||||
export type DocNode =
|
||||
| { kind: "h2"; text: string }
|
||||
| { kind: "h3"; text: string }
|
||||
| TableNode;
|
||||
|
||||
/**
|
||||
* Walk a document in source order, yielding h2/h3 headings and tables.
|
||||
*
|
||||
* Pure: no network, no clock. Given identical input it always yields identical
|
||||
* output, which is what makes fixture tests meaningful.
|
||||
*/
|
||||
export function scanDocument(rawHtml: string): DocNode[] {
|
||||
const html = rawHtml.replace(/<!--[\s\S]*?-->/g, "").replace(/\s+/g, " ");
|
||||
const nodes: DocNode[] = [];
|
||||
|
||||
const re = /<h2\b[^>]*>([\s\S]*?)<\/h2>|<h3\b[^>]*>([\s\S]*?)<\/h3>|<table\b[^>]*>([\s\S]*?)<\/table>/gi;
|
||||
|
||||
for (const m of html.matchAll(re)) {
|
||||
const [, h2, h3, table] = m;
|
||||
if (h2 !== undefined) {
|
||||
nodes.push({ kind: "h2", text: text(h2) });
|
||||
} else if (h3 !== undefined) {
|
||||
nodes.push({ kind: "h3", text: text(h3) });
|
||||
} else if (table !== undefined) {
|
||||
nodes.push(readTable(table));
|
||||
}
|
||||
}
|
||||
|
||||
return nodes;
|
||||
}
|
||||
|
||||
function readTable(body: string): TableNode {
|
||||
const headers: string[] = [];
|
||||
for (const m of body.matchAll(/<th\b[^>]*>([\s\S]*?)<\/th>/gi)) {
|
||||
headers.push(text(m[1] ?? ""));
|
||||
}
|
||||
|
||||
// Label/value rows: a <th> immediately followed by a <td>.
|
||||
const pairs: Array<{ label: string; value: string }> = [];
|
||||
const pairRe = /<th\b[^>]*>([\s\S]*?)<\/th>\s*<td\b[^>]*>([\s\S]*?)<\/td>/gi;
|
||||
for (const m of body.matchAll(pairRe)) {
|
||||
pairs.push({ label: text(m[1] ?? ""), value: text(m[2] ?? "") });
|
||||
}
|
||||
|
||||
// Row/cell grid, for column-oriented tables.
|
||||
const rows: string[][] = [];
|
||||
for (const tr of body.matchAll(/<tr\b[^>]*>([\s\S]*?)<\/tr>/gi)) {
|
||||
const cells: string[] = [];
|
||||
for (const cell of (tr[1] ?? "").matchAll(
|
||||
/<t([dh])\b[^>]*>([\s\S]*?)<\/t\1>/gi,
|
||||
)) {
|
||||
cells.push(text(cell[2] ?? ""));
|
||||
}
|
||||
if (cells.length > 0) rows.push(cells);
|
||||
}
|
||||
|
||||
return { kind: "table", pairs, headers, rows };
|
||||
}
|
||||
|
||||
/** True when the document contains no nested <table> elements. */
|
||||
export function assertFlatTables(rawHtml: string): boolean {
|
||||
let depth = 0;
|
||||
for (const m of rawHtml.matchAll(/<table\b[^>]*>|<\/table>/gi)) {
|
||||
depth += m[0].startsWith("</") ? -1 : 1;
|
||||
if (depth > 1) return false;
|
||||
}
|
||||
return depth === 0;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
parseMonthDayRange,
|
||||
parseMonthDayYear,
|
||||
parseSlashDateTimeRange,
|
||||
} from "../src/ingest/dates.ts";
|
||||
|
||||
describe("parseMonthDayYear", () => {
|
||||
test("parses a full date at day precision", () => {
|
||||
expect(parseMonthDayYear("August 12, 2026")).toEqual({
|
||||
iso: "2026-08-12T00:00:00.000Z",
|
||||
precision: "day",
|
||||
});
|
||||
});
|
||||
|
||||
test("accepts abbreviated months with and without a period", () => {
|
||||
expect(parseMonthDayYear("Apr. 29, 2026")?.iso).toBe(
|
||||
"2026-04-29T00:00:00.000Z",
|
||||
);
|
||||
expect(parseMonthDayYear("Sept 3, 2026")?.iso).toBe(
|
||||
"2026-09-03T00:00:00.000Z",
|
||||
);
|
||||
});
|
||||
|
||||
test("returns null rather than guessing a missing year", () => {
|
||||
expect(parseMonthDayYear("August 12")).toBeNull();
|
||||
});
|
||||
|
||||
test("rejects impossible calendar dates", () => {
|
||||
expect(parseMonthDayYear("February 30, 2026")).toBeNull();
|
||||
});
|
||||
|
||||
test("rejects an unknown month name", () => {
|
||||
expect(parseMonthDayYear("Smarch 3, 2026")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseMonthDayRange", () => {
|
||||
test("applies the stated year to both ends", () => {
|
||||
expect(parseMonthDayRange("August 12 - September 21, 2026")).toEqual({
|
||||
start: { iso: "2026-08-12T00:00:00.000Z", precision: "day" },
|
||||
end: { iso: "2026-09-21T00:00:00.000Z", precision: "day" },
|
||||
});
|
||||
});
|
||||
|
||||
test("rolls the start back a year when the range crosses New Year", () => {
|
||||
// "December 28 - January 4, 2027" — the stated year belongs to the end.
|
||||
const r = parseMonthDayRange("December 28 - January 4, 2027");
|
||||
expect(r?.start.iso).toBe("2026-12-28T00:00:00.000Z");
|
||||
expect(r?.end.iso).toBe("2027-01-04T00:00:00.000Z");
|
||||
});
|
||||
|
||||
test("handles en dash and abbreviated months", () => {
|
||||
const r = parseMonthDayRange("Apr. 29 – May 13, 2026");
|
||||
expect(r?.start.iso).toBe("2026-04-29T00:00:00.000Z");
|
||||
expect(r?.end.iso).toBe("2026-05-13T00:00:00.000Z");
|
||||
});
|
||||
|
||||
test("returns null for a year-less range", () => {
|
||||
// Game8 summary tables render "08/12 - 08/24". Guessing the year here is
|
||||
// exactly the failure the product exists to prevent.
|
||||
expect(parseMonthDayRange("08/12 - 08/24")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseSlashDateTimeRange", () => {
|
||||
test("parses a timed range at exact precision", () => {
|
||||
expect(
|
||||
parseSlashDateTimeRange("2021/01/16 04:00 - 2021/01/31 03:59"),
|
||||
).toEqual({
|
||||
start: { iso: "2021-01-16T04:00:00.000Z", precision: "exact" },
|
||||
end: { iso: "2021-01-31T03:59:00.000Z", precision: "exact" },
|
||||
});
|
||||
});
|
||||
|
||||
test("ignores trailing prose after the range", () => {
|
||||
const r = parseSlashDateTimeRange(
|
||||
"2021/01/16 04:00 - 2021/01/31 03:59 Currently Unavailable",
|
||||
);
|
||||
expect(r?.end.iso).toBe("2021-01-31T03:59:00.000Z");
|
||||
});
|
||||
|
||||
test("returns null for non-date prose", () => {
|
||||
expect(parseSlashDateTimeRange("Permanently Available")).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user