feat: add HTML table reader and date parsing
Every date function returns null rather than inferring a missing year, month or end date. A source that does not state an end yields endsAt: null, never a plausible guess — a confidently wrong end date is the failure this product exists to prevent. Handles the three formats seen on Game8: "August 12, 2026", "August 12 - September 21, 2026" (year applies to both ends, rolling the start back when the range crosses New Year), and a slash datetime range. Abbreviated months parse too, so a source switching to "Apr. 29" does not silently drop events. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
e8293c7e8c
commit
caefe6b0d7
@@ -0,0 +1,110 @@
|
|||||||
|
/**
|
||||||
|
* Deterministic date parsing for adapter sources.
|
||||||
|
*
|
||||||
|
* Every function here returns null rather than guessing. A source that does not
|
||||||
|
* state a year, a month, or an end does not get one invented — see
|
||||||
|
* docs/PRD.md § Quality bar. Returning null is a correct outcome.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { Precision } from "../shared/schema.ts";
|
||||||
|
|
||||||
|
export interface ParsedInstant {
|
||||||
|
/** UTC ISO 8601. */
|
||||||
|
iso: string;
|
||||||
|
precision: Extract<Precision, "exact" | "day">;
|
||||||
|
}
|
||||||
|
|
||||||
|
const MONTHS: Record<string, number> = {
|
||||||
|
january: 1, february: 2, march: 3, april: 4, may: 5, june: 6,
|
||||||
|
july: 7, august: 8, september: 9, october: 10, november: 11, december: 12,
|
||||||
|
// Game8 abbreviates in some tables ("Apr. 29 - May 13, 2026"). Without these
|
||||||
|
// such rows parse as null and the events vanish silently, which is a worse
|
||||||
|
// failure than a wrong date because nothing surfaces it.
|
||||||
|
jan: 1, feb: 2, mar: 3, apr: 4, jun: 6, jul: 7,
|
||||||
|
aug: 8, sep: 9, sept: 9, oct: 10, nov: 11, dec: 12,
|
||||||
|
};
|
||||||
|
|
||||||
|
function monthNumber(name: string): number | null {
|
||||||
|
return MONTHS[name.toLowerCase().replace(/\.$/, "")] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function iso(
|
||||||
|
y: number,
|
||||||
|
m: number,
|
||||||
|
d: number,
|
||||||
|
hh = 0,
|
||||||
|
mm = 0,
|
||||||
|
): string | null {
|
||||||
|
if (m < 1 || m > 12 || d < 1 || d > 31 || hh > 23 || mm > 59) return null;
|
||||||
|
const date = new Date(Date.UTC(y, m - 1, d, hh, mm, 0, 0));
|
||||||
|
// Rejects impossible calendar dates such as February 30.
|
||||||
|
if (date.getUTCMonth() !== m - 1 || date.getUTCDate() !== d) return null;
|
||||||
|
return date.toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "August 12, 2026" → 2026-08-12T00:00:00.000Z, day precision. */
|
||||||
|
export function parseMonthDayYear(input: string): ParsedInstant | null {
|
||||||
|
const m = /([A-Za-z]+)\.?\s+(\d{1,2}),\s*(\d{4})/.exec(input);
|
||||||
|
if (!m) return null;
|
||||||
|
const month = monthNumber(m[1] ?? "");
|
||||||
|
if (month === null) return null;
|
||||||
|
const value = iso(Number(m[3]), month, Number(m[2]));
|
||||||
|
return value === null ? null : { iso: value, precision: "day" };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "August 12 - September 21, 2026" → both instants, year taken from the end.
|
||||||
|
* A range whose end carries no year is unresolvable and returns null.
|
||||||
|
*/
|
||||||
|
export function parseMonthDayRange(
|
||||||
|
input: string,
|
||||||
|
): { start: ParsedInstant; end: ParsedInstant } | null {
|
||||||
|
const m =
|
||||||
|
/([A-Za-z]+)\.?\s+(\d{1,2})\s*[-–—]\s*([A-Za-z]+)\.?\s+(\d{1,2}),\s*(\d{4})/.exec(
|
||||||
|
input,
|
||||||
|
);
|
||||||
|
if (!m) return null;
|
||||||
|
const startMonth = monthNumber(m[1] ?? "");
|
||||||
|
const endMonth = monthNumber(m[3] ?? "");
|
||||||
|
if (startMonth === null || endMonth === null) return null;
|
||||||
|
|
||||||
|
const year = Number(m[5]);
|
||||||
|
// A range that crosses New Year renders as "December 28 - January 4, 2027",
|
||||||
|
// where the stated year belongs to the end. Roll the start back a year.
|
||||||
|
const startYear = startMonth > endMonth ? year - 1 : year;
|
||||||
|
|
||||||
|
const startIso = iso(startYear, startMonth, Number(m[2]));
|
||||||
|
const endIso = iso(year, endMonth, Number(m[4]));
|
||||||
|
if (startIso === null || endIso === null) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
start: { iso: startIso, precision: "day" },
|
||||||
|
end: { iso: endIso, precision: "day" },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "2021/01/16 04:00 - 2021/01/31 03:59" → both instants, exact precision.
|
||||||
|
* Trailing prose after the range (e.g. "Currently Unavailable") is ignored.
|
||||||
|
*
|
||||||
|
* NOTE: the source states a wall-clock time but not a timezone. These are read
|
||||||
|
* as UTC. See the timezone caveat in the Genshin adapter.
|
||||||
|
*/
|
||||||
|
export function parseSlashDateTimeRange(
|
||||||
|
input: string,
|
||||||
|
): { start: ParsedInstant; end: ParsedInstant } | null {
|
||||||
|
const re =
|
||||||
|
/(\d{4})\/(\d{1,2})\/(\d{1,2})\s+(\d{1,2}):(\d{2})\s*[-–—]\s*(\d{4})\/(\d{1,2})\/(\d{1,2})\s+(\d{1,2}):(\d{2})/;
|
||||||
|
const m = re.exec(input);
|
||||||
|
if (!m) return null;
|
||||||
|
|
||||||
|
const n = (i: number) => Number(m[i]);
|
||||||
|
const startIso = iso(n(1), n(2), n(3), n(4), n(5));
|
||||||
|
const endIso = iso(n(6), n(7), n(8), n(9), n(10));
|
||||||
|
if (startIso === null || endIso === null) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
start: { iso: startIso, precision: "exact" },
|
||||||
|
end: { iso: endIso, precision: "exact" },
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
/**
|
||||||
|
* Minimal, dependency-free HTML reading for adapters.
|
||||||
|
*
|
||||||
|
* Deliberately not a general HTML parser. It handles the one shape adapters
|
||||||
|
* need — a linear walk of headings and flat tables — and is only safe because
|
||||||
|
* adapters assert their fixtures have no nested tables. If a source ever needs
|
||||||
|
* more than this, use Bun's built-in HTMLRewriter rather than growing regexes.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const ENTITIES: Record<string, string> = {
|
||||||
|
amp: "&",
|
||||||
|
lt: "<",
|
||||||
|
gt: ">",
|
||||||
|
quot: '"',
|
||||||
|
apos: "'",
|
||||||
|
nbsp: " ",
|
||||||
|
ndash: "–",
|
||||||
|
mdash: "—",
|
||||||
|
hellip: "…",
|
||||||
|
rsquo: "’",
|
||||||
|
lsquo: "‘",
|
||||||
|
ldquo: "“",
|
||||||
|
rdquo: "”",
|
||||||
|
};
|
||||||
|
|
||||||
|
export function decodeEntities(input: string): string {
|
||||||
|
return input
|
||||||
|
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) =>
|
||||||
|
String.fromCodePoint(parseInt(hex, 16)),
|
||||||
|
)
|
||||||
|
.replace(/&#(\d+);/g, (_, dec: string) =>
|
||||||
|
String.fromCodePoint(parseInt(dec, 10)),
|
||||||
|
)
|
||||||
|
.replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Strip tags, decode entities, collapse whitespace. */
|
||||||
|
export function text(html: string): string {
|
||||||
|
return decodeEntities(html.replace(/<[^>]*>/g, " "))
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TableNode {
|
||||||
|
kind: "table";
|
||||||
|
/** Every <th> → next-sibling <td> pair, text-only. Empty for column tables. */
|
||||||
|
pairs: Array<{ label: string; value: string }>;
|
||||||
|
/** All <th> texts, in order — lets a caller recognise a header-row table. */
|
||||||
|
headers: string[];
|
||||||
|
/** Every <tr> as its cell texts, header row included. For column tables. */
|
||||||
|
rows: string[][];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DocNode =
|
||||||
|
| { kind: "h2"; text: string }
|
||||||
|
| { kind: "h3"; text: string }
|
||||||
|
| TableNode;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Walk a document in source order, yielding h2/h3 headings and tables.
|
||||||
|
*
|
||||||
|
* Pure: no network, no clock. Given identical input it always yields identical
|
||||||
|
* output, which is what makes fixture tests meaningful.
|
||||||
|
*/
|
||||||
|
export function scanDocument(rawHtml: string): DocNode[] {
|
||||||
|
const html = rawHtml.replace(/<!--[\s\S]*?-->/g, "").replace(/\s+/g, " ");
|
||||||
|
const nodes: DocNode[] = [];
|
||||||
|
|
||||||
|
const re = /<h2\b[^>]*>([\s\S]*?)<\/h2>|<h3\b[^>]*>([\s\S]*?)<\/h3>|<table\b[^>]*>([\s\S]*?)<\/table>/gi;
|
||||||
|
|
||||||
|
for (const m of html.matchAll(re)) {
|
||||||
|
const [, h2, h3, table] = m;
|
||||||
|
if (h2 !== undefined) {
|
||||||
|
nodes.push({ kind: "h2", text: text(h2) });
|
||||||
|
} else if (h3 !== undefined) {
|
||||||
|
nodes.push({ kind: "h3", text: text(h3) });
|
||||||
|
} else if (table !== undefined) {
|
||||||
|
nodes.push(readTable(table));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readTable(body: string): TableNode {
|
||||||
|
const headers: string[] = [];
|
||||||
|
for (const m of body.matchAll(/<th\b[^>]*>([\s\S]*?)<\/th>/gi)) {
|
||||||
|
headers.push(text(m[1] ?? ""));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Label/value rows: a <th> immediately followed by a <td>.
|
||||||
|
const pairs: Array<{ label: string; value: string }> = [];
|
||||||
|
const pairRe = /<th\b[^>]*>([\s\S]*?)<\/th>\s*<td\b[^>]*>([\s\S]*?)<\/td>/gi;
|
||||||
|
for (const m of body.matchAll(pairRe)) {
|
||||||
|
pairs.push({ label: text(m[1] ?? ""), value: text(m[2] ?? "") });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Row/cell grid, for column-oriented tables.
|
||||||
|
const rows: string[][] = [];
|
||||||
|
for (const tr of body.matchAll(/<tr\b[^>]*>([\s\S]*?)<\/tr>/gi)) {
|
||||||
|
const cells: string[] = [];
|
||||||
|
for (const cell of (tr[1] ?? "").matchAll(
|
||||||
|
/<t([dh])\b[^>]*>([\s\S]*?)<\/t\1>/gi,
|
||||||
|
)) {
|
||||||
|
cells.push(text(cell[2] ?? ""));
|
||||||
|
}
|
||||||
|
if (cells.length > 0) rows.push(cells);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { kind: "table", pairs, headers, rows };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when the document contains no nested <table> elements. */
|
||||||
|
export function assertFlatTables(rawHtml: string): boolean {
|
||||||
|
let depth = 0;
|
||||||
|
for (const m of rawHtml.matchAll(/<table\b[^>]*>|<\/table>/gi)) {
|
||||||
|
depth += m[0].startsWith("</") ? -1 : 1;
|
||||||
|
if (depth > 1) return false;
|
||||||
|
}
|
||||||
|
return depth === 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import {
|
||||||
|
parseMonthDayRange,
|
||||||
|
parseMonthDayYear,
|
||||||
|
parseSlashDateTimeRange,
|
||||||
|
} from "../src/ingest/dates.ts";
|
||||||
|
|
||||||
|
describe("parseMonthDayYear", () => {
|
||||||
|
test("parses a full date at day precision", () => {
|
||||||
|
expect(parseMonthDayYear("August 12, 2026")).toEqual({
|
||||||
|
iso: "2026-08-12T00:00:00.000Z",
|
||||||
|
precision: "day",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("accepts abbreviated months with and without a period", () => {
|
||||||
|
expect(parseMonthDayYear("Apr. 29, 2026")?.iso).toBe(
|
||||||
|
"2026-04-29T00:00:00.000Z",
|
||||||
|
);
|
||||||
|
expect(parseMonthDayYear("Sept 3, 2026")?.iso).toBe(
|
||||||
|
"2026-09-03T00:00:00.000Z",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns null rather than guessing a missing year", () => {
|
||||||
|
expect(parseMonthDayYear("August 12")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects impossible calendar dates", () => {
|
||||||
|
expect(parseMonthDayYear("February 30, 2026")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects an unknown month name", () => {
|
||||||
|
expect(parseMonthDayYear("Smarch 3, 2026")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseMonthDayRange", () => {
|
||||||
|
test("applies the stated year to both ends", () => {
|
||||||
|
expect(parseMonthDayRange("August 12 - September 21, 2026")).toEqual({
|
||||||
|
start: { iso: "2026-08-12T00:00:00.000Z", precision: "day" },
|
||||||
|
end: { iso: "2026-09-21T00:00:00.000Z", precision: "day" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rolls the start back a year when the range crosses New Year", () => {
|
||||||
|
// "December 28 - January 4, 2027" — the stated year belongs to the end.
|
||||||
|
const r = parseMonthDayRange("December 28 - January 4, 2027");
|
||||||
|
expect(r?.start.iso).toBe("2026-12-28T00:00:00.000Z");
|
||||||
|
expect(r?.end.iso).toBe("2027-01-04T00:00:00.000Z");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("handles en dash and abbreviated months", () => {
|
||||||
|
const r = parseMonthDayRange("Apr. 29 – May 13, 2026");
|
||||||
|
expect(r?.start.iso).toBe("2026-04-29T00:00:00.000Z");
|
||||||
|
expect(r?.end.iso).toBe("2026-05-13T00:00:00.000Z");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns null for a year-less range", () => {
|
||||||
|
// Game8 summary tables render "08/12 - 08/24". Guessing the year here is
|
||||||
|
// exactly the failure the product exists to prevent.
|
||||||
|
expect(parseMonthDayRange("08/12 - 08/24")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseSlashDateTimeRange", () => {
|
||||||
|
test("parses a timed range at exact precision", () => {
|
||||||
|
expect(
|
||||||
|
parseSlashDateTimeRange("2021/01/16 04:00 - 2021/01/31 03:59"),
|
||||||
|
).toEqual({
|
||||||
|
start: { iso: "2021-01-16T04:00:00.000Z", precision: "exact" },
|
||||||
|
end: { iso: "2021-01-31T03:59:00.000Z", precision: "exact" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ignores trailing prose after the range", () => {
|
||||||
|
const r = parseSlashDateTimeRange(
|
||||||
|
"2021/01/16 04:00 - 2021/01/31 03:59 Currently Unavailable",
|
||||||
|
);
|
||||||
|
expect(r?.end.iso).toBe("2021-01-31T03:59:00.000Z");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns null for non-date prose", () => {
|
||||||
|
expect(parseSlashDateTimeRange("Permanently Available")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user