From 18b9652aedf457a173b3d72561f54fcd50c1e8bd Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Sat, 15 Aug 2026 21:15:13 +0200 Subject: [PATCH] feat: sanitise every string a source publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scraped text becomes React content, JSON on disk and eventually SQLite rows, so it gets cleaned at one seam: the parse wrapper in toAdapter(). Every source passes through it, a source added tomorrow is covered without its author doing anything, and no parser can opt out — parsers stay pure readers of one site's markup. Removes script/style/comment content and residual tags, decodes entities to a fixed point so an encoded tag cannot resurrect in a later decoder, NFKC-normalises, strips control, zero-width and bidi-override characters (an RTL override visually spoofs a title), bounds each field to the cap the schema already declares, and requires sourceUrl to be absolute http(s). Three things it will not do: touch a date, drop an event it could clean instead, or repair in silence — every repair and drop is logged by default. Event IDs are localStorage keys, so an ID is recomputed only when a sanitised title actually changed and the ID was minted the standard way; all seven fixtures pass through unrepaired and byte-identical, which is the regression guard. Also stops decodeEntities throwing RangeError on an out-of-range numeric reference, which would have taken a whole source's events down. Co-Authored-By: Claude Opus 5 (1M context) --- src/ingest/adapters/index.ts | 16 +- src/ingest/html.ts | 22 +- src/ingest/sanitize.ts | 362 ++++++++++++++++++++++++ test/sanitize.test.ts | 528 +++++++++++++++++++++++++++++++++++ 4 files changed, 923 insertions(+), 5 deletions(-) create mode 100644 src/ingest/sanitize.ts create mode 100644 test/sanitize.test.ts diff --git a/src/ingest/adapters/index.ts b/src/ingest/adapters/index.ts index 6055f15..9a42ff1 100644 --- a/src/ingest/adapters/index.ts +++ b/src/ingest/adapters/index.ts @@ -1,6 +1,7 @@ import type { GachaEvent, GameId } from "../../shared/schema.ts"; import { mergeEvents, type MergeResult } from "../merge.ts"; import { parserById } from "../parsers/index.ts"; +import { sanitizeEvents } from "../sanitize.ts"; import { SIX_HOURS_MS, type Adapter, type ParseContext } from "./types.ts"; /** @@ -90,7 +91,20 @@ function toAdapter(spec: SourceSpec): Adapter { `${spec.id}: document does not match the '${parser.label}' template; the source has likely been redesigned`, ); } - return parser.parse(html, ctx); + + // The trust boundary. Everything a parser produces came from a page we do + // not control, and this is the one place every source passes through: + // `ADAPTERS` is built from `SOURCES` via this function, so a source added + // tomorrow is sanitised without its author doing anything, and a parser + // cannot opt out. Sanitising here rather than inside the parsers also + // keeps parsers what they are — pure readers of one site's markup. + // + // `sanitizeEvents` logs to console.warn by default, so a repaired or + // dropped event is never silent (CLAUDE.md § Silent drops). + return sanitizeEvents(parser.parse(html, ctx), { + sourceId: ctx.sourceId, + fallbackUrl: ctx.sourceUrl, + }).events; }, }; } diff --git a/src/ingest/html.ts b/src/ingest/html.ts index 3826929..f696f7c 100644 --- a/src/ingest/html.ts +++ b/src/ingest/html.ts @@ -23,13 +23,27 @@ const ENTITIES: Record = { rdquo: "”", }; +/** + * A numeric reference the page may or may not mean literally. + * + * Out-of-range and non-finite code points come back as the original text rather + * than throwing: `�` and `�` are junk a hostile or merely + * broken page can emit, and `String.fromCodePoint` throws RangeError on both. + * A parser crashing on one bad character would take a whole source's events + * down with it. + */ +function fromCodePoint(value: number, original: string): string { + if (!Number.isInteger(value) || value < 0 || value > 0x10ffff) return original; + return String.fromCodePoint(value); +} + export function decodeEntities(input: string): string { return input - .replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) => - String.fromCodePoint(parseInt(hex, 16)), + .replace(/&#x([0-9a-fA-F]+);/g, (whole, hex: string) => + fromCodePoint(parseInt(hex, 16), whole), ) - .replace(/&#(\d+);/g, (_, dec: string) => - String.fromCodePoint(parseInt(dec, 10)), + .replace(/&#(\d+);/g, (whole, dec: string) => + fromCodePoint(parseInt(dec, 10), whole), ) .replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole); } diff --git a/src/ingest/sanitize.ts b/src/ingest/sanitize.ts new file mode 100644 index 0000000..2622167 --- /dev/null +++ b/src/ingest/sanitize.ts @@ -0,0 +1,362 @@ +import { eventId, type GachaEvent } from "../shared/schema.ts"; +import { decodeEntities } from "./html.ts"; + +/** + * The trust boundary for scraped text. + * + * Every string on a `GachaEvent` starts life as bytes from a community wiki we + * do not control. Between the parser and validation it passes through here, so + * that what reaches the feed — and therefore React, `localStorage`, JSON on + * disk and eventually SQLite — is plain, bounded, normalised text. + * + * Three principles, in priority order: + * + * 1. **Never invent or alter a date.** Nothing in this module reads, writes or + * reformats a timestamp. Dates are the product's whole promise; the + * sanitiser's job stops at prose and URLs. + * 2. **Clean, do not drop.** A hostile title is truncated and stripped, not + * rejected — an event vanishing without a trace is the failure mode this + * codebase fears most (CLAUDE.md § Silent drops). The one unrecoverable + * case is a title that sanitises to nothing, and that emits a note the + * caller is expected to surface. + * 3. **Never throw on junk.** Malformed entities, lone surrogates, absurd code + * points and 5MB strings all have to come out the other side as a string. + * + * Dependency-free by design: no DOMPurify, no sanitize-html, no parse5. + */ + +/** + * Length caps, mirroring `title`/`summary` in `src/shared/schema.ts`. + * + * The schema stays the single source of truth — these exist so a hostile page + * is truncated *before* validation instead of failing it, and + * `test/sanitize.test.ts` asserts that a string of exactly this length is + * accepted by `GachaEvent` and one character more is not, so the two cannot + * drift apart unnoticed. + */ +export const LIMITS = { + title: 200, + summary: 500, + /** Not a schema cap: a defensive ceiling so a junk href cannot be a novel. */ + url: 2048, +} as const; + +/** How many decode/strip rounds before giving up and hard-scrubbing. */ +const MAX_PASSES = 5; + +/** + * Characters that are invisible, that control how surrounding text is + * *displayed*, or that are not legal text at all. + * + * The bidi overrides and isolates (U+202A–U+202E, U+2066–U+2069) matter most: + * they let a source render "Login Event" as something else entirely, or hide a + * suffix from a reader while it still lands in the title, the slug and the + * user's saved state. Zero-width characters do the same job more crudely and + * additionally let an attacker smuggle `&amp;lt;` past a naive decoder. + * + * Note ZWJ (U+200D) goes too, which splits multi-part emoji into their + * components. Our sources are English-language wikis; a mangled family emoji is + * a fair price for no invisible characters anywhere in a title. + */ +const INVISIBLE = + /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u00AD\u061C\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\uFEFF\uFFF9-\uFFFB\uFFFE\uFFFF]/g; + +/** Unicode tag characters (U+E0000–U+E007F) — invisible, as surrogate pairs. */ +const TAG_CHARS = /\uDB40[\uDC00-\uDC7F]/g; + +/** Half of a surrogate pair with no partner: not valid text, breaks JSON. */ +const LONE_SURROGATE = + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/g; + +/** Elements whose *content* is code, not text, and must go with the tag. */ +const CODE_BLOCKS = + /<(script|style|template|noscript|iframe|object|embed|svg|math)\b[\s\S]*?(?:<\/\1\s*>|$)/gi; + +const COMMENTS = /|$)||$)|<\?[\s\S]*?(?:\?>|$)/g; + +const TAG = /<[^>]*>/g; + +function stripInvisible(input: string): string { + return input + .replace(TAG_CHARS, "") + .replace(INVISIBLE, "") + .replace(LONE_SURROGATE, (m) => (m.length === 2 ? m[0] ?? "" : "")); +} + +function stripTags(input: string): string { + let out = input; + for (let i = 0; i < MAX_PASSES; i += 1) { + const next = out.replace(TAG, " "); + if (next === out) return out; + out = next; + } + return out; +} + +/** + * One round of neutralising. + * + * Order is deliberate. NFKC runs *first* because it turns look-alike forms into + * their canonical ones (U+FF1C FULLWIDTH LESS-THAN becomes `<`), and a + * normaliser running after the tag stripper would hand back live markup. + * Invisible characters go next so they cannot break up an entity or a tag name. + * Only then is anything decoded, and whatever the decode produced is stripped + * in the same round. + */ +function pass(input: string): string { + const normalised = stripInvisible(input.normalize("NFKC")); + const withoutCode = normalised.replace(CODE_BLOCKS, " ").replace(COMMENTS, " "); + return stripTags(decodeEntities(withoutCode)); +} + +/** + * Reduce text to a fixed point of `pass`. + * + * Running to a fixed point is what makes the whole module idempotent, and it is + * also the answer to double-encoding: `&lt;script>` decodes to + * `<script>` on the first round and to a tag on the second, which the second + * round then strips. Stopping after one decode would leave a string that a + * later `sanitizeText` — or any other decoder downstream — turns into markup. + */ +function toFixedPoint(input: string): string { + let out = input; + for (let i = 0; i < MAX_PASSES; i += 1) { + const next = pass(out); + if (next === out) return out; + out = next; + } + // Pathological input that keeps re-encoding itself — `&#38;#38;…` nested + // deeper than the round limit. Removing every `<`, `>` and `&` both kills any + // tag shape and guarantees the result is a fixed point (nothing left to + // decode), which is what keeps `sanitizeText` idempotent even here. + return out.replace(/[<>&]/g, " "); +} + +/** Collapse every run of whitespace — including NBSP and U+2028 — to one space. */ +function collapse(input: string): string { + return input + .replace(/[\s\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]+/g, " ") + .trim(); +} + +/** + * Cut to `max` characters at a word boundary, marking the cut with an ellipsis. + * + * Truncating beats rejecting: a real event with a bloated description is still + * a real event, and the user would rather see it than not. The result is always + * `<= max`, so a second pass never truncates again. + */ +function truncate(input: string, max: number): string { + if (input.length <= max) return input; + + let cut = input.slice(0, max - 1); + // Never end on half a surrogate pair. + if (/[\uD800-\uDBFF]$/.test(cut)) cut = cut.slice(0, -1); + + const lastSpace = cut.lastIndexOf(" "); + if (lastSpace > max * 0.6) cut = cut.slice(0, lastSpace); + + return `${cut.replace(/[\s,;:.–—-]+$/, "")}…`; +} + +export interface SanitizeTextOptions { + /** Hard cap; the result is never longer. Defaults to the summary cap. */ + maxLength?: number; +} + +/** + * Clean one string extracted from a source. + * + * Total: any input, including `null`, a number or a 5MB blob of markup, yields + * a string. Idempotent: `sanitizeText(sanitizeText(x)) === sanitizeText(x)`. + */ +export function sanitizeText( + input: unknown, + options: SanitizeTextOptions = {}, +): string { + // Anything that is not a primitive is not text a source stated; "" is the + // honest reading of it, and stringifying an object would invent content. + const raw = + typeof input === "string" + ? input + : typeof input === "number" || typeof input === "boolean" + ? String(input) + : ""; + const max = options.maxLength ?? LIMITS.summary; + return truncate(collapse(toFixedPoint(raw)), max); +} + +export interface SanitizeUrlOptions { + /** Resolves a relative href, exactly as `new URL(href, base)` would. */ + base?: string; +} + +/** + * Return `input` as an absolute http(s) URL, or null if it is not one. + * + * `sourceUrl` is rendered as an attribution link, so a `javascript:` or `data:` + * URL that reached the client would be a live XSS vector in an app that + * otherwise never handles untrusted URLs. Anything that is not plainly http(s) + * — including a URL carrying credentials, which is only ever a phishing shape — + * is refused, and the caller falls back to the source's registered URL. + */ +export function sanitizeUrl( + input: unknown, + options: SanitizeUrlOptions = {}, +): string | null { + const raw = sanitizeText(input, { maxLength: LIMITS.url }); + if (raw.length === 0) return null; + + let url: URL; + try { + url = options.base === undefined ? new URL(raw) : new URL(raw, options.base); + } catch { + return null; + } + + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username !== "" || url.password !== "") return null; + return url.toString(); +} + +export type NoteLevel = "repaired" | "dropped"; + +export interface SanitizeNote { + level: NoteLevel; + sourceId: string; + field: "title" | "summary" | "sourceUrl" | "id"; + /** Human-readable, already truncated — safe to print to a log. */ + message: string; +} + +export interface SanitizeEventOptions { + /** + * The source's registered URL. Known good (it came from `SOURCES`, not from + * the page), so it is the fallback when an event's own `sourceUrl` is junk — + * attribution to the right page beats discarding the event. + */ + fallbackUrl?: string; + /** Reported on every note, so a log line names the source that produced it. */ + sourceId?: string; +} + +export interface SanitizeEventResult { + /** Null only when the event could not be repaired into a publishable shape. */ + event: GachaEvent | null; + notes: SanitizeNote[]; +} + +/** + * Clean every source-derived string on one event. + * + * Timestamps, precisions, confidence, region data and `sourceId` are passed + * through untouched: the first three are numbers and dates this module has no + * business rewriting, and the last comes from our own registry rather than from + * the page. + * + * The `id` is only recomputed when sanitising actually changed the title *and* + * the incoming id was minted the standard way (`eventId(game, title, startsAt)`). + * That keeps two guarantees at once: an id never disagrees with the title it + * encodes, and — because sanitising a clean title is a no-op — no id in the + * current feed moves. Ids are localStorage keys; a gratuitous change there + * orphans completion marks with no server-side recovery. + */ +export function sanitizeEvent( + event: GachaEvent, + options: SanitizeEventOptions = {}, +): SanitizeEventResult { + const sourceId = options.sourceId ?? sanitizeText(event.sourceId, { maxLength: 120 }); + const notes: SanitizeNote[] = []; + const note = (level: NoteLevel, field: SanitizeNote["field"], message: string) => { + notes.push({ level, sourceId, field, message }); + }; + + const rawTitle = typeof event.title === "string" ? event.title : ""; + const title = sanitizeText(rawTitle, { maxLength: LIMITS.title }); + if (title.length === 0) { + note( + "dropped", + "title", + `title sanitised to nothing (raw: ${JSON.stringify(rawTitle.slice(0, 80))})`, + ); + return { event: null, notes }; + } + if (title !== rawTitle) { + note("repaired", "title", `${JSON.stringify(rawTitle.slice(0, 80))} → ${JSON.stringify(title)}`); + } + + let summary: string | null = null; + if (typeof event.summary === "string") { + const cleaned = sanitizeText(event.summary, { maxLength: LIMITS.summary }); + summary = cleaned.length === 0 ? null : cleaned; + if (cleaned !== event.summary) { + note("repaired", "summary", `summary cleaned (${event.summary.length} → ${cleaned.length} chars)`); + } + } + + const base = options.fallbackUrl; + const sourceUrl = + sanitizeUrl(event.sourceUrl, base === undefined ? {} : { base }) ?? + sanitizeUrl(base); + if (sourceUrl === null) { + note( + "dropped", + "sourceUrl", + `no usable http(s) source URL (raw: ${JSON.stringify(String(event.sourceUrl).slice(0, 120))})`, + ); + return { event: null, notes }; + } + if (sourceUrl !== event.sourceUrl) { + note("repaired", "sourceUrl", `${JSON.stringify(String(event.sourceUrl).slice(0, 120))} → ${JSON.stringify(sourceUrl)}`); + } + + let id = event.id; + if (title !== rawTitle && id === eventId(event.game, rawTitle, event.startsAt)) { + id = eventId(event.game, title, event.startsAt); + if (id !== event.id) { + note("repaired", "id", `${event.id} → ${id} (title was sanitised)`); + } + } + + return { event: { ...event, id, title, summary, sourceUrl }, notes }; +} + +export interface SanitizeEventsResult { + events: GachaEvent[]; + notes: SanitizeNote[]; +} + +export interface SanitizeEventsOptions extends SanitizeEventOptions { + /** + * Where notes go. Defaults to `console.warn`, on purpose: a dropped event + * must never be silent, and defaulting to a no-op would make silence the + * behaviour a future caller gets for free. + */ + onNote?: (note: SanitizeNote) => void; +} + +/** Sanitise a parser's whole output. Order is preserved; drops are reported. */ +export function sanitizeEvents( + events: readonly GachaEvent[], + options: SanitizeEventsOptions = {}, +): SanitizeEventsResult { + const onNote = options.onNote ?? defaultReporter; + const kept: GachaEvent[] = []; + const notes: SanitizeNote[] = []; + + for (const event of events) { + const result = sanitizeEvent(event, options); + for (const n of result.notes) { + notes.push(n); + onNote(n); + } + if (result.event !== null) kept.push(result.event); + } + + return { events: kept, notes }; +} + +function defaultReporter(note: SanitizeNote): void { + const prefix = note.level === "dropped" ? "! dropped" : " repaired"; + console.warn(`${prefix} ${note.sourceId} ${note.field}: ${note.message}`); +} diff --git a/test/sanitize.test.ts b/test/sanitize.test.ts new file mode 100644 index 0000000..03c1c31 --- /dev/null +++ b/test/sanitize.test.ts @@ -0,0 +1,528 @@ +import { describe, expect, test } from "bun:test"; +import { adapterById, ADAPTERS } from "../src/ingest/adapters/index.ts"; +import { parserById } from "../src/ingest/parsers/index.ts"; +import { + LIMITS, + sanitizeEvent, + sanitizeEvents, + sanitizeText, + sanitizeUrl, + type SanitizeNote, +} from "../src/ingest/sanitize.ts"; +import { eventId, GachaEvent } from "../src/shared/schema.ts"; + +const NOW = "2026-08-14T00:00:00.000Z"; + +function event(overrides: Partial = {}): GachaEvent { + return { + id: "genshin:test-event:2026-08-12", + game: "genshin", + title: "Test Event", + type: "other", + summary: null, + startsAt: "2026-08-12T00:00:00.000Z", + startPrecision: "day", + endsAt: "2026-08-24T00:00:00.000Z", + endPrecision: "day", + regionScoped: false, + regionEnds: null, + sourceUrl: "https://example.test/a", + sourceId: "source-a", + status: "published", + confidence: 0.9, + extractionMethod: "parser", + version: 1, + firstSeenAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +/** Collects notes instead of printing them, so a test can assert on them. */ +function collector() { + const notes: SanitizeNote[] = []; + return { notes, onNote: (n: SanitizeNote) => void notes.push(n) }; +} + +/** + * Everything hostile this module claims to handle, in one list. + * + * Reused by the idempotency and the no-throw properties below: any case added + * here is automatically held to both. + */ +const HOSTILE: Array<[label: string, input: string]> = [ + ["bare script", `Windblume Festival`], + ["style block", `Windblume`], + ["nested tags", `
Windblume
`], + ["malformed tag", `<`], + ["unclosed tag", `Windblume alert(1) -->blume`], + ["entity-encoded tag", `<script>alert(1)</script>`], + ["double-encoded tag", `&lt;script&gt;alert(1)&lt;/script&gt;`], + ["numeric entity tag", `<script>alert(1)</script>`], + ["hex entity tag", `<script>alert(1)</script>`], + ["fullwidth tag", `＜script＞alert(1)＜/script＞`], + ["zero-width in entity", `&am\u200bp;lt;script>`], + ["deep entity nesting", `&#38;#38;#38;#38;#38;#38;lt;script>`], + ["rtl override", `Login \u202eEvent\u202c`], + ["bidi isolates", `\u2066Free\u2069\u2067Primogems\u2069`], + ["control characters", `Wind\u0000blume\u0007 Fest\u001bival`], + ["lone high surrogate", `Windblume \ud800`], + ["lone low surrogate", `\udc00 Windblume`], + ["out-of-range code point", `Windblume � �`], + ["absurdly long", `A${"b".repeat(50_000)}`], + ["whitespace storm", `\n\n Wind\t\t blume  \u3000 Festival \n`], + ["empty", ``], + ["only markup", `
`], +]; + +describe("sanitizeText — markup", () => { + test("strips tags, keeping the text between them", () => { + expect(sanitizeText(`
Windblume Festival
`)).toBe( + "Wind blume Festival", + ); + }); + + test("removes script and style content, not just the tags", () => { + expect(sanitizeText(`Windblume`)).toBe("Windblume"); + expect(sanitizeText(`Windblume`)).toBe( + "Windblume", + ); + expect(sanitizeText(`Windblume`)).toBe( + "Windblume", + ); + }); + + test("removes comments and their contents", () => { + expect(sanitizeText(`Windblume`)).toBe( + "Wind blume", + ); + }); + + test("leaves no tag-shaped substring, however malformed the input", () => { + for (const [label, input] of HOSTILE) { + expect(sanitizeText(input), label).not.toMatch(/<[^>]*>/); + } + }); + + test("an unclosed tag does not swallow the rest of the string", () => { + // The tag itself goes, but the words around it survive — a source that + // forgets a `>` should cost us markup, not an event. + expect(sanitizeText(`Windblume Festival`)).toBe("Windblume Festival"); + }); +}); + +describe("sanitizeText — entities", () => { + test("decodes ordinary entities", () => { + expect(sanitizeText(`Tea & Cakes – Act II`)).toBe( + "Tea & Cakes – Act II", + ); + }); + + test("an entity-encoded tag never becomes a live tag", () => { + expect(sanitizeText(`<script>alert(1)</script>`)).toBe("alert(1)"); + }); + + test("double-encoding does not survive one decode into markup", () => { + // The dangerous case: one decode yields `<script>`, which a second + // decoder anywhere downstream would turn into a tag. Decoding to a fixed + // point and stripping each round means the output is inert for every later + // reader too. + const out = sanitizeText(`&lt;script&gt;alert(1)&lt;/script&gt;`); + expect(out).not.toContain("<"); + expect(out).not.toContain("<"); + expect(sanitizeText(out)).toBe(out); + }); + + test("numeric, hex and fullwidth spellings of a tag are all neutralised", () => { + for (const input of [ + `<script>alert(1)</script>`, + `<script>alert(1)</script>`, + `＜script＞alert(1)＜/script＞`, + ]) { + expect(sanitizeText(input)).not.toMatch(/<[^>]*>/); + } + }); + + test("an out-of-range code point is left alone rather than throwing", () => { + expect(() => sanitizeText(`� �`)).not.toThrow(); + expect(sanitizeText(`Windblume �`)).toBe("Windblume �"); + }); +}); + +describe("sanitizeText — unicode", () => { + test("normalises to NFKC", () => { + // Compatibility forms are how a source spoofs a title that reads the same. + expect(sanitizeText("Ｗｉｎｄｂｌｕｍｅ")).toBe("Windblume"); + expect(sanitizeText("éclair")).toBe("éclair"); + }); + + test("strips control characters", () => { + expect(sanitizeText("Wind\u0000blume\u0007 Fest\u001bival")).toBe( + "Windblume Festival", + ); + }); + + test("strips zero-width characters", () => { + expect(sanitizeText("Wind\u200bblume\u200c\u200d\ufeff")).toBe("Windblume"); + }); + + test("strips bidi overrides that would spoof a title", () => { + const spoofed = "Login \u202ednellA\u202c"; + const clean = sanitizeText(spoofed); + expect(clean).not.toMatch(/[\u202a-\u202e\u2066-\u2069]/); + expect(clean).toBe("Login dnellA"); + }); + + test("drops unpaired surrogates but keeps real astral characters", () => { + expect(sanitizeText("Windblume \ud800")).toBe("Windblume"); + expect(sanitizeText("\udc00 Windblume")).toBe("Windblume"); + expect(sanitizeText("Windblume 🎉")).toBe("Windblume 🎉"); + }); + + test("collapses every flavour of whitespace to single spaces", () => { + expect(sanitizeText("\n\n Wind\t\t blume  \u3000 Festival \n")).toBe( + "Wind blume Festival", + ); + }); +}); + +describe("sanitizeText — length", () => { + test("truncates to the cap rather than discarding the value", () => { + const long = `Windblume ${"a".repeat(5_000)}`; + const title = sanitizeText(long, { maxLength: LIMITS.title }); + expect(title.length).toBeLessThanOrEqual(LIMITS.title); + expect(title.startsWith("Windblume")).toBe(true); + expect(title.endsWith("…")).toBe(true); + }); + + test("cuts at a word boundary when there is one", () => { + const words = "Windblume Festival Returns To Mondstadt In Full Bloom"; + expect(sanitizeText(words, { maxLength: 20 })).toBe("Windblume Festival…"); + }); + + test("a value at the cap is left exactly as it is", () => { + const exact = "a".repeat(LIMITS.title); + expect(sanitizeText(exact, { maxLength: LIMITS.title })).toBe(exact); + }); + + test("the caps match what the schema will accept", () => { + // The schema is the single source of truth; LIMITS only mirrors it. If + // someone widens or narrows `title`/`summary` there, this fails here. + const ok = GachaEvent.safeParse( + event({ title: "a".repeat(LIMITS.title), summary: "b".repeat(LIMITS.summary) }), + ); + expect(ok.success).toBe(true); + + expect(GachaEvent.safeParse(event({ title: "a".repeat(LIMITS.title + 1) })).success).toBe(false); + expect(GachaEvent.safeParse(event({ summary: "b".repeat(LIMITS.summary + 1) })).success).toBe(false); + }); +}); + +describe("sanitizeText — totality", () => { + test("is idempotent for every hostile input", () => { + for (const [label, input] of HOSTILE) { + const once = sanitizeText(input); + expect(sanitizeText(once), label).toBe(once); + } + }); + + test("never throws on junk", () => { + for (const [label, input] of HOSTILE) { + expect(() => sanitizeText(input), label).not.toThrow(); + } + for (const junk of [null, undefined, 42, true, {}, [], Symbol("x")]) { + expect(() => sanitizeText(junk)).not.toThrow(); + } + expect(sanitizeText(null)).toBe(""); + expect(sanitizeText(undefined)).toBe(""); + expect(sanitizeText({})).toBe(""); + }); +}); + +describe("sanitizeUrl", () => { + test("keeps http and https", () => { + expect(sanitizeUrl("https://game8.co/games/Genshin-Impact/archives/301601")).toBe( + "https://game8.co/games/Genshin-Impact/archives/301601", + ); + expect(sanitizeUrl("http://example.test/a")).toBe("http://example.test/a"); + }); + + test("rejects javascript:, data: and every other scheme", () => { + for (const bad of [ + "javascript:alert(1)", + "JaVaScRiPt:alert(1)", + "java\tscript:alert(1)", + "javascript:alert(1)", + "data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==", + "vbscript:msgbox(1)", + "file:///etc/passwd", + "about:blank", + ]) { + expect(sanitizeUrl(bad), bad).toBeNull(); + } + }); + + test("rejects a URL carrying credentials", () => { + expect(sanitizeUrl("https://user:pass@game8.co/x")).toBeNull(); + }); + + test("rejects junk instead of throwing", () => { + for (const junk of ["", " ", "not a url", null, undefined, 42, {}]) { + expect(() => sanitizeUrl(junk)).not.toThrow(); + expect(sanitizeUrl(junk)).toBeNull(); + } + }); + + test("resolves a relative href against the source page", () => { + expect(sanitizeUrl("/wiki/Event", { base: "https://endfield.wiki.gg/wiki/Home" })).toBe( + "https://endfield.wiki.gg/wiki/Event", + ); + }); + + test("a relative href cannot smuggle in another scheme via the base", () => { + expect( + sanitizeUrl("javascript:alert(1)", { base: "https://endfield.wiki.gg/wiki/Home" }), + ).toBeNull(); + }); +}); + +describe("sanitizeEvent", () => { + test("leaves a clean event completely alone", () => { + const clean = event({ + id: eventId("genshin", "Windblume Festival", "2026-08-12T00:00:00.000Z"), + title: "Windblume Festival", + summary: "A festival in Mondstadt.", + }); + const { event: out, notes } = sanitizeEvent(clean); + expect(out).toEqual(clean); + expect(notes).toEqual([]); + }); + + test("never touches timestamps, precision or confidence", () => { + const dirty = event({ title: "Windblume", summary: "Blurb" }); + const { event: out } = sanitizeEvent(dirty); + expect(out?.startsAt).toBe(dirty.startsAt); + expect(out?.endsAt).toBe(dirty.endsAt); + expect(out?.startPrecision).toBe(dirty.startPrecision); + expect(out?.endPrecision).toBe(dirty.endPrecision); + expect(out?.confidence).toBe(dirty.confidence); + expect(out?.regionEnds).toBe(dirty.regionEnds); + }); + + test("cleans a spoofed title and keeps the id agreeing with it", () => { + const startsAt = "2026-08-12T00:00:00.000Z"; + const rawTitle = "Login \u202eEvent\u202c"; + const dirty = event({ + id: eventId("genshin", rawTitle, startsAt), + title: rawTitle, + startsAt, + }); + + // The override is what made this render as something other than its + // characters; removing it leaves the honest text, and the id follows. + const { event: out, notes } = sanitizeEvent(dirty, { sourceId: "source-a" }); + expect(out?.title).toBe("Login Event"); + expect(out?.id).toBe(eventId("genshin", "Login Event", startsAt)); + expect(notes.every((n) => n.level === "repaired")).toBe(true); + + // No id note here, and that is the point: `slugify` already drops + // characters like these, so cleaning the title moved nothing a user has + // saved state under. + expect(notes.map((n) => n.field)).toEqual(["title"]); + }); + + test("recomputes the id when cleaning genuinely changes the slug", () => { + const startsAt = "2026-08-12T00:00:00.000Z"; + const rawTitle = "Windblume Festival"; + const dirty = event({ + id: eventId("genshin", rawTitle, startsAt), + title: rawTitle, + startsAt, + }); + + const { event: out, notes } = sanitizeEvent(dirty); + expect(out?.title).toBe("Windblume Festival"); + expect(out?.id).toBe("genshin:windblume-festival:2026-08-12"); + expect(notes.some((n) => n.field === "id")).toBe(true); + }); + + test("leaves an id alone when it was not minted from the title", () => { + // Reconciliation keeps an existing id when a wiki renames an event + // (docs/INGESTION.md § Stage 5). Sanitising must not undo that. + const dirty = event({ id: "genshin:kept-across-a-rename:2026-08-12", title: "New Name" }); + const { event: out } = sanitizeEvent(dirty); + expect(out?.title).toBe("New Name"); + expect(out?.id).toBe("genshin:kept-across-a-rename:2026-08-12"); + }); + + test("truncates a hostile title instead of losing the event", () => { + const dirty = event({ title: `Windblume ${"a".repeat(10_000)}` }); + const { event: out } = sanitizeEvent(dirty); + expect(out).not.toBeNull(); + expect((out?.title ?? "").length).toBeLessThanOrEqual(LIMITS.title); + expect(GachaEvent.safeParse(out).success).toBe(true); + }); + + test("an empty summary becomes null rather than an empty string", () => { + const { event: out } = sanitizeEvent(event({ summary: " " })); + expect(out?.summary).toBeNull(); + }); + + test("falls back to the source's own URL when the scraped one is hostile", () => { + const dirty = event({ sourceUrl: "javascript:alert(1)" }); + const { event: out, notes } = sanitizeEvent(dirty, { + fallbackUrl: "https://game8.co/games/Genshin-Impact/archives/301601", + sourceId: "genshin-game8-events", + }); + expect(out?.sourceUrl).toBe("https://game8.co/games/Genshin-Impact/archives/301601"); + expect(notes.some((n) => n.field === "sourceUrl" && n.level === "repaired")).toBe(true); + }); + + test("drops an event only when nothing is left to publish, and says so", () => { + const gone = event({ title: "" }); + const { event: out, notes } = sanitizeEvent(gone, { sourceId: "source-a" }); + expect(out).toBeNull(); + expect(notes).toHaveLength(1); + expect(notes[0]?.level).toBe("dropped"); + expect(notes[0]?.field).toBe("title"); + expect(notes[0]?.sourceId).toBe("source-a"); + }); + + test("drops an event with no usable URL and no fallback, and says so", () => { + const { event: out, notes } = sanitizeEvent(event({ sourceUrl: "javascript:alert(1)" })); + expect(out).toBeNull(); + expect(notes.some((n) => n.level === "dropped" && n.field === "sourceUrl")).toBe(true); + }); + + test("output always satisfies the schema", () => { + for (const [label, input] of HOSTILE) { + const { event: out } = sanitizeEvent( + event({ title: input, summary: input, sourceUrl: input }), + { fallbackUrl: "https://example.test/a" }, + ); + if (out === null) continue; + expect(GachaEvent.safeParse(out).success, label).toBe(true); + } + }); +}); + +describe("sanitizeEvents", () => { + test("keeps order and reports every note", () => { + const { notes, onNote } = collector(); + const { events } = sanitizeEvents( + [ + event({ id: "genshin:a:2026-08-12", title: "Alpha" }), + event({ id: "genshin:b:2026-08-12", title: "Beta" }), + event({ id: "genshin:c:2026-08-12", title: "Gamma" }), + ], + { onNote, sourceId: "source-a" }, + ); + + expect(events.map((e) => e.title)).toEqual(["Alpha", "Beta", "Gamma"]); + expect(notes).toHaveLength(1); + expect(notes[0]?.field).toBe("title"); + }); + + test("a dropped event never disappears silently", () => { + const { notes, onNote } = collector(); + const { events } = sanitizeEvents( + [event({ title: "Alpha" }), event({ title: "" })], + { onNote }, + ); + + expect(events).toHaveLength(1); + expect(notes.filter((n) => n.level === "dropped")).toHaveLength(1); + }); + + test("siblings survive a dropped event", () => { + const { onNote } = collector(); + const { events } = sanitizeEvents( + [event({ title: "" }), event({ title: "Beta" })], + { onNote }, + ); + expect(events.map((e) => e.title)).toEqual(["Beta"]); + }); +}); + +describe("the adapter seam", () => { + /** + * A minimal page in Game8's shape, carrying everything a hostile source could + * put in a title. The point is not the parser — it is that `adapter.parse` + * cannot return unsanitised events, whichever parser produced them. + */ + const HOSTILE_PAGE = ` +

Current Events

+

Windblume \u202eFestival\u202c\u200b & Friends

+ + + +
Event StartAugust 12, 2026
Event EndAugust 24, 2026
+

A blurb with <script>alert(1)</script> in it.

+ `; + + test("events coming out of an adapter are already sanitised", () => { + const adapter = adapterById("genshin-game8-events"); + if (adapter === undefined) throw new Error("no adapter"); + + const events = adapter.parse(HOSTILE_PAGE, { + now: NOW, + sourceUrl: adapter.url, + sourceId: adapter.id, + game: adapter.game, + }); + + expect(events).toHaveLength(1); + const parsed = events[0]; + expect(parsed?.title).toBe("Windblume Festival & Friends"); + expect(parsed?.title).not.toMatch(/[\u200b-\u200f\u202a-\u202e]/); + expect(parsed?.summary ?? "").not.toMatch(/<[^>]*>/); + expect(parsed?.sourceUrl).toBe(adapter.url); + // Dates are the one thing sanitising must never touch. + expect(parsed?.startsAt).toBe("2026-08-12T00:00:00.000Z"); + expect(parsed?.endsAt).toBe("2026-08-24T00:00:00.000Z"); + // The id agrees with the title the user actually sees. + expect(parsed?.id).toBe( + eventId("genshin", parsed?.title ?? "", parsed?.startsAt ?? ""), + ); + expect(GachaEvent.safeParse(parsed).success).toBe(true); + }); + + /** + * The guard that matters most. + * + * Event ids are localStorage keys, so sanitising must be a *no-op* on the + * pages we actually parse — if it ever starts repairing a real title, the id + * derived from that title can move and every completion mark saved under the + * old one is orphaned with no server-side recovery. Running the parsers + * directly (before the adapter seam cleans anything) and asserting zero notes + * is what makes that visible the moment it changes. + */ + test("real fixtures need no repair at all", async () => { + for (const adapter of ADAPTERS) { + const parser = parserById(adapter.parserId); + if (parser === undefined) throw new Error(`no parser ${adapter.parserId}`); + + const pattern = `fixtures/${adapter.game}/${adapter.parserId}-*.html`; + const file = [...new Bun.Glob(pattern).scanSync(".")].sort().at(-1); + if (file === undefined) throw new Error(`no fixture for ${adapter.id}`); + + const html = await Bun.file(file).text(); + const raw = parser.parse(html, { + now: NOW, + sourceUrl: adapter.url, + sourceId: adapter.id, + game: adapter.game, + }); + + const { notes, events } = sanitizeEvents(raw, { + sourceId: adapter.id, + fallbackUrl: adapter.url, + onNote: () => {}, + }); + + expect(notes.map((n) => `${n.field}: ${n.message}`), adapter.id).toEqual([]); + expect(events.map((e) => e.id), adapter.id).toEqual(raw.map((e) => e.id)); + } + }); +});