feat: sanitise every string a source publishes

Scraped text becomes React content, JSON on disk and eventually SQLite
rows, so it gets cleaned at one seam: the parse wrapper in toAdapter().
Every source passes through it, a source added tomorrow is covered
without its author doing anything, and no parser can opt out — parsers
stay pure readers of one site's markup.

Removes script/style/comment content and residual tags, decodes entities
to a fixed point so an encoded tag cannot resurrect in a later decoder,
NFKC-normalises, strips control, zero-width and bidi-override characters
(an RTL override visually spoofs a title), bounds each field to the cap
the schema already declares, and requires sourceUrl to be absolute
http(s).

Three things it will not do: touch a date, drop an event it could clean
instead, or repair in silence — every repair and drop is logged by
default. Event IDs are localStorage keys, so an ID is recomputed only
when a sanitised title actually changed and the ID was minted the
standard way; all seven fixtures pass through unrepaired and
byte-identical, which is the regression guard.

Also stops decodeEntities throwing RangeError on an out-of-range
numeric reference, which would have taken a whole source's events down.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Lucas Winther
2026-08-15 21:15:13 +02:00
co-authored by Claude Opus 5
parent 2b9338a8b0
commit 18b9652aed
4 changed files with 923 additions and 5 deletions
+15 -1
View File
@@ -1,6 +1,7 @@
import type { GachaEvent, GameId } from "../../shared/schema.ts";
import { mergeEvents, type MergeResult } from "../merge.ts";
import { parserById } from "../parsers/index.ts";
import { sanitizeEvents } from "../sanitize.ts";
import { SIX_HOURS_MS, type Adapter, type ParseContext } from "./types.ts";
/**
@@ -90,7 +91,20 @@ function toAdapter(spec: SourceSpec): Adapter {
`${spec.id}: document does not match the '${parser.label}' template; the source has likely been redesigned`,
);
}
return parser.parse(html, ctx);
// The trust boundary. Everything a parser produces came from a page we do
// not control, and this is the one place every source passes through:
// `ADAPTERS` is built from `SOURCES` via this function, so a source added
// tomorrow is sanitised without its author doing anything, and a parser
// cannot opt out. Sanitising here rather than inside the parsers also
// keeps parsers what they are — pure readers of one site's markup.
//
// `sanitizeEvents` logs to console.warn by default, so a repaired or
// dropped event is never silent (CLAUDE.md § Silent drops).
return sanitizeEvents(parser.parse(html, ctx), {
sourceId: ctx.sourceId,
fallbackUrl: ctx.sourceUrl,
}).events;
},
};
}
+18 -4
View File
@@ -23,13 +23,27 @@ const ENTITIES: Record<string, string> = {
rdquo: "”",
};
/**
* A numeric reference the page may or may not mean literally.
*
* Out-of-range and non-finite code points come back as the original text rather
* than throwing: `&#1114112;` and `&#x110000;` are junk a hostile or merely
* broken page can emit, and `String.fromCodePoint` throws RangeError on both.
* A parser crashing on one bad character would take a whole source's events
* down with it.
*/
function fromCodePoint(value: number, original: string): string {
if (!Number.isInteger(value) || value < 0 || value > 0x10ffff) return original;
return String.fromCodePoint(value);
}
export function decodeEntities(input: string): string {
return input
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) =>
String.fromCodePoint(parseInt(hex, 16)),
.replace(/&#x([0-9a-fA-F]+);/g, (whole, hex: string) =>
fromCodePoint(parseInt(hex, 16), whole),
)
.replace(/&#(\d+);/g, (_, dec: string) =>
String.fromCodePoint(parseInt(dec, 10)),
.replace(/&#(\d+);/g, (whole, dec: string) =>
fromCodePoint(parseInt(dec, 10), whole),
)
.replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole);
}
+362
View File
@@ -0,0 +1,362 @@
import { eventId, type GachaEvent } from "../shared/schema.ts";
import { decodeEntities } from "./html.ts";
/**
* The trust boundary for scraped text.
*
* Every string on a `GachaEvent` starts life as bytes from a community wiki we
* do not control. Between the parser and validation it passes through here, so
* that what reaches the feed — and therefore React, `localStorage`, JSON on
* disk and eventually SQLite — is plain, bounded, normalised text.
*
* Three principles, in priority order:
*
* 1. **Never invent or alter a date.** Nothing in this module reads, writes or
* reformats a timestamp. Dates are the product's whole promise; the
* sanitiser's job stops at prose and URLs.
* 2. **Clean, do not drop.** A hostile title is truncated and stripped, not
* rejected — an event vanishing without a trace is the failure mode this
* codebase fears most (CLAUDE.md § Silent drops). The one unrecoverable
* case is a title that sanitises to nothing, and that emits a note the
* caller is expected to surface.
* 3. **Never throw on junk.** Malformed entities, lone surrogates, absurd code
* points and 5MB strings all have to come out the other side as a string.
*
* Dependency-free by design: no DOMPurify, no sanitize-html, no parse5.
*/
/**
* Length caps, mirroring `title`/`summary` in `src/shared/schema.ts`.
*
* The schema stays the single source of truth — these exist so a hostile page
* is truncated *before* validation instead of failing it, and
* `test/sanitize.test.ts` asserts that a string of exactly this length is
* accepted by `GachaEvent` and one character more is not, so the two cannot
* drift apart unnoticed.
*/
export const LIMITS = {
title: 200,
summary: 500,
/** Not a schema cap: a defensive ceiling so a junk href cannot be a novel. */
url: 2048,
} as const;
/** How many decode/strip rounds before giving up and hard-scrubbing. */
const MAX_PASSES = 5;
/**
* Characters that are invisible, that control how surrounding text is
* *displayed*, or that are not legal text at all.
*
* The bidi overrides and isolates (U+202AU+202E, U+2066U+2069) matter most:
* they let a source render "Login Event" as something else entirely, or hide a
* suffix from a reader while it still lands in the title, the slug and the
* user's saved state. Zero-width characters do the same job more crudely and
* additionally let an attacker smuggle `&am<ZWSP>p;lt;` past a naive decoder.
*
* Note ZWJ (U+200D) goes too, which splits multi-part emoji into their
* components. Our sources are English-language wikis; a mangled family emoji is
* a fair price for no invisible characters anywhere in a title.
*/
const INVISIBLE =
/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u00AD\u061C\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\uFEFF\uFFF9-\uFFFB\uFFFE\uFFFF]/g;
/** Unicode tag characters (U+E0000U+E007F) — invisible, as surrogate pairs. */
const TAG_CHARS = /\uDB40[\uDC00-\uDC7F]/g;
/** Half of a surrogate pair with no partner: not valid text, breaks JSON. */
const LONE_SURROGATE =
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/g;
/** Elements whose *content* is code, not text, and must go with the tag. */
const CODE_BLOCKS =
/<(script|style|template|noscript|iframe|object|embed|svg|math)\b[\s\S]*?(?:<\/\1\s*>|$)/gi;
const COMMENTS = /<!--[\s\S]*?(?:-->|$)|<!\[CDATA\[[\s\S]*?(?:\]\]>|$)|<\?[\s\S]*?(?:\?>|$)/g;
const TAG = /<[^>]*>/g;
function stripInvisible(input: string): string {
return input
.replace(TAG_CHARS, "")
.replace(INVISIBLE, "")
.replace(LONE_SURROGATE, (m) => (m.length === 2 ? m[0] ?? "" : ""));
}
function stripTags(input: string): string {
let out = input;
for (let i = 0; i < MAX_PASSES; i += 1) {
const next = out.replace(TAG, " ");
if (next === out) return out;
out = next;
}
return out;
}
/**
* One round of neutralising.
*
* Order is deliberate. NFKC runs *first* because it turns look-alike forms into
* their canonical ones (U+FF1C FULLWIDTH LESS-THAN becomes `<`), and a
* normaliser running after the tag stripper would hand back live markup.
* Invisible characters go next so they cannot break up an entity or a tag name.
* Only then is anything decoded, and whatever the decode produced is stripped
* in the same round.
*/
function pass(input: string): string {
const normalised = stripInvisible(input.normalize("NFKC"));
const withoutCode = normalised.replace(CODE_BLOCKS, " ").replace(COMMENTS, " ");
return stripTags(decodeEntities(withoutCode));
}
/**
* Reduce text to a fixed point of `pass`.
*
* Running to a fixed point is what makes the whole module idempotent, and it is
* also the answer to double-encoding: `&amp;lt;script&gt;` decodes to
* `&lt;script>` on the first round and to a tag on the second, which the second
* round then strips. Stopping after one decode would leave a string that a
* later `sanitizeText` — or any other decoder downstream — turns into markup.
*/
function toFixedPoint(input: string): string {
let out = input;
for (let i = 0; i < MAX_PASSES; i += 1) {
const next = pass(out);
if (next === out) return out;
out = next;
}
// Pathological input that keeps re-encoding itself — `&#38;#38;#38;…` nested
// deeper than the round limit. Removing every `<`, `>` and `&` both kills any
// tag shape and guarantees the result is a fixed point (nothing left to
// decode), which is what keeps `sanitizeText` idempotent even here.
return out.replace(/[<>&]/g, " ");
}
/** Collapse every run of whitespace — including NBSP and U+2028 — to one space. */
function collapse(input: string): string {
return input
.replace(/[\s\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]+/g, " ")
.trim();
}
/**
* Cut to `max` characters at a word boundary, marking the cut with an ellipsis.
*
* Truncating beats rejecting: a real event with a bloated description is still
* a real event, and the user would rather see it than not. The result is always
* `<= max`, so a second pass never truncates again.
*/
function truncate(input: string, max: number): string {
if (input.length <= max) return input;
let cut = input.slice(0, max - 1);
// Never end on half a surrogate pair.
if (/[\uD800-\uDBFF]$/.test(cut)) cut = cut.slice(0, -1);
const lastSpace = cut.lastIndexOf(" ");
if (lastSpace > max * 0.6) cut = cut.slice(0, lastSpace);
return `${cut.replace(/[\s,;:.–—-]+$/, "")}`;
}
export interface SanitizeTextOptions {
/** Hard cap; the result is never longer. Defaults to the summary cap. */
maxLength?: number;
}
/**
* Clean one string extracted from a source.
*
* Total: any input, including `null`, a number or a 5MB blob of markup, yields
* a string. Idempotent: `sanitizeText(sanitizeText(x)) === sanitizeText(x)`.
*/
export function sanitizeText(
input: unknown,
options: SanitizeTextOptions = {},
): string {
// Anything that is not a primitive is not text a source stated; "" is the
// honest reading of it, and stringifying an object would invent content.
const raw =
typeof input === "string"
? input
: typeof input === "number" || typeof input === "boolean"
? String(input)
: "";
const max = options.maxLength ?? LIMITS.summary;
return truncate(collapse(toFixedPoint(raw)), max);
}
export interface SanitizeUrlOptions {
/** Resolves a relative href, exactly as `new URL(href, base)` would. */
base?: string;
}
/**
* Return `input` as an absolute http(s) URL, or null if it is not one.
*
* `sourceUrl` is rendered as an attribution link, so a `javascript:` or `data:`
* URL that reached the client would be a live XSS vector in an app that
* otherwise never handles untrusted URLs. Anything that is not plainly http(s)
* — including a URL carrying credentials, which is only ever a phishing shape —
* is refused, and the caller falls back to the source's registered URL.
*/
export function sanitizeUrl(
input: unknown,
options: SanitizeUrlOptions = {},
): string | null {
const raw = sanitizeText(input, { maxLength: LIMITS.url });
if (raw.length === 0) return null;
let url: URL;
try {
url = options.base === undefined ? new URL(raw) : new URL(raw, options.base);
} catch {
return null;
}
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
if (url.username !== "" || url.password !== "") return null;
return url.toString();
}
export type NoteLevel = "repaired" | "dropped";
export interface SanitizeNote {
level: NoteLevel;
sourceId: string;
field: "title" | "summary" | "sourceUrl" | "id";
/** Human-readable, already truncated — safe to print to a log. */
message: string;
}
export interface SanitizeEventOptions {
/**
* The source's registered URL. Known good (it came from `SOURCES`, not from
* the page), so it is the fallback when an event's own `sourceUrl` is junk —
* attribution to the right page beats discarding the event.
*/
fallbackUrl?: string;
/** Reported on every note, so a log line names the source that produced it. */
sourceId?: string;
}
export interface SanitizeEventResult {
/** Null only when the event could not be repaired into a publishable shape. */
event: GachaEvent | null;
notes: SanitizeNote[];
}
/**
* Clean every source-derived string on one event.
*
* Timestamps, precisions, confidence, region data and `sourceId` are passed
* through untouched: the first three are numbers and dates this module has no
* business rewriting, and the last comes from our own registry rather than from
* the page.
*
* The `id` is only recomputed when sanitising actually changed the title *and*
* the incoming id was minted the standard way (`eventId(game, title, startsAt)`).
* That keeps two guarantees at once: an id never disagrees with the title it
* encodes, and — because sanitising a clean title is a no-op — no id in the
* current feed moves. Ids are localStorage keys; a gratuitous change there
* orphans completion marks with no server-side recovery.
*/
export function sanitizeEvent(
event: GachaEvent,
options: SanitizeEventOptions = {},
): SanitizeEventResult {
const sourceId = options.sourceId ?? sanitizeText(event.sourceId, { maxLength: 120 });
const notes: SanitizeNote[] = [];
const note = (level: NoteLevel, field: SanitizeNote["field"], message: string) => {
notes.push({ level, sourceId, field, message });
};
const rawTitle = typeof event.title === "string" ? event.title : "";
const title = sanitizeText(rawTitle, { maxLength: LIMITS.title });
if (title.length === 0) {
note(
"dropped",
"title",
`title sanitised to nothing (raw: ${JSON.stringify(rawTitle.slice(0, 80))})`,
);
return { event: null, notes };
}
if (title !== rawTitle) {
note("repaired", "title", `${JSON.stringify(rawTitle.slice(0, 80))}${JSON.stringify(title)}`);
}
let summary: string | null = null;
if (typeof event.summary === "string") {
const cleaned = sanitizeText(event.summary, { maxLength: LIMITS.summary });
summary = cleaned.length === 0 ? null : cleaned;
if (cleaned !== event.summary) {
note("repaired", "summary", `summary cleaned (${event.summary.length}${cleaned.length} chars)`);
}
}
const base = options.fallbackUrl;
const sourceUrl =
sanitizeUrl(event.sourceUrl, base === undefined ? {} : { base }) ??
sanitizeUrl(base);
if (sourceUrl === null) {
note(
"dropped",
"sourceUrl",
`no usable http(s) source URL (raw: ${JSON.stringify(String(event.sourceUrl).slice(0, 120))})`,
);
return { event: null, notes };
}
if (sourceUrl !== event.sourceUrl) {
note("repaired", "sourceUrl", `${JSON.stringify(String(event.sourceUrl).slice(0, 120))}${JSON.stringify(sourceUrl)}`);
}
let id = event.id;
if (title !== rawTitle && id === eventId(event.game, rawTitle, event.startsAt)) {
id = eventId(event.game, title, event.startsAt);
if (id !== event.id) {
note("repaired", "id", `${event.id}${id} (title was sanitised)`);
}
}
return { event: { ...event, id, title, summary, sourceUrl }, notes };
}
export interface SanitizeEventsResult {
events: GachaEvent[];
notes: SanitizeNote[];
}
export interface SanitizeEventsOptions extends SanitizeEventOptions {
/**
* Where notes go. Defaults to `console.warn`, on purpose: a dropped event
* must never be silent, and defaulting to a no-op would make silence the
* behaviour a future caller gets for free.
*/
onNote?: (note: SanitizeNote) => void;
}
/** Sanitise a parser's whole output. Order is preserved; drops are reported. */
export function sanitizeEvents(
events: readonly GachaEvent[],
options: SanitizeEventsOptions = {},
): SanitizeEventsResult {
const onNote = options.onNote ?? defaultReporter;
const kept: GachaEvent[] = [];
const notes: SanitizeNote[] = [];
for (const event of events) {
const result = sanitizeEvent(event, options);
for (const n of result.notes) {
notes.push(n);
onNote(n);
}
if (result.event !== null) kept.push(result.event);
}
return { events: kept, notes };
}
function defaultReporter(note: SanitizeNote): void {
const prefix = note.level === "dropped" ? "! dropped" : " repaired";
console.warn(`${prefix} ${note.sourceId} ${note.field}: ${note.message}`);
}
+528
View File
@@ -0,0 +1,528 @@
import { describe, expect, test } from "bun:test";
import { adapterById, ADAPTERS } from "../src/ingest/adapters/index.ts";
import { parserById } from "../src/ingest/parsers/index.ts";
import {
LIMITS,
sanitizeEvent,
sanitizeEvents,
sanitizeText,
sanitizeUrl,
type SanitizeNote,
} from "../src/ingest/sanitize.ts";
import { eventId, GachaEvent } from "../src/shared/schema.ts";
const NOW = "2026-08-14T00:00:00.000Z";
function event(overrides: Partial<GachaEvent> = {}): GachaEvent {
return {
id: "genshin:test-event:2026-08-12",
game: "genshin",
title: "Test Event",
type: "other",
summary: null,
startsAt: "2026-08-12T00:00:00.000Z",
startPrecision: "day",
endsAt: "2026-08-24T00:00:00.000Z",
endPrecision: "day",
regionScoped: false,
regionEnds: null,
sourceUrl: "https://example.test/a",
sourceId: "source-a",
status: "published",
confidence: 0.9,
extractionMethod: "parser",
version: 1,
firstSeenAt: NOW,
updatedAt: NOW,
...overrides,
};
}
/** Collects notes instead of printing them, so a test can assert on them. */
function collector() {
const notes: SanitizeNote[] = [];
return { notes, onNote: (n: SanitizeNote) => void notes.push(n) };
}
/**
* Everything hostile this module claims to handle, in one list.
*
* Reused by the idempotency and the no-throw properties below: any case added
* here is automatically held to both.
*/
const HOSTILE: Array<[label: string, input: string]> = [
["bare script", `<script>alert(1)</script>Windblume Festival`],
["style block", `<style>body{display:none}</style>Windblume`],
["nested tags", `<div><b>Wind<i>blume</i></b></div>`],
["malformed tag", `<<script>script>alert(1)</script>`],
["unclosed tag", `Windblume <img src=x onerror=alert(1)`],
["comment", `Wind<!-- <script>alert(1)</script> -->blume`],
["entity-encoded tag", `&lt;script&gt;alert(1)&lt;/script&gt;`],
["double-encoded tag", `&amp;lt;script&amp;gt;alert(1)&amp;lt;/script&amp;gt;`],
["numeric entity tag", `&#60;script&#62;alert(1)&#60;/script&#62;`],
["hex entity tag", `&#x3C;script&#x3E;alert(1)&#x3C;/script&#x3E;`],
["fullwidth tag", `scriptalert(1)/script`],
["zero-width in entity", `&am\u200bp;lt;script&gt;`],
["deep entity nesting", `&#38;#38;#38;#38;#38;#38;#38;lt;script&gt;`],
["rtl override", `Login \u202eEvent\u202c`],
["bidi isolates", `\u2066Free\u2069\u2067Primogems\u2069`],
["control characters", `Wind\u0000blume\u0007 Fest\u001bival`],
["lone high surrogate", `Windblume \ud800`],
["lone low surrogate", `\udc00 Windblume`],
["out-of-range code point", `Windblume &#1114112; &#x110000;`],
["absurdly long", `A${"b".repeat(50_000)}`],
["whitespace storm", `\n\n Wind\t\t blume  \u3000 Festival \n`],
["empty", ``],
["only markup", `<div></div><!-- x -->`],
];
describe("sanitizeText — markup", () => {
test("strips tags, keeping the text between them", () => {
expect(sanitizeText(`<div><b>Wind</b>blume <i>Festival</i></div>`)).toBe(
"Wind blume Festival",
);
});
test("removes script and style content, not just the tags", () => {
expect(sanitizeText(`<script>alert(1)</script>Windblume`)).toBe("Windblume");
expect(sanitizeText(`<style>body{display:none}</style>Windblume`)).toBe(
"Windblume",
);
expect(sanitizeText(`<iframe src="//evil.test"></iframe>Windblume`)).toBe(
"Windblume",
);
});
test("removes comments and their contents", () => {
expect(sanitizeText(`Wind<!-- <script>alert(1)</script> -->blume`)).toBe(
"Wind blume",
);
});
test("leaves no tag-shaped substring, however malformed the input", () => {
for (const [label, input] of HOSTILE) {
expect(sanitizeText(input), label).not.toMatch(/<[^>]*>/);
}
});
test("an unclosed tag does not swallow the rest of the string", () => {
// The tag itself goes, but the words around it survive — a source that
// forgets a `>` should cost us markup, not an event.
expect(sanitizeText(`Windblume <b>Festival`)).toBe("Windblume Festival");
});
});
describe("sanitizeText — entities", () => {
test("decodes ordinary entities", () => {
expect(sanitizeText(`Tea &amp; Cakes &ndash; Act&nbsp;II`)).toBe(
"Tea & Cakes Act II",
);
});
test("an entity-encoded tag never becomes a live tag", () => {
expect(sanitizeText(`&lt;script&gt;alert(1)&lt;/script&gt;`)).toBe("alert(1)");
});
test("double-encoding does not survive one decode into markup", () => {
// The dangerous case: one decode yields `&lt;script>`, which a second
// decoder anywhere downstream would turn into a tag. Decoding to a fixed
// point and stripping each round means the output is inert for every later
// reader too.
const out = sanitizeText(`&amp;lt;script&amp;gt;alert(1)&amp;lt;/script&amp;gt;`);
expect(out).not.toContain("<");
expect(out).not.toContain("&lt;");
expect(sanitizeText(out)).toBe(out);
});
test("numeric, hex and fullwidth spellings of a tag are all neutralised", () => {
for (const input of [
`&#60;script&#62;alert(1)&#60;/script&#62;`,
`&#x3C;script&#x3E;alert(1)&#x3C;/script&#x3E;`,
`scriptalert(1)/script`,
]) {
expect(sanitizeText(input)).not.toMatch(/<[^>]*>/);
}
});
test("an out-of-range code point is left alone rather than throwing", () => {
expect(() => sanitizeText(`&#1114112; &#x110000;`)).not.toThrow();
expect(sanitizeText(`Windblume &#1114112;`)).toBe("Windblume &#1114112;");
});
});
describe("sanitizeText — unicode", () => {
test("normalises to NFKC", () => {
// Compatibility forms are how a source spoofs a title that reads the same.
expect(sanitizeText("Windblume")).toBe("Windblume");
expect(sanitizeText("éclair")).toBe("éclair");
});
test("strips control characters", () => {
expect(sanitizeText("Wind\u0000blume\u0007 Fest\u001bival")).toBe(
"Windblume Festival",
);
});
test("strips zero-width characters", () => {
expect(sanitizeText("Wind\u200bblume\u200c\u200d\ufeff")).toBe("Windblume");
});
test("strips bidi overrides that would spoof a title", () => {
const spoofed = "Login \u202ednellA\u202c";
const clean = sanitizeText(spoofed);
expect(clean).not.toMatch(/[\u202a-\u202e\u2066-\u2069]/);
expect(clean).toBe("Login dnellA");
});
test("drops unpaired surrogates but keeps real astral characters", () => {
expect(sanitizeText("Windblume \ud800")).toBe("Windblume");
expect(sanitizeText("\udc00 Windblume")).toBe("Windblume");
expect(sanitizeText("Windblume 🎉")).toBe("Windblume 🎉");
});
test("collapses every flavour of whitespace to single spaces", () => {
expect(sanitizeText("\n\n Wind\t\t blume  \u3000 Festival \n")).toBe(
"Wind blume Festival",
);
});
});
describe("sanitizeText — length", () => {
test("truncates to the cap rather than discarding the value", () => {
const long = `Windblume ${"a".repeat(5_000)}`;
const title = sanitizeText(long, { maxLength: LIMITS.title });
expect(title.length).toBeLessThanOrEqual(LIMITS.title);
expect(title.startsWith("Windblume")).toBe(true);
expect(title.endsWith("…")).toBe(true);
});
test("cuts at a word boundary when there is one", () => {
const words = "Windblume Festival Returns To Mondstadt In Full Bloom";
expect(sanitizeText(words, { maxLength: 20 })).toBe("Windblume Festival…");
});
test("a value at the cap is left exactly as it is", () => {
const exact = "a".repeat(LIMITS.title);
expect(sanitizeText(exact, { maxLength: LIMITS.title })).toBe(exact);
});
test("the caps match what the schema will accept", () => {
// The schema is the single source of truth; LIMITS only mirrors it. If
// someone widens or narrows `title`/`summary` there, this fails here.
const ok = GachaEvent.safeParse(
event({ title: "a".repeat(LIMITS.title), summary: "b".repeat(LIMITS.summary) }),
);
expect(ok.success).toBe(true);
expect(GachaEvent.safeParse(event({ title: "a".repeat(LIMITS.title + 1) })).success).toBe(false);
expect(GachaEvent.safeParse(event({ summary: "b".repeat(LIMITS.summary + 1) })).success).toBe(false);
});
});
describe("sanitizeText — totality", () => {
test("is idempotent for every hostile input", () => {
for (const [label, input] of HOSTILE) {
const once = sanitizeText(input);
expect(sanitizeText(once), label).toBe(once);
}
});
test("never throws on junk", () => {
for (const [label, input] of HOSTILE) {
expect(() => sanitizeText(input), label).not.toThrow();
}
for (const junk of [null, undefined, 42, true, {}, [], Symbol("x")]) {
expect(() => sanitizeText(junk)).not.toThrow();
}
expect(sanitizeText(null)).toBe("");
expect(sanitizeText(undefined)).toBe("");
expect(sanitizeText({})).toBe("");
});
});
describe("sanitizeUrl", () => {
test("keeps http and https", () => {
expect(sanitizeUrl("https://game8.co/games/Genshin-Impact/archives/301601")).toBe(
"https://game8.co/games/Genshin-Impact/archives/301601",
);
expect(sanitizeUrl("http://example.test/a")).toBe("http://example.test/a");
});
test("rejects javascript:, data: and every other scheme", () => {
for (const bad of [
"javascript:alert(1)",
"JaVaScRiPt:alert(1)",
"java\tscript:alert(1)",
"&#106;avascript:alert(1)",
"data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
"vbscript:msgbox(1)",
"file:///etc/passwd",
"about:blank",
]) {
expect(sanitizeUrl(bad), bad).toBeNull();
}
});
test("rejects a URL carrying credentials", () => {
expect(sanitizeUrl("https://user:[email protected]/x")).toBeNull();
});
test("rejects junk instead of throwing", () => {
for (const junk of ["", " ", "not a url", null, undefined, 42, {}]) {
expect(() => sanitizeUrl(junk)).not.toThrow();
expect(sanitizeUrl(junk)).toBeNull();
}
});
test("resolves a relative href against the source page", () => {
expect(sanitizeUrl("/wiki/Event", { base: "https://endfield.wiki.gg/wiki/Home" })).toBe(
"https://endfield.wiki.gg/wiki/Event",
);
});
test("a relative href cannot smuggle in another scheme via the base", () => {
expect(
sanitizeUrl("javascript:alert(1)", { base: "https://endfield.wiki.gg/wiki/Home" }),
).toBeNull();
});
});
describe("sanitizeEvent", () => {
test("leaves a clean event completely alone", () => {
const clean = event({
id: eventId("genshin", "Windblume Festival", "2026-08-12T00:00:00.000Z"),
title: "Windblume Festival",
summary: "A festival in Mondstadt.",
});
const { event: out, notes } = sanitizeEvent(clean);
expect(out).toEqual(clean);
expect(notes).toEqual([]);
});
test("never touches timestamps, precision or confidence", () => {
const dirty = event({ title: "<b>Windblume</b>", summary: "<i>Blurb</i>" });
const { event: out } = sanitizeEvent(dirty);
expect(out?.startsAt).toBe(dirty.startsAt);
expect(out?.endsAt).toBe(dirty.endsAt);
expect(out?.startPrecision).toBe(dirty.startPrecision);
expect(out?.endPrecision).toBe(dirty.endPrecision);
expect(out?.confidence).toBe(dirty.confidence);
expect(out?.regionEnds).toBe(dirty.regionEnds);
});
test("cleans a spoofed title and keeps the id agreeing with it", () => {
const startsAt = "2026-08-12T00:00:00.000Z";
const rawTitle = "Login \u202eEvent\u202c";
const dirty = event({
id: eventId("genshin", rawTitle, startsAt),
title: rawTitle,
startsAt,
});
// The override is what made this render as something other than its
// characters; removing it leaves the honest text, and the id follows.
const { event: out, notes } = sanitizeEvent(dirty, { sourceId: "source-a" });
expect(out?.title).toBe("Login Event");
expect(out?.id).toBe(eventId("genshin", "Login Event", startsAt));
expect(notes.every((n) => n.level === "repaired")).toBe(true);
// No id note here, and that is the point: `slugify` already drops
// characters like these, so cleaning the title moved nothing a user has
// saved state under.
expect(notes.map((n) => n.field)).toEqual(["title"]);
});
test("recomputes the id when cleaning genuinely changes the slug", () => {
const startsAt = "2026-08-12T00:00:00.000Z";
const rawTitle = "<b>Windblume</b> Festival";
const dirty = event({
id: eventId("genshin", rawTitle, startsAt),
title: rawTitle,
startsAt,
});
const { event: out, notes } = sanitizeEvent(dirty);
expect(out?.title).toBe("Windblume Festival");
expect(out?.id).toBe("genshin:windblume-festival:2026-08-12");
expect(notes.some((n) => n.field === "id")).toBe(true);
});
test("leaves an id alone when it was not minted from the title", () => {
// Reconciliation keeps an existing id when a wiki renames an event
// (docs/INGESTION.md § Stage 5). Sanitising must not undo that.
const dirty = event({ id: "genshin:kept-across-a-rename:2026-08-12", title: "<b>New Name</b>" });
const { event: out } = sanitizeEvent(dirty);
expect(out?.title).toBe("New Name");
expect(out?.id).toBe("genshin:kept-across-a-rename:2026-08-12");
});
test("truncates a hostile title instead of losing the event", () => {
const dirty = event({ title: `Windblume ${"a".repeat(10_000)}` });
const { event: out } = sanitizeEvent(dirty);
expect(out).not.toBeNull();
expect((out?.title ?? "").length).toBeLessThanOrEqual(LIMITS.title);
expect(GachaEvent.safeParse(out).success).toBe(true);
});
test("an empty summary becomes null rather than an empty string", () => {
const { event: out } = sanitizeEvent(event({ summary: "<span> </span>" }));
expect(out?.summary).toBeNull();
});
test("falls back to the source's own URL when the scraped one is hostile", () => {
const dirty = event({ sourceUrl: "javascript:alert(1)" });
const { event: out, notes } = sanitizeEvent(dirty, {
fallbackUrl: "https://game8.co/games/Genshin-Impact/archives/301601",
sourceId: "genshin-game8-events",
});
expect(out?.sourceUrl).toBe("https://game8.co/games/Genshin-Impact/archives/301601");
expect(notes.some((n) => n.field === "sourceUrl" && n.level === "repaired")).toBe(true);
});
test("drops an event only when nothing is left to publish, and says so", () => {
const gone = event({ title: "<script>alert(1)</script>" });
const { event: out, notes } = sanitizeEvent(gone, { sourceId: "source-a" });
expect(out).toBeNull();
expect(notes).toHaveLength(1);
expect(notes[0]?.level).toBe("dropped");
expect(notes[0]?.field).toBe("title");
expect(notes[0]?.sourceId).toBe("source-a");
});
test("drops an event with no usable URL and no fallback, and says so", () => {
const { event: out, notes } = sanitizeEvent(event({ sourceUrl: "javascript:alert(1)" }));
expect(out).toBeNull();
expect(notes.some((n) => n.level === "dropped" && n.field === "sourceUrl")).toBe(true);
});
test("output always satisfies the schema", () => {
for (const [label, input] of HOSTILE) {
const { event: out } = sanitizeEvent(
event({ title: input, summary: input, sourceUrl: input }),
{ fallbackUrl: "https://example.test/a" },
);
if (out === null) continue;
expect(GachaEvent.safeParse(out).success, label).toBe(true);
}
});
});
describe("sanitizeEvents", () => {
test("keeps order and reports every note", () => {
const { notes, onNote } = collector();
const { events } = sanitizeEvents(
[
event({ id: "genshin:a:2026-08-12", title: "Alpha" }),
event({ id: "genshin:b:2026-08-12", title: "<b>Beta</b>" }),
event({ id: "genshin:c:2026-08-12", title: "Gamma" }),
],
{ onNote, sourceId: "source-a" },
);
expect(events.map((e) => e.title)).toEqual(["Alpha", "Beta", "Gamma"]);
expect(notes).toHaveLength(1);
expect(notes[0]?.field).toBe("title");
});
test("a dropped event never disappears silently", () => {
const { notes, onNote } = collector();
const { events } = sanitizeEvents(
[event({ title: "Alpha" }), event({ title: "<!-- nothing -->" })],
{ onNote },
);
expect(events).toHaveLength(1);
expect(notes.filter((n) => n.level === "dropped")).toHaveLength(1);
});
test("siblings survive a dropped event", () => {
const { onNote } = collector();
const { events } = sanitizeEvents(
[event({ title: "<script>x</script>" }), event({ title: "Beta" })],
{ onNote },
);
expect(events.map((e) => e.title)).toEqual(["Beta"]);
});
});
describe("the adapter seam", () => {
/**
* A minimal page in Game8's shape, carrying everything a hostile source could
* put in a title. The point is not the parser — it is that `adapter.parse`
* cannot return unsanitised events, whichever parser produced them.
*/
const HOSTILE_PAGE = `
<h2 class="a-header--2">Current Events</h2>
<h3 class="a-header--3">Windblume \u202eFestival\u202c\u200b &amp; Friends</h3>
<table class="a-table">
<tr><th>Event Start</th><td>August 12, 2026</td></tr>
<tr><th>Event End</th><td>August 24, 2026</td></tr>
</table>
<p class="a-paragraph">A blurb with &lt;script&gt;alert(1)&lt;/script&gt; in it.</p>
`;
test("events coming out of an adapter are already sanitised", () => {
const adapter = adapterById("genshin-game8-events");
if (adapter === undefined) throw new Error("no adapter");
const events = adapter.parse(HOSTILE_PAGE, {
now: NOW,
sourceUrl: adapter.url,
sourceId: adapter.id,
game: adapter.game,
});
expect(events).toHaveLength(1);
const parsed = events[0];
expect(parsed?.title).toBe("Windblume Festival & Friends");
expect(parsed?.title).not.toMatch(/[\u200b-\u200f\u202a-\u202e]/);
expect(parsed?.summary ?? "").not.toMatch(/<[^>]*>/);
expect(parsed?.sourceUrl).toBe(adapter.url);
// Dates are the one thing sanitising must never touch.
expect(parsed?.startsAt).toBe("2026-08-12T00:00:00.000Z");
expect(parsed?.endsAt).toBe("2026-08-24T00:00:00.000Z");
// The id agrees with the title the user actually sees.
expect(parsed?.id).toBe(
eventId("genshin", parsed?.title ?? "", parsed?.startsAt ?? ""),
);
expect(GachaEvent.safeParse(parsed).success).toBe(true);
});
/**
* The guard that matters most.
*
* Event ids are localStorage keys, so sanitising must be a *no-op* on the
* pages we actually parse — if it ever starts repairing a real title, the id
* derived from that title can move and every completion mark saved under the
* old one is orphaned with no server-side recovery. Running the parsers
* directly (before the adapter seam cleans anything) and asserting zero notes
* is what makes that visible the moment it changes.
*/
test("real fixtures need no repair at all", async () => {
for (const adapter of ADAPTERS) {
const parser = parserById(adapter.parserId);
if (parser === undefined) throw new Error(`no parser ${adapter.parserId}`);
const pattern = `fixtures/${adapter.game}/${adapter.parserId}-*.html`;
const file = [...new Bun.Glob(pattern).scanSync(".")].sort().at(-1);
if (file === undefined) throw new Error(`no fixture for ${adapter.id}`);
const html = await Bun.file(file).text();
const raw = parser.parse(html, {
now: NOW,
sourceUrl: adapter.url,
sourceId: adapter.id,
game: adapter.game,
});
const { notes, events } = sanitizeEvents(raw, {
sourceId: adapter.id,
fallbackUrl: adapter.url,
onNote: () => {},
});
expect(notes.map((n) => `${n.field}: ${n.message}`), adapter.id).toEqual([]);
expect(events.map((e) => e.id), adapter.id).toEqual(raw.map((e) => e.id));
}
});
});