feat: sanitise every string a source publishes
Scraped text becomes React content, JSON on disk and eventually SQLite rows, so it gets cleaned at one seam: the parse wrapper in toAdapter(). Every source passes through it, a source added tomorrow is covered without its author doing anything, and no parser can opt out — parsers stay pure readers of one site's markup. Removes script/style/comment content and residual tags, decodes entities to a fixed point so an encoded tag cannot resurrect in a later decoder, NFKC-normalises, strips control, zero-width and bidi-override characters (an RTL override visually spoofs a title), bounds each field to the cap the schema already declares, and requires sourceUrl to be absolute http(s). Three things it will not do: touch a date, drop an event it could clean instead, or repair in silence — every repair and drop is logged by default. Event IDs are localStorage keys, so an ID is recomputed only when a sanitised title actually changed and the ID was minted the standard way; all seven fixtures pass through unrepaired and byte-identical, which is the regression guard. Also stops decodeEntities throwing RangeError on an out-of-range numeric reference, which would have taken a whole source's events down. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
2b9338a8b0
commit
18b9652aed
@@ -1,6 +1,7 @@
|
|||||||
import type { GachaEvent, GameId } from "../../shared/schema.ts";
|
import type { GachaEvent, GameId } from "../../shared/schema.ts";
|
||||||
import { mergeEvents, type MergeResult } from "../merge.ts";
|
import { mergeEvents, type MergeResult } from "../merge.ts";
|
||||||
import { parserById } from "../parsers/index.ts";
|
import { parserById } from "../parsers/index.ts";
|
||||||
|
import { sanitizeEvents } from "../sanitize.ts";
|
||||||
import { SIX_HOURS_MS, type Adapter, type ParseContext } from "./types.ts";
|
import { SIX_HOURS_MS, type Adapter, type ParseContext } from "./types.ts";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -90,7 +91,20 @@ function toAdapter(spec: SourceSpec): Adapter {
|
|||||||
`${spec.id}: document does not match the '${parser.label}' template; the source has likely been redesigned`,
|
`${spec.id}: document does not match the '${parser.label}' template; the source has likely been redesigned`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return parser.parse(html, ctx);
|
|
||||||
|
// The trust boundary. Everything a parser produces came from a page we do
|
||||||
|
// not control, and this is the one place every source passes through:
|
||||||
|
// `ADAPTERS` is built from `SOURCES` via this function, so a source added
|
||||||
|
// tomorrow is sanitised without its author doing anything, and a parser
|
||||||
|
// cannot opt out. Sanitising here rather than inside the parsers also
|
||||||
|
// keeps parsers what they are — pure readers of one site's markup.
|
||||||
|
//
|
||||||
|
// `sanitizeEvents` logs to console.warn by default, so a repaired or
|
||||||
|
// dropped event is never silent (CLAUDE.md § Silent drops).
|
||||||
|
return sanitizeEvents(parser.parse(html, ctx), {
|
||||||
|
sourceId: ctx.sourceId,
|
||||||
|
fallbackUrl: ctx.sourceUrl,
|
||||||
|
}).events;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
+18
-4
@@ -23,13 +23,27 @@ const ENTITIES: Record<string, string> = {
|
|||||||
rdquo: "”",
|
rdquo: "”",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A numeric reference the page may or may not mean literally.
|
||||||
|
*
|
||||||
|
* Out-of-range and non-finite code points come back as the original text rather
|
||||||
|
* than throwing: `�` and `�` are junk a hostile or merely
|
||||||
|
* broken page can emit, and `String.fromCodePoint` throws RangeError on both.
|
||||||
|
* A parser crashing on one bad character would take a whole source's events
|
||||||
|
* down with it.
|
||||||
|
*/
|
||||||
|
function fromCodePoint(value: number, original: string): string {
|
||||||
|
if (!Number.isInteger(value) || value < 0 || value > 0x10ffff) return original;
|
||||||
|
return String.fromCodePoint(value);
|
||||||
|
}
|
||||||
|
|
||||||
export function decodeEntities(input: string): string {
|
export function decodeEntities(input: string): string {
|
||||||
return input
|
return input
|
||||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) =>
|
.replace(/&#x([0-9a-fA-F]+);/g, (whole, hex: string) =>
|
||||||
String.fromCodePoint(parseInt(hex, 16)),
|
fromCodePoint(parseInt(hex, 16), whole),
|
||||||
)
|
)
|
||||||
.replace(/&#(\d+);/g, (_, dec: string) =>
|
.replace(/&#(\d+);/g, (whole, dec: string) =>
|
||||||
String.fromCodePoint(parseInt(dec, 10)),
|
fromCodePoint(parseInt(dec, 10), whole),
|
||||||
)
|
)
|
||||||
.replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole);
|
.replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,362 @@
|
|||||||
|
import { eventId, type GachaEvent } from "../shared/schema.ts";
|
||||||
|
import { decodeEntities } from "./html.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The trust boundary for scraped text.
|
||||||
|
*
|
||||||
|
* Every string on a `GachaEvent` starts life as bytes from a community wiki we
|
||||||
|
* do not control. Between the parser and validation it passes through here, so
|
||||||
|
* that what reaches the feed — and therefore React, `localStorage`, JSON on
|
||||||
|
* disk and eventually SQLite — is plain, bounded, normalised text.
|
||||||
|
*
|
||||||
|
* Three principles, in priority order:
|
||||||
|
*
|
||||||
|
* 1. **Never invent or alter a date.** Nothing in this module reads, writes or
|
||||||
|
* reformats a timestamp. Dates are the product's whole promise; the
|
||||||
|
* sanitiser's job stops at prose and URLs.
|
||||||
|
* 2. **Clean, do not drop.** A hostile title is truncated and stripped, not
|
||||||
|
* rejected — an event vanishing without a trace is the failure mode this
|
||||||
|
* codebase fears most (CLAUDE.md § Silent drops). The one unrecoverable
|
||||||
|
* case is a title that sanitises to nothing, and that emits a note the
|
||||||
|
* caller is expected to surface.
|
||||||
|
* 3. **Never throw on junk.** Malformed entities, lone surrogates, absurd code
|
||||||
|
* points and 5MB strings all have to come out the other side as a string.
|
||||||
|
*
|
||||||
|
* Dependency-free by design: no DOMPurify, no sanitize-html, no parse5.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Length caps, mirroring `title`/`summary` in `src/shared/schema.ts`.
|
||||||
|
*
|
||||||
|
* The schema stays the single source of truth — these exist so a hostile page
|
||||||
|
* is truncated *before* validation instead of failing it, and
|
||||||
|
* `test/sanitize.test.ts` asserts that a string of exactly this length is
|
||||||
|
* accepted by `GachaEvent` and one character more is not, so the two cannot
|
||||||
|
* drift apart unnoticed.
|
||||||
|
*/
|
||||||
|
export const LIMITS = {
|
||||||
|
title: 200,
|
||||||
|
summary: 500,
|
||||||
|
/** Not a schema cap: a defensive ceiling so a junk href cannot be a novel. */
|
||||||
|
url: 2048,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
/** How many decode/strip rounds before giving up and hard-scrubbing. */
|
||||||
|
const MAX_PASSES = 5;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Characters that are invisible, that control how surrounding text is
|
||||||
|
* *displayed*, or that are not legal text at all.
|
||||||
|
*
|
||||||
|
* The bidi overrides and isolates (U+202A–U+202E, U+2066–U+2069) matter most:
|
||||||
|
* they let a source render "Login Event" as something else entirely, or hide a
|
||||||
|
* suffix from a reader while it still lands in the title, the slug and the
|
||||||
|
* user's saved state. Zero-width characters do the same job more crudely and
|
||||||
|
* additionally let an attacker smuggle `&am<ZWSP>p;lt;` past a naive decoder.
|
||||||
|
*
|
||||||
|
* Note ZWJ (U+200D) goes too, which splits multi-part emoji into their
|
||||||
|
* components. Our sources are English-language wikis; a mangled family emoji is
|
||||||
|
* a fair price for no invisible characters anywhere in a title.
|
||||||
|
*/
|
||||||
|
const INVISIBLE =
|
||||||
|
/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u00AD\u061C\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\uFEFF\uFFF9-\uFFFB\uFFFE\uFFFF]/g;
|
||||||
|
|
||||||
|
/** Unicode tag characters (U+E0000–U+E007F) — invisible, as surrogate pairs. */
|
||||||
|
const TAG_CHARS = /\uDB40[\uDC00-\uDC7F]/g;
|
||||||
|
|
||||||
|
/** Half of a surrogate pair with no partner: not valid text, breaks JSON. */
|
||||||
|
const LONE_SURROGATE =
|
||||||
|
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/g;
|
||||||
|
|
||||||
|
/** Elements whose *content* is code, not text, and must go with the tag. */
|
||||||
|
const CODE_BLOCKS =
|
||||||
|
/<(script|style|template|noscript|iframe|object|embed|svg|math)\b[\s\S]*?(?:<\/\1\s*>|$)/gi;
|
||||||
|
|
||||||
|
const COMMENTS = /<!--[\s\S]*?(?:-->|$)|<!\[CDATA\[[\s\S]*?(?:\]\]>|$)|<\?[\s\S]*?(?:\?>|$)/g;
|
||||||
|
|
||||||
|
const TAG = /<[^>]*>/g;
|
||||||
|
|
||||||
|
function stripInvisible(input: string): string {
|
||||||
|
return input
|
||||||
|
.replace(TAG_CHARS, "")
|
||||||
|
.replace(INVISIBLE, "")
|
||||||
|
.replace(LONE_SURROGATE, (m) => (m.length === 2 ? m[0] ?? "" : ""));
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripTags(input: string): string {
|
||||||
|
let out = input;
|
||||||
|
for (let i = 0; i < MAX_PASSES; i += 1) {
|
||||||
|
const next = out.replace(TAG, " ");
|
||||||
|
if (next === out) return out;
|
||||||
|
out = next;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One round of neutralising.
|
||||||
|
*
|
||||||
|
* Order is deliberate. NFKC runs *first* because it turns look-alike forms into
|
||||||
|
* their canonical ones (U+FF1C FULLWIDTH LESS-THAN becomes `<`), and a
|
||||||
|
* normaliser running after the tag stripper would hand back live markup.
|
||||||
|
* Invisible characters go next so they cannot break up an entity or a tag name.
|
||||||
|
* Only then is anything decoded, and whatever the decode produced is stripped
|
||||||
|
* in the same round.
|
||||||
|
*/
|
||||||
|
function pass(input: string): string {
|
||||||
|
const normalised = stripInvisible(input.normalize("NFKC"));
|
||||||
|
const withoutCode = normalised.replace(CODE_BLOCKS, " ").replace(COMMENTS, " ");
|
||||||
|
return stripTags(decodeEntities(withoutCode));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reduce text to a fixed point of `pass`.
|
||||||
|
*
|
||||||
|
* Running to a fixed point is what makes the whole module idempotent, and it is
|
||||||
|
* also the answer to double-encoding: `&lt;script>` decodes to
|
||||||
|
* `<script>` on the first round and to a tag on the second, which the second
|
||||||
|
* round then strips. Stopping after one decode would leave a string that a
|
||||||
|
* later `sanitizeText` — or any other decoder downstream — turns into markup.
|
||||||
|
*/
|
||||||
|
function toFixedPoint(input: string): string {
|
||||||
|
let out = input;
|
||||||
|
for (let i = 0; i < MAX_PASSES; i += 1) {
|
||||||
|
const next = pass(out);
|
||||||
|
if (next === out) return out;
|
||||||
|
out = next;
|
||||||
|
}
|
||||||
|
// Pathological input that keeps re-encoding itself — `&#38;#38;…` nested
|
||||||
|
// deeper than the round limit. Removing every `<`, `>` and `&` both kills any
|
||||||
|
// tag shape and guarantees the result is a fixed point (nothing left to
|
||||||
|
// decode), which is what keeps `sanitizeText` idempotent even here.
|
||||||
|
return out.replace(/[<>&]/g, " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Collapse every run of whitespace — including NBSP and U+2028 — to one space. */
|
||||||
|
function collapse(input: string): string {
|
||||||
|
return input
|
||||||
|
.replace(/[\s\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]+/g, " ")
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cut to `max` characters at a word boundary, marking the cut with an ellipsis.
|
||||||
|
*
|
||||||
|
* Truncating beats rejecting: a real event with a bloated description is still
|
||||||
|
* a real event, and the user would rather see it than not. The result is always
|
||||||
|
* `<= max`, so a second pass never truncates again.
|
||||||
|
*/
|
||||||
|
function truncate(input: string, max: number): string {
|
||||||
|
if (input.length <= max) return input;
|
||||||
|
|
||||||
|
let cut = input.slice(0, max - 1);
|
||||||
|
// Never end on half a surrogate pair.
|
||||||
|
if (/[\uD800-\uDBFF]$/.test(cut)) cut = cut.slice(0, -1);
|
||||||
|
|
||||||
|
const lastSpace = cut.lastIndexOf(" ");
|
||||||
|
if (lastSpace > max * 0.6) cut = cut.slice(0, lastSpace);
|
||||||
|
|
||||||
|
return `${cut.replace(/[\s,;:.–—-]+$/, "")}…`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SanitizeTextOptions {
|
||||||
|
/** Hard cap; the result is never longer. Defaults to the summary cap. */
|
||||||
|
maxLength?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clean one string extracted from a source.
|
||||||
|
*
|
||||||
|
* Total: any input, including `null`, a number or a 5MB blob of markup, yields
|
||||||
|
* a string. Idempotent: `sanitizeText(sanitizeText(x)) === sanitizeText(x)`.
|
||||||
|
*/
|
||||||
|
export function sanitizeText(
|
||||||
|
input: unknown,
|
||||||
|
options: SanitizeTextOptions = {},
|
||||||
|
): string {
|
||||||
|
// Anything that is not a primitive is not text a source stated; "" is the
|
||||||
|
// honest reading of it, and stringifying an object would invent content.
|
||||||
|
const raw =
|
||||||
|
typeof input === "string"
|
||||||
|
? input
|
||||||
|
: typeof input === "number" || typeof input === "boolean"
|
||||||
|
? String(input)
|
||||||
|
: "";
|
||||||
|
const max = options.maxLength ?? LIMITS.summary;
|
||||||
|
return truncate(collapse(toFixedPoint(raw)), max);
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SanitizeUrlOptions {
|
||||||
|
/** Resolves a relative href, exactly as `new URL(href, base)` would. */
|
||||||
|
base?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return `input` as an absolute http(s) URL, or null if it is not one.
|
||||||
|
*
|
||||||
|
* `sourceUrl` is rendered as an attribution link, so a `javascript:` or `data:`
|
||||||
|
* URL that reached the client would be a live XSS vector in an app that
|
||||||
|
* otherwise never handles untrusted URLs. Anything that is not plainly http(s)
|
||||||
|
* — including a URL carrying credentials, which is only ever a phishing shape —
|
||||||
|
* is refused, and the caller falls back to the source's registered URL.
|
||||||
|
*/
|
||||||
|
export function sanitizeUrl(
|
||||||
|
input: unknown,
|
||||||
|
options: SanitizeUrlOptions = {},
|
||||||
|
): string | null {
|
||||||
|
const raw = sanitizeText(input, { maxLength: LIMITS.url });
|
||||||
|
if (raw.length === 0) return null;
|
||||||
|
|
||||||
|
let url: URL;
|
||||||
|
try {
|
||||||
|
url = options.base === undefined ? new URL(raw) : new URL(raw, options.base);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||||
|
if (url.username !== "" || url.password !== "") return null;
|
||||||
|
return url.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
export type NoteLevel = "repaired" | "dropped";
|
||||||
|
|
||||||
|
export interface SanitizeNote {
|
||||||
|
level: NoteLevel;
|
||||||
|
sourceId: string;
|
||||||
|
field: "title" | "summary" | "sourceUrl" | "id";
|
||||||
|
/** Human-readable, already truncated — safe to print to a log. */
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SanitizeEventOptions {
|
||||||
|
/**
|
||||||
|
* The source's registered URL. Known good (it came from `SOURCES`, not from
|
||||||
|
* the page), so it is the fallback when an event's own `sourceUrl` is junk —
|
||||||
|
* attribution to the right page beats discarding the event.
|
||||||
|
*/
|
||||||
|
fallbackUrl?: string;
|
||||||
|
/** Reported on every note, so a log line names the source that produced it. */
|
||||||
|
sourceId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SanitizeEventResult {
|
||||||
|
/** Null only when the event could not be repaired into a publishable shape. */
|
||||||
|
event: GachaEvent | null;
|
||||||
|
notes: SanitizeNote[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clean every source-derived string on one event.
|
||||||
|
*
|
||||||
|
* Timestamps, precisions, confidence, region data and `sourceId` are passed
|
||||||
|
* through untouched: the first three are numbers and dates this module has no
|
||||||
|
* business rewriting, and the last comes from our own registry rather than from
|
||||||
|
* the page.
|
||||||
|
*
|
||||||
|
* The `id` is only recomputed when sanitising actually changed the title *and*
|
||||||
|
* the incoming id was minted the standard way (`eventId(game, title, startsAt)`).
|
||||||
|
* That keeps two guarantees at once: an id never disagrees with the title it
|
||||||
|
* encodes, and — because sanitising a clean title is a no-op — no id in the
|
||||||
|
* current feed moves. Ids are localStorage keys; a gratuitous change there
|
||||||
|
* orphans completion marks with no server-side recovery.
|
||||||
|
*/
|
||||||
|
export function sanitizeEvent(
|
||||||
|
event: GachaEvent,
|
||||||
|
options: SanitizeEventOptions = {},
|
||||||
|
): SanitizeEventResult {
|
||||||
|
const sourceId = options.sourceId ?? sanitizeText(event.sourceId, { maxLength: 120 });
|
||||||
|
const notes: SanitizeNote[] = [];
|
||||||
|
const note = (level: NoteLevel, field: SanitizeNote["field"], message: string) => {
|
||||||
|
notes.push({ level, sourceId, field, message });
|
||||||
|
};
|
||||||
|
|
||||||
|
const rawTitle = typeof event.title === "string" ? event.title : "";
|
||||||
|
const title = sanitizeText(rawTitle, { maxLength: LIMITS.title });
|
||||||
|
if (title.length === 0) {
|
||||||
|
note(
|
||||||
|
"dropped",
|
||||||
|
"title",
|
||||||
|
`title sanitised to nothing (raw: ${JSON.stringify(rawTitle.slice(0, 80))})`,
|
||||||
|
);
|
||||||
|
return { event: null, notes };
|
||||||
|
}
|
||||||
|
if (title !== rawTitle) {
|
||||||
|
note("repaired", "title", `${JSON.stringify(rawTitle.slice(0, 80))} → ${JSON.stringify(title)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary: string | null = null;
|
||||||
|
if (typeof event.summary === "string") {
|
||||||
|
const cleaned = sanitizeText(event.summary, { maxLength: LIMITS.summary });
|
||||||
|
summary = cleaned.length === 0 ? null : cleaned;
|
||||||
|
if (cleaned !== event.summary) {
|
||||||
|
note("repaired", "summary", `summary cleaned (${event.summary.length} → ${cleaned.length} chars)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const base = options.fallbackUrl;
|
||||||
|
const sourceUrl =
|
||||||
|
sanitizeUrl(event.sourceUrl, base === undefined ? {} : { base }) ??
|
||||||
|
sanitizeUrl(base);
|
||||||
|
if (sourceUrl === null) {
|
||||||
|
note(
|
||||||
|
"dropped",
|
||||||
|
"sourceUrl",
|
||||||
|
`no usable http(s) source URL (raw: ${JSON.stringify(String(event.sourceUrl).slice(0, 120))})`,
|
||||||
|
);
|
||||||
|
return { event: null, notes };
|
||||||
|
}
|
||||||
|
if (sourceUrl !== event.sourceUrl) {
|
||||||
|
note("repaired", "sourceUrl", `${JSON.stringify(String(event.sourceUrl).slice(0, 120))} → ${JSON.stringify(sourceUrl)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let id = event.id;
|
||||||
|
if (title !== rawTitle && id === eventId(event.game, rawTitle, event.startsAt)) {
|
||||||
|
id = eventId(event.game, title, event.startsAt);
|
||||||
|
if (id !== event.id) {
|
||||||
|
note("repaired", "id", `${event.id} → ${id} (title was sanitised)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { event: { ...event, id, title, summary, sourceUrl }, notes };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SanitizeEventsResult {
|
||||||
|
events: GachaEvent[];
|
||||||
|
notes: SanitizeNote[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SanitizeEventsOptions extends SanitizeEventOptions {
|
||||||
|
/**
|
||||||
|
* Where notes go. Defaults to `console.warn`, on purpose: a dropped event
|
||||||
|
* must never be silent, and defaulting to a no-op would make silence the
|
||||||
|
* behaviour a future caller gets for free.
|
||||||
|
*/
|
||||||
|
onNote?: (note: SanitizeNote) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sanitise a parser's whole output. Order is preserved; drops are reported. */
|
||||||
|
export function sanitizeEvents(
|
||||||
|
events: readonly GachaEvent[],
|
||||||
|
options: SanitizeEventsOptions = {},
|
||||||
|
): SanitizeEventsResult {
|
||||||
|
const onNote = options.onNote ?? defaultReporter;
|
||||||
|
const kept: GachaEvent[] = [];
|
||||||
|
const notes: SanitizeNote[] = [];
|
||||||
|
|
||||||
|
for (const event of events) {
|
||||||
|
const result = sanitizeEvent(event, options);
|
||||||
|
for (const n of result.notes) {
|
||||||
|
notes.push(n);
|
||||||
|
onNote(n);
|
||||||
|
}
|
||||||
|
if (result.event !== null) kept.push(result.event);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { events: kept, notes };
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultReporter(note: SanitizeNote): void {
|
||||||
|
const prefix = note.level === "dropped" ? "! dropped" : " repaired";
|
||||||
|
console.warn(`${prefix} ${note.sourceId} ${note.field}: ${note.message}`);
|
||||||
|
}
|
||||||
@@ -0,0 +1,528 @@
|
|||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import { adapterById, ADAPTERS } from "../src/ingest/adapters/index.ts";
|
||||||
|
import { parserById } from "../src/ingest/parsers/index.ts";
|
||||||
|
import {
|
||||||
|
LIMITS,
|
||||||
|
sanitizeEvent,
|
||||||
|
sanitizeEvents,
|
||||||
|
sanitizeText,
|
||||||
|
sanitizeUrl,
|
||||||
|
type SanitizeNote,
|
||||||
|
} from "../src/ingest/sanitize.ts";
|
||||||
|
import { eventId, GachaEvent } from "../src/shared/schema.ts";
|
||||||
|
|
||||||
|
const NOW = "2026-08-14T00:00:00.000Z";
|
||||||
|
|
||||||
|
function event(overrides: Partial<GachaEvent> = {}): GachaEvent {
|
||||||
|
return {
|
||||||
|
id: "genshin:test-event:2026-08-12",
|
||||||
|
game: "genshin",
|
||||||
|
title: "Test Event",
|
||||||
|
type: "other",
|
||||||
|
summary: null,
|
||||||
|
startsAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
startPrecision: "day",
|
||||||
|
endsAt: "2026-08-24T00:00:00.000Z",
|
||||||
|
endPrecision: "day",
|
||||||
|
regionScoped: false,
|
||||||
|
regionEnds: null,
|
||||||
|
sourceUrl: "https://example.test/a",
|
||||||
|
sourceId: "source-a",
|
||||||
|
status: "published",
|
||||||
|
confidence: 0.9,
|
||||||
|
extractionMethod: "parser",
|
||||||
|
version: 1,
|
||||||
|
firstSeenAt: NOW,
|
||||||
|
updatedAt: NOW,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Collects notes instead of printing them, so a test can assert on them. */
|
||||||
|
function collector() {
|
||||||
|
const notes: SanitizeNote[] = [];
|
||||||
|
return { notes, onNote: (n: SanitizeNote) => void notes.push(n) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Everything hostile this module claims to handle, in one list.
|
||||||
|
*
|
||||||
|
* Reused by the idempotency and the no-throw properties below: any case added
|
||||||
|
* here is automatically held to both.
|
||||||
|
*/
|
||||||
|
const HOSTILE: Array<[label: string, input: string]> = [
|
||||||
|
["bare script", `<script>alert(1)</script>Windblume Festival`],
|
||||||
|
["style block", `<style>body{display:none}</style>Windblume`],
|
||||||
|
["nested tags", `<div><b>Wind<i>blume</i></b></div>`],
|
||||||
|
["malformed tag", `<<script>script>alert(1)</script>`],
|
||||||
|
["unclosed tag", `Windblume <img src=x onerror=alert(1)`],
|
||||||
|
["comment", `Wind<!-- <script>alert(1)</script> -->blume`],
|
||||||
|
["entity-encoded tag", `<script>alert(1)</script>`],
|
||||||
|
["double-encoded tag", `&lt;script&gt;alert(1)&lt;/script&gt;`],
|
||||||
|
["numeric entity tag", `<script>alert(1)</script>`],
|
||||||
|
["hex entity tag", `<script>alert(1)</script>`],
|
||||||
|
["fullwidth tag", `<script>alert(1)</script>`],
|
||||||
|
["zero-width in entity", `&am\u200bp;lt;script>`],
|
||||||
|
["deep entity nesting", `&#38;#38;#38;#38;#38;#38;lt;script>`],
|
||||||
|
["rtl override", `Login \u202eEvent\u202c`],
|
||||||
|
["bidi isolates", `\u2066Free\u2069\u2067Primogems\u2069`],
|
||||||
|
["control characters", `Wind\u0000blume\u0007 Fest\u001bival`],
|
||||||
|
["lone high surrogate", `Windblume \ud800`],
|
||||||
|
["lone low surrogate", `\udc00 Windblume`],
|
||||||
|
["out-of-range code point", `Windblume � �`],
|
||||||
|
["absurdly long", `A${"b".repeat(50_000)}`],
|
||||||
|
["whitespace storm", `\n\n Wind\t\t blume \u3000 Festival \n`],
|
||||||
|
["empty", ``],
|
||||||
|
["only markup", `<div></div><!-- x -->`],
|
||||||
|
];
|
||||||
|
|
||||||
|
describe("sanitizeText — markup", () => {
|
||||||
|
test("strips tags, keeping the text between them", () => {
|
||||||
|
expect(sanitizeText(`<div><b>Wind</b>blume <i>Festival</i></div>`)).toBe(
|
||||||
|
"Wind blume Festival",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("removes script and style content, not just the tags", () => {
|
||||||
|
expect(sanitizeText(`<script>alert(1)</script>Windblume`)).toBe("Windblume");
|
||||||
|
expect(sanitizeText(`<style>body{display:none}</style>Windblume`)).toBe(
|
||||||
|
"Windblume",
|
||||||
|
);
|
||||||
|
expect(sanitizeText(`<iframe src="//evil.test"></iframe>Windblume`)).toBe(
|
||||||
|
"Windblume",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("removes comments and their contents", () => {
|
||||||
|
expect(sanitizeText(`Wind<!-- <script>alert(1)</script> -->blume`)).toBe(
|
||||||
|
"Wind blume",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("leaves no tag-shaped substring, however malformed the input", () => {
|
||||||
|
for (const [label, input] of HOSTILE) {
|
||||||
|
expect(sanitizeText(input), label).not.toMatch(/<[^>]*>/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an unclosed tag does not swallow the rest of the string", () => {
|
||||||
|
// The tag itself goes, but the words around it survive — a source that
|
||||||
|
// forgets a `>` should cost us markup, not an event.
|
||||||
|
expect(sanitizeText(`Windblume <b>Festival`)).toBe("Windblume Festival");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeText — entities", () => {
|
||||||
|
test("decodes ordinary entities", () => {
|
||||||
|
expect(sanitizeText(`Tea & Cakes – Act II`)).toBe(
|
||||||
|
"Tea & Cakes – Act II",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an entity-encoded tag never becomes a live tag", () => {
|
||||||
|
expect(sanitizeText(`<script>alert(1)</script>`)).toBe("alert(1)");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("double-encoding does not survive one decode into markup", () => {
|
||||||
|
// The dangerous case: one decode yields `<script>`, which a second
|
||||||
|
// decoder anywhere downstream would turn into a tag. Decoding to a fixed
|
||||||
|
// point and stripping each round means the output is inert for every later
|
||||||
|
// reader too.
|
||||||
|
const out = sanitizeText(`&lt;script&gt;alert(1)&lt;/script&gt;`);
|
||||||
|
expect(out).not.toContain("<");
|
||||||
|
expect(out).not.toContain("<");
|
||||||
|
expect(sanitizeText(out)).toBe(out);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("numeric, hex and fullwidth spellings of a tag are all neutralised", () => {
|
||||||
|
for (const input of [
|
||||||
|
`<script>alert(1)</script>`,
|
||||||
|
`<script>alert(1)</script>`,
|
||||||
|
`<script>alert(1)</script>`,
|
||||||
|
]) {
|
||||||
|
expect(sanitizeText(input)).not.toMatch(/<[^>]*>/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an out-of-range code point is left alone rather than throwing", () => {
|
||||||
|
expect(() => sanitizeText(`� �`)).not.toThrow();
|
||||||
|
expect(sanitizeText(`Windblume �`)).toBe("Windblume �");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeText — unicode", () => {
|
||||||
|
test("normalises to NFKC", () => {
|
||||||
|
// Compatibility forms are how a source spoofs a title that reads the same.
|
||||||
|
expect(sanitizeText("Windblume")).toBe("Windblume");
|
||||||
|
expect(sanitizeText("éclair")).toBe("éclair");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("strips control characters", () => {
|
||||||
|
expect(sanitizeText("Wind\u0000blume\u0007 Fest\u001bival")).toBe(
|
||||||
|
"Windblume Festival",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("strips zero-width characters", () => {
|
||||||
|
expect(sanitizeText("Wind\u200bblume\u200c\u200d\ufeff")).toBe("Windblume");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("strips bidi overrides that would spoof a title", () => {
|
||||||
|
const spoofed = "Login \u202ednellA\u202c";
|
||||||
|
const clean = sanitizeText(spoofed);
|
||||||
|
expect(clean).not.toMatch(/[\u202a-\u202e\u2066-\u2069]/);
|
||||||
|
expect(clean).toBe("Login dnellA");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("drops unpaired surrogates but keeps real astral characters", () => {
|
||||||
|
expect(sanitizeText("Windblume \ud800")).toBe("Windblume");
|
||||||
|
expect(sanitizeText("\udc00 Windblume")).toBe("Windblume");
|
||||||
|
expect(sanitizeText("Windblume 🎉")).toBe("Windblume 🎉");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("collapses every flavour of whitespace to single spaces", () => {
|
||||||
|
expect(sanitizeText("\n\n Wind\t\t blume \u3000 Festival \n")).toBe(
|
||||||
|
"Wind blume Festival",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeText — length", () => {
|
||||||
|
test("truncates to the cap rather than discarding the value", () => {
|
||||||
|
const long = `Windblume ${"a".repeat(5_000)}`;
|
||||||
|
const title = sanitizeText(long, { maxLength: LIMITS.title });
|
||||||
|
expect(title.length).toBeLessThanOrEqual(LIMITS.title);
|
||||||
|
expect(title.startsWith("Windblume")).toBe(true);
|
||||||
|
expect(title.endsWith("…")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("cuts at a word boundary when there is one", () => {
|
||||||
|
const words = "Windblume Festival Returns To Mondstadt In Full Bloom";
|
||||||
|
expect(sanitizeText(words, { maxLength: 20 })).toBe("Windblume Festival…");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a value at the cap is left exactly as it is", () => {
|
||||||
|
const exact = "a".repeat(LIMITS.title);
|
||||||
|
expect(sanitizeText(exact, { maxLength: LIMITS.title })).toBe(exact);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the caps match what the schema will accept", () => {
|
||||||
|
// The schema is the single source of truth; LIMITS only mirrors it. If
|
||||||
|
// someone widens or narrows `title`/`summary` there, this fails here.
|
||||||
|
const ok = GachaEvent.safeParse(
|
||||||
|
event({ title: "a".repeat(LIMITS.title), summary: "b".repeat(LIMITS.summary) }),
|
||||||
|
);
|
||||||
|
expect(ok.success).toBe(true);
|
||||||
|
|
||||||
|
expect(GachaEvent.safeParse(event({ title: "a".repeat(LIMITS.title + 1) })).success).toBe(false);
|
||||||
|
expect(GachaEvent.safeParse(event({ summary: "b".repeat(LIMITS.summary + 1) })).success).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeText — totality", () => {
|
||||||
|
test("is idempotent for every hostile input", () => {
|
||||||
|
for (const [label, input] of HOSTILE) {
|
||||||
|
const once = sanitizeText(input);
|
||||||
|
expect(sanitizeText(once), label).toBe(once);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("never throws on junk", () => {
|
||||||
|
for (const [label, input] of HOSTILE) {
|
||||||
|
expect(() => sanitizeText(input), label).not.toThrow();
|
||||||
|
}
|
||||||
|
for (const junk of [null, undefined, 42, true, {}, [], Symbol("x")]) {
|
||||||
|
expect(() => sanitizeText(junk)).not.toThrow();
|
||||||
|
}
|
||||||
|
expect(sanitizeText(null)).toBe("");
|
||||||
|
expect(sanitizeText(undefined)).toBe("");
|
||||||
|
expect(sanitizeText({})).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeUrl", () => {
|
||||||
|
test("keeps http and https", () => {
|
||||||
|
expect(sanitizeUrl("https://game8.co/games/Genshin-Impact/archives/301601")).toBe(
|
||||||
|
"https://game8.co/games/Genshin-Impact/archives/301601",
|
||||||
|
);
|
||||||
|
expect(sanitizeUrl("http://example.test/a")).toBe("http://example.test/a");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects javascript:, data: and every other scheme", () => {
|
||||||
|
for (const bad of [
|
||||||
|
"javascript:alert(1)",
|
||||||
|
"JaVaScRiPt:alert(1)",
|
||||||
|
"java\tscript:alert(1)",
|
||||||
|
"javascript:alert(1)",
|
||||||
|
"data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
|
||||||
|
"vbscript:msgbox(1)",
|
||||||
|
"file:///etc/passwd",
|
||||||
|
"about:blank",
|
||||||
|
]) {
|
||||||
|
expect(sanitizeUrl(bad), bad).toBeNull();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects a URL carrying credentials", () => {
|
||||||
|
expect(sanitizeUrl("https://user:[email protected]/x")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects junk instead of throwing", () => {
|
||||||
|
for (const junk of ["", " ", "not a url", null, undefined, 42, {}]) {
|
||||||
|
expect(() => sanitizeUrl(junk)).not.toThrow();
|
||||||
|
expect(sanitizeUrl(junk)).toBeNull();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("resolves a relative href against the source page", () => {
|
||||||
|
expect(sanitizeUrl("/wiki/Event", { base: "https://endfield.wiki.gg/wiki/Home" })).toBe(
|
||||||
|
"https://endfield.wiki.gg/wiki/Event",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a relative href cannot smuggle in another scheme via the base", () => {
|
||||||
|
expect(
|
||||||
|
sanitizeUrl("javascript:alert(1)", { base: "https://endfield.wiki.gg/wiki/Home" }),
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeEvent", () => {
|
||||||
|
test("leaves a clean event completely alone", () => {
|
||||||
|
const clean = event({
|
||||||
|
id: eventId("genshin", "Windblume Festival", "2026-08-12T00:00:00.000Z"),
|
||||||
|
title: "Windblume Festival",
|
||||||
|
summary: "A festival in Mondstadt.",
|
||||||
|
});
|
||||||
|
const { event: out, notes } = sanitizeEvent(clean);
|
||||||
|
expect(out).toEqual(clean);
|
||||||
|
expect(notes).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("never touches timestamps, precision or confidence", () => {
|
||||||
|
const dirty = event({ title: "<b>Windblume</b>", summary: "<i>Blurb</i>" });
|
||||||
|
const { event: out } = sanitizeEvent(dirty);
|
||||||
|
expect(out?.startsAt).toBe(dirty.startsAt);
|
||||||
|
expect(out?.endsAt).toBe(dirty.endsAt);
|
||||||
|
expect(out?.startPrecision).toBe(dirty.startPrecision);
|
||||||
|
expect(out?.endPrecision).toBe(dirty.endPrecision);
|
||||||
|
expect(out?.confidence).toBe(dirty.confidence);
|
||||||
|
expect(out?.regionEnds).toBe(dirty.regionEnds);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("cleans a spoofed title and keeps the id agreeing with it", () => {
|
||||||
|
const startsAt = "2026-08-12T00:00:00.000Z";
|
||||||
|
const rawTitle = "Login \u202eEvent\u202c";
|
||||||
|
const dirty = event({
|
||||||
|
id: eventId("genshin", rawTitle, startsAt),
|
||||||
|
title: rawTitle,
|
||||||
|
startsAt,
|
||||||
|
});
|
||||||
|
|
||||||
|
// The override is what made this render as something other than its
|
||||||
|
// characters; removing it leaves the honest text, and the id follows.
|
||||||
|
const { event: out, notes } = sanitizeEvent(dirty, { sourceId: "source-a" });
|
||||||
|
expect(out?.title).toBe("Login Event");
|
||||||
|
expect(out?.id).toBe(eventId("genshin", "Login Event", startsAt));
|
||||||
|
expect(notes.every((n) => n.level === "repaired")).toBe(true);
|
||||||
|
|
||||||
|
// No id note here, and that is the point: `slugify` already drops
|
||||||
|
// characters like these, so cleaning the title moved nothing a user has
|
||||||
|
// saved state under.
|
||||||
|
expect(notes.map((n) => n.field)).toEqual(["title"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("recomputes the id when cleaning genuinely changes the slug", () => {
|
||||||
|
const startsAt = "2026-08-12T00:00:00.000Z";
|
||||||
|
const rawTitle = "<b>Windblume</b> Festival";
|
||||||
|
const dirty = event({
|
||||||
|
id: eventId("genshin", rawTitle, startsAt),
|
||||||
|
title: rawTitle,
|
||||||
|
startsAt,
|
||||||
|
});
|
||||||
|
|
||||||
|
const { event: out, notes } = sanitizeEvent(dirty);
|
||||||
|
expect(out?.title).toBe("Windblume Festival");
|
||||||
|
expect(out?.id).toBe("genshin:windblume-festival:2026-08-12");
|
||||||
|
expect(notes.some((n) => n.field === "id")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("leaves an id alone when it was not minted from the title", () => {
|
||||||
|
// Reconciliation keeps an existing id when a wiki renames an event
|
||||||
|
// (docs/INGESTION.md § Stage 5). Sanitising must not undo that.
|
||||||
|
const dirty = event({ id: "genshin:kept-across-a-rename:2026-08-12", title: "<b>New Name</b>" });
|
||||||
|
const { event: out } = sanitizeEvent(dirty);
|
||||||
|
expect(out?.title).toBe("New Name");
|
||||||
|
expect(out?.id).toBe("genshin:kept-across-a-rename:2026-08-12");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("truncates a hostile title instead of losing the event", () => {
|
||||||
|
const dirty = event({ title: `Windblume ${"a".repeat(10_000)}` });
|
||||||
|
const { event: out } = sanitizeEvent(dirty);
|
||||||
|
expect(out).not.toBeNull();
|
||||||
|
expect((out?.title ?? "").length).toBeLessThanOrEqual(LIMITS.title);
|
||||||
|
expect(GachaEvent.safeParse(out).success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an empty summary becomes null rather than an empty string", () => {
|
||||||
|
const { event: out } = sanitizeEvent(event({ summary: "<span> </span>" }));
|
||||||
|
expect(out?.summary).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("falls back to the source's own URL when the scraped one is hostile", () => {
|
||||||
|
const dirty = event({ sourceUrl: "javascript:alert(1)" });
|
||||||
|
const { event: out, notes } = sanitizeEvent(dirty, {
|
||||||
|
fallbackUrl: "https://game8.co/games/Genshin-Impact/archives/301601",
|
||||||
|
sourceId: "genshin-game8-events",
|
||||||
|
});
|
||||||
|
expect(out?.sourceUrl).toBe("https://game8.co/games/Genshin-Impact/archives/301601");
|
||||||
|
expect(notes.some((n) => n.field === "sourceUrl" && n.level === "repaired")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("drops an event only when nothing is left to publish, and says so", () => {
|
||||||
|
const gone = event({ title: "<script>alert(1)</script>" });
|
||||||
|
const { event: out, notes } = sanitizeEvent(gone, { sourceId: "source-a" });
|
||||||
|
expect(out).toBeNull();
|
||||||
|
expect(notes).toHaveLength(1);
|
||||||
|
expect(notes[0]?.level).toBe("dropped");
|
||||||
|
expect(notes[0]?.field).toBe("title");
|
||||||
|
expect(notes[0]?.sourceId).toBe("source-a");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("drops an event with no usable URL and no fallback, and says so", () => {
|
||||||
|
const { event: out, notes } = sanitizeEvent(event({ sourceUrl: "javascript:alert(1)" }));
|
||||||
|
expect(out).toBeNull();
|
||||||
|
expect(notes.some((n) => n.level === "dropped" && n.field === "sourceUrl")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("output always satisfies the schema", () => {
|
||||||
|
for (const [label, input] of HOSTILE) {
|
||||||
|
const { event: out } = sanitizeEvent(
|
||||||
|
event({ title: input, summary: input, sourceUrl: input }),
|
||||||
|
{ fallbackUrl: "https://example.test/a" },
|
||||||
|
);
|
||||||
|
if (out === null) continue;
|
||||||
|
expect(GachaEvent.safeParse(out).success, label).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeEvents", () => {
|
||||||
|
test("keeps order and reports every note", () => {
|
||||||
|
const { notes, onNote } = collector();
|
||||||
|
const { events } = sanitizeEvents(
|
||||||
|
[
|
||||||
|
event({ id: "genshin:a:2026-08-12", title: "Alpha" }),
|
||||||
|
event({ id: "genshin:b:2026-08-12", title: "<b>Beta</b>" }),
|
||||||
|
event({ id: "genshin:c:2026-08-12", title: "Gamma" }),
|
||||||
|
],
|
||||||
|
{ onNote, sourceId: "source-a" },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(events.map((e) => e.title)).toEqual(["Alpha", "Beta", "Gamma"]);
|
||||||
|
expect(notes).toHaveLength(1);
|
||||||
|
expect(notes[0]?.field).toBe("title");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a dropped event never disappears silently", () => {
|
||||||
|
const { notes, onNote } = collector();
|
||||||
|
const { events } = sanitizeEvents(
|
||||||
|
[event({ title: "Alpha" }), event({ title: "<!-- nothing -->" })],
|
||||||
|
{ onNote },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(events).toHaveLength(1);
|
||||||
|
expect(notes.filter((n) => n.level === "dropped")).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("siblings survive a dropped event", () => {
|
||||||
|
const { onNote } = collector();
|
||||||
|
const { events } = sanitizeEvents(
|
||||||
|
[event({ title: "<script>x</script>" }), event({ title: "Beta" })],
|
||||||
|
{ onNote },
|
||||||
|
);
|
||||||
|
expect(events.map((e) => e.title)).toEqual(["Beta"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("the adapter seam", () => {
|
||||||
|
/**
|
||||||
|
* A minimal page in Game8's shape, carrying everything a hostile source could
|
||||||
|
* put in a title. The point is not the parser — it is that `adapter.parse`
|
||||||
|
* cannot return unsanitised events, whichever parser produced them.
|
||||||
|
*/
|
||||||
|
const HOSTILE_PAGE = `
|
||||||
|
<h2 class="a-header--2">Current Events</h2>
|
||||||
|
<h3 class="a-header--3">Windblume \u202eFestival\u202c\u200b & Friends</h3>
|
||||||
|
<table class="a-table">
|
||||||
|
<tr><th>Event Start</th><td>August 12, 2026</td></tr>
|
||||||
|
<tr><th>Event End</th><td>August 24, 2026</td></tr>
|
||||||
|
</table>
|
||||||
|
<p class="a-paragraph">A blurb with <script>alert(1)</script> in it.</p>
|
||||||
|
`;
|
||||||
|
|
||||||
|
test("events coming out of an adapter are already sanitised", () => {
|
||||||
|
const adapter = adapterById("genshin-game8-events");
|
||||||
|
if (adapter === undefined) throw new Error("no adapter");
|
||||||
|
|
||||||
|
const events = adapter.parse(HOSTILE_PAGE, {
|
||||||
|
now: NOW,
|
||||||
|
sourceUrl: adapter.url,
|
||||||
|
sourceId: adapter.id,
|
||||||
|
game: adapter.game,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(events).toHaveLength(1);
|
||||||
|
const parsed = events[0];
|
||||||
|
expect(parsed?.title).toBe("Windblume Festival & Friends");
|
||||||
|
expect(parsed?.title).not.toMatch(/[\u200b-\u200f\u202a-\u202e]/);
|
||||||
|
expect(parsed?.summary ?? "").not.toMatch(/<[^>]*>/);
|
||||||
|
expect(parsed?.sourceUrl).toBe(adapter.url);
|
||||||
|
// Dates are the one thing sanitising must never touch.
|
||||||
|
expect(parsed?.startsAt).toBe("2026-08-12T00:00:00.000Z");
|
||||||
|
expect(parsed?.endsAt).toBe("2026-08-24T00:00:00.000Z");
|
||||||
|
// The id agrees with the title the user actually sees.
|
||||||
|
expect(parsed?.id).toBe(
|
||||||
|
eventId("genshin", parsed?.title ?? "", parsed?.startsAt ?? ""),
|
||||||
|
);
|
||||||
|
expect(GachaEvent.safeParse(parsed).success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The guard that matters most.
|
||||||
|
*
|
||||||
|
* Event ids are localStorage keys, so sanitising must be a *no-op* on the
|
||||||
|
* pages we actually parse — if it ever starts repairing a real title, the id
|
||||||
|
* derived from that title can move and every completion mark saved under the
|
||||||
|
* old one is orphaned with no server-side recovery. Running the parsers
|
||||||
|
* directly (before the adapter seam cleans anything) and asserting zero notes
|
||||||
|
* is what makes that visible the moment it changes.
|
||||||
|
*/
|
||||||
|
test("real fixtures need no repair at all", async () => {
|
||||||
|
for (const adapter of ADAPTERS) {
|
||||||
|
const parser = parserById(adapter.parserId);
|
||||||
|
if (parser === undefined) throw new Error(`no parser ${adapter.parserId}`);
|
||||||
|
|
||||||
|
const pattern = `fixtures/${adapter.game}/${adapter.parserId}-*.html`;
|
||||||
|
const file = [...new Bun.Glob(pattern).scanSync(".")].sort().at(-1);
|
||||||
|
if (file === undefined) throw new Error(`no fixture for ${adapter.id}`);
|
||||||
|
|
||||||
|
const html = await Bun.file(file).text();
|
||||||
|
const raw = parser.parse(html, {
|
||||||
|
now: NOW,
|
||||||
|
sourceUrl: adapter.url,
|
||||||
|
sourceId: adapter.id,
|
||||||
|
game: adapter.game,
|
||||||
|
});
|
||||||
|
|
||||||
|
const { notes, events } = sanitizeEvents(raw, {
|
||||||
|
sourceId: adapter.id,
|
||||||
|
fallbackUrl: adapter.url,
|
||||||
|
onNote: () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(notes.map((n) => `${n.field}: ${n.message}`), adapter.id).toEqual([]);
|
||||||
|
expect(events.map((e) => e.id), adapter.id).toEqual(raw.map((e) => e.id));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user