feat: sanitise every string a source publishes
Scraped text becomes React content, JSON on disk and eventually SQLite rows, so it gets cleaned at one seam: the parse wrapper in toAdapter(). Every source passes through it, a source added tomorrow is covered without its author doing anything, and no parser can opt out — parsers stay pure readers of one site's markup. Removes script/style/comment content and residual tags, decodes entities to a fixed point so an encoded tag cannot resurrect in a later decoder, NFKC-normalises, strips control, zero-width and bidi-override characters (an RTL override visually spoofs a title), bounds each field to the cap the schema already declares, and requires sourceUrl to be absolute http(s). Three things it will not do: touch a date, drop an event it could clean instead, or repair in silence — every repair and drop is logged by default. Event IDs are localStorage keys, so an ID is recomputed only when a sanitised title actually changed and the ID was minted the standard way; all seven fixtures pass through unrepaired and byte-identical, which is the regression guard. Also stops decodeEntities throwing RangeError on an out-of-range numeric reference, which would have taken a whole source's events down. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5
Claude Opus 5
parent
2b9338a8b0
commit
18b9652aed
@@ -0,0 +1,362 @@
|
||||
import { eventId, type GachaEvent } from "../shared/schema.ts";
|
||||
import { decodeEntities } from "./html.ts";
|
||||
|
||||
/**
|
||||
* The trust boundary for scraped text.
|
||||
*
|
||||
* Every string on a `GachaEvent` starts life as bytes from a community wiki we
|
||||
* do not control. Between the parser and validation it passes through here, so
|
||||
* that what reaches the feed — and therefore React, `localStorage`, JSON on
|
||||
* disk and eventually SQLite — is plain, bounded, normalised text.
|
||||
*
|
||||
* Three principles, in priority order:
|
||||
*
|
||||
* 1. **Never invent or alter a date.** Nothing in this module reads, writes or
|
||||
* reformats a timestamp. Dates are the product's whole promise; the
|
||||
* sanitiser's job stops at prose and URLs.
|
||||
* 2. **Clean, do not drop.** A hostile title is truncated and stripped, not
|
||||
* rejected — an event vanishing without a trace is the failure mode this
|
||||
* codebase fears most (CLAUDE.md § Silent drops). The one unrecoverable
|
||||
* case is a title that sanitises to nothing, and that emits a note the
|
||||
* caller is expected to surface.
|
||||
* 3. **Never throw on junk.** Malformed entities, lone surrogates, absurd code
|
||||
* points and 5MB strings all have to come out the other side as a string.
|
||||
*
|
||||
* Dependency-free by design: no DOMPurify, no sanitize-html, no parse5.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Length caps, mirroring `title`/`summary` in `src/shared/schema.ts`.
|
||||
*
|
||||
* The schema stays the single source of truth — these exist so a hostile page
|
||||
* is truncated *before* validation instead of failing it, and
|
||||
* `test/sanitize.test.ts` asserts that a string of exactly this length is
|
||||
* accepted by `GachaEvent` and one character more is not, so the two cannot
|
||||
* drift apart unnoticed.
|
||||
*/
|
||||
export const LIMITS = {
|
||||
title: 200,
|
||||
summary: 500,
|
||||
/** Not a schema cap: a defensive ceiling so a junk href cannot be a novel. */
|
||||
url: 2048,
|
||||
} as const;
|
||||
|
||||
/** How many decode/strip rounds before giving up and hard-scrubbing. */
|
||||
const MAX_PASSES = 5;
|
||||
|
||||
/**
|
||||
* Characters that are invisible, that control how surrounding text is
|
||||
* *displayed*, or that are not legal text at all.
|
||||
*
|
||||
* The bidi overrides and isolates (U+202A–U+202E, U+2066–U+2069) matter most:
|
||||
* they let a source render "Login Event" as something else entirely, or hide a
|
||||
* suffix from a reader while it still lands in the title, the slug and the
|
||||
* user's saved state. Zero-width characters do the same job more crudely and
|
||||
* additionally let an attacker smuggle `&am<ZWSP>p;lt;` past a naive decoder.
|
||||
*
|
||||
* Note ZWJ (U+200D) goes too, which splits multi-part emoji into their
|
||||
* components. Our sources are English-language wikis; a mangled family emoji is
|
||||
* a fair price for no invisible characters anywhere in a title.
|
||||
*/
|
||||
const INVISIBLE =
|
||||
/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u00AD\u061C\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\uFEFF\uFFF9-\uFFFB\uFFFE\uFFFF]/g;
|
||||
|
||||
/** Unicode tag characters (U+E0000–U+E007F) — invisible, as surrogate pairs. */
|
||||
const TAG_CHARS = /\uDB40[\uDC00-\uDC7F]/g;
|
||||
|
||||
/** Half of a surrogate pair with no partner: not valid text, breaks JSON. */
|
||||
const LONE_SURROGATE =
|
||||
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/g;
|
||||
|
||||
/** Elements whose *content* is code, not text, and must go with the tag. */
|
||||
const CODE_BLOCKS =
|
||||
/<(script|style|template|noscript|iframe|object|embed|svg|math)\b[\s\S]*?(?:<\/\1\s*>|$)/gi;
|
||||
|
||||
const COMMENTS = /<!--[\s\S]*?(?:-->|$)|<!\[CDATA\[[\s\S]*?(?:\]\]>|$)|<\?[\s\S]*?(?:\?>|$)/g;
|
||||
|
||||
const TAG = /<[^>]*>/g;
|
||||
|
||||
function stripInvisible(input: string): string {
|
||||
return input
|
||||
.replace(TAG_CHARS, "")
|
||||
.replace(INVISIBLE, "")
|
||||
.replace(LONE_SURROGATE, (m) => (m.length === 2 ? m[0] ?? "" : ""));
|
||||
}
|
||||
|
||||
function stripTags(input: string): string {
|
||||
let out = input;
|
||||
for (let i = 0; i < MAX_PASSES; i += 1) {
|
||||
const next = out.replace(TAG, " ");
|
||||
if (next === out) return out;
|
||||
out = next;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* One round of neutralising.
|
||||
*
|
||||
* Order is deliberate. NFKC runs *first* because it turns look-alike forms into
|
||||
* their canonical ones (U+FF1C FULLWIDTH LESS-THAN becomes `<`), and a
|
||||
* normaliser running after the tag stripper would hand back live markup.
|
||||
* Invisible characters go next so they cannot break up an entity or a tag name.
|
||||
* Only then is anything decoded, and whatever the decode produced is stripped
|
||||
* in the same round.
|
||||
*/
|
||||
function pass(input: string): string {
|
||||
const normalised = stripInvisible(input.normalize("NFKC"));
|
||||
const withoutCode = normalised.replace(CODE_BLOCKS, " ").replace(COMMENTS, " ");
|
||||
return stripTags(decodeEntities(withoutCode));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reduce text to a fixed point of `pass`.
|
||||
*
|
||||
* Running to a fixed point is what makes the whole module idempotent, and it is
|
||||
* also the answer to double-encoding: `&lt;script>` decodes to
|
||||
* `<script>` on the first round and to a tag on the second, which the second
|
||||
* round then strips. Stopping after one decode would leave a string that a
|
||||
* later `sanitizeText` — or any other decoder downstream — turns into markup.
|
||||
*/
|
||||
function toFixedPoint(input: string): string {
|
||||
let out = input;
|
||||
for (let i = 0; i < MAX_PASSES; i += 1) {
|
||||
const next = pass(out);
|
||||
if (next === out) return out;
|
||||
out = next;
|
||||
}
|
||||
// Pathological input that keeps re-encoding itself — `&#38;#38;…` nested
|
||||
// deeper than the round limit. Removing every `<`, `>` and `&` both kills any
|
||||
// tag shape and guarantees the result is a fixed point (nothing left to
|
||||
// decode), which is what keeps `sanitizeText` idempotent even here.
|
||||
return out.replace(/[<>&]/g, " ");
|
||||
}
|
||||
|
||||
/** Collapse every run of whitespace — including NBSP and U+2028 — to one space. */
|
||||
function collapse(input: string): string {
|
||||
return input
|
||||
.replace(/[\s\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Cut to `max` characters at a word boundary, marking the cut with an ellipsis.
|
||||
*
|
||||
* Truncating beats rejecting: a real event with a bloated description is still
|
||||
* a real event, and the user would rather see it than not. The result is always
|
||||
* `<= max`, so a second pass never truncates again.
|
||||
*/
|
||||
function truncate(input: string, max: number): string {
|
||||
if (input.length <= max) return input;
|
||||
|
||||
let cut = input.slice(0, max - 1);
|
||||
// Never end on half a surrogate pair.
|
||||
if (/[\uD800-\uDBFF]$/.test(cut)) cut = cut.slice(0, -1);
|
||||
|
||||
const lastSpace = cut.lastIndexOf(" ");
|
||||
if (lastSpace > max * 0.6) cut = cut.slice(0, lastSpace);
|
||||
|
||||
return `${cut.replace(/[\s,;:.–—-]+$/, "")}…`;
|
||||
}
|
||||
|
||||
export interface SanitizeTextOptions {
|
||||
/** Hard cap; the result is never longer. Defaults to the summary cap. */
|
||||
maxLength?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean one string extracted from a source.
|
||||
*
|
||||
* Total: any input, including `null`, a number or a 5MB blob of markup, yields
|
||||
* a string. Idempotent: `sanitizeText(sanitizeText(x)) === sanitizeText(x)`.
|
||||
*/
|
||||
export function sanitizeText(
|
||||
input: unknown,
|
||||
options: SanitizeTextOptions = {},
|
||||
): string {
|
||||
// Anything that is not a primitive is not text a source stated; "" is the
|
||||
// honest reading of it, and stringifying an object would invent content.
|
||||
const raw =
|
||||
typeof input === "string"
|
||||
? input
|
||||
: typeof input === "number" || typeof input === "boolean"
|
||||
? String(input)
|
||||
: "";
|
||||
const max = options.maxLength ?? LIMITS.summary;
|
||||
return truncate(collapse(toFixedPoint(raw)), max);
|
||||
}
|
||||
|
||||
export interface SanitizeUrlOptions {
|
||||
/** Resolves a relative href, exactly as `new URL(href, base)` would. */
|
||||
base?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return `input` as an absolute http(s) URL, or null if it is not one.
|
||||
*
|
||||
* `sourceUrl` is rendered as an attribution link, so a `javascript:` or `data:`
|
||||
* URL that reached the client would be a live XSS vector in an app that
|
||||
* otherwise never handles untrusted URLs. Anything that is not plainly http(s)
|
||||
* — including a URL carrying credentials, which is only ever a phishing shape —
|
||||
* is refused, and the caller falls back to the source's registered URL.
|
||||
*/
|
||||
export function sanitizeUrl(
|
||||
input: unknown,
|
||||
options: SanitizeUrlOptions = {},
|
||||
): string | null {
|
||||
const raw = sanitizeText(input, { maxLength: LIMITS.url });
|
||||
if (raw.length === 0) return null;
|
||||
|
||||
let url: URL;
|
||||
try {
|
||||
url = options.base === undefined ? new URL(raw) : new URL(raw, options.base);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (url.username !== "" || url.password !== "") return null;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export type NoteLevel = "repaired" | "dropped";
|
||||
|
||||
export interface SanitizeNote {
|
||||
level: NoteLevel;
|
||||
sourceId: string;
|
||||
field: "title" | "summary" | "sourceUrl" | "id";
|
||||
/** Human-readable, already truncated — safe to print to a log. */
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface SanitizeEventOptions {
|
||||
/**
|
||||
* The source's registered URL. Known good (it came from `SOURCES`, not from
|
||||
* the page), so it is the fallback when an event's own `sourceUrl` is junk —
|
||||
* attribution to the right page beats discarding the event.
|
||||
*/
|
||||
fallbackUrl?: string;
|
||||
/** Reported on every note, so a log line names the source that produced it. */
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface SanitizeEventResult {
|
||||
/** Null only when the event could not be repaired into a publishable shape. */
|
||||
event: GachaEvent | null;
|
||||
notes: SanitizeNote[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean every source-derived string on one event.
|
||||
*
|
||||
* Timestamps, precisions, confidence, region data and `sourceId` are passed
|
||||
* through untouched: the first three are numbers and dates this module has no
|
||||
* business rewriting, and the last comes from our own registry rather than from
|
||||
* the page.
|
||||
*
|
||||
* The `id` is only recomputed when sanitising actually changed the title *and*
|
||||
* the incoming id was minted the standard way (`eventId(game, title, startsAt)`).
|
||||
* That keeps two guarantees at once: an id never disagrees with the title it
|
||||
* encodes, and — because sanitising a clean title is a no-op — no id in the
|
||||
* current feed moves. Ids are localStorage keys; a gratuitous change there
|
||||
* orphans completion marks with no server-side recovery.
|
||||
*/
|
||||
export function sanitizeEvent(
|
||||
event: GachaEvent,
|
||||
options: SanitizeEventOptions = {},
|
||||
): SanitizeEventResult {
|
||||
const sourceId = options.sourceId ?? sanitizeText(event.sourceId, { maxLength: 120 });
|
||||
const notes: SanitizeNote[] = [];
|
||||
const note = (level: NoteLevel, field: SanitizeNote["field"], message: string) => {
|
||||
notes.push({ level, sourceId, field, message });
|
||||
};
|
||||
|
||||
const rawTitle = typeof event.title === "string" ? event.title : "";
|
||||
const title = sanitizeText(rawTitle, { maxLength: LIMITS.title });
|
||||
if (title.length === 0) {
|
||||
note(
|
||||
"dropped",
|
||||
"title",
|
||||
`title sanitised to nothing (raw: ${JSON.stringify(rawTitle.slice(0, 80))})`,
|
||||
);
|
||||
return { event: null, notes };
|
||||
}
|
||||
if (title !== rawTitle) {
|
||||
note("repaired", "title", `${JSON.stringify(rawTitle.slice(0, 80))} → ${JSON.stringify(title)}`);
|
||||
}
|
||||
|
||||
let summary: string | null = null;
|
||||
if (typeof event.summary === "string") {
|
||||
const cleaned = sanitizeText(event.summary, { maxLength: LIMITS.summary });
|
||||
summary = cleaned.length === 0 ? null : cleaned;
|
||||
if (cleaned !== event.summary) {
|
||||
note("repaired", "summary", `summary cleaned (${event.summary.length} → ${cleaned.length} chars)`);
|
||||
}
|
||||
}
|
||||
|
||||
const base = options.fallbackUrl;
|
||||
const sourceUrl =
|
||||
sanitizeUrl(event.sourceUrl, base === undefined ? {} : { base }) ??
|
||||
sanitizeUrl(base);
|
||||
if (sourceUrl === null) {
|
||||
note(
|
||||
"dropped",
|
||||
"sourceUrl",
|
||||
`no usable http(s) source URL (raw: ${JSON.stringify(String(event.sourceUrl).slice(0, 120))})`,
|
||||
);
|
||||
return { event: null, notes };
|
||||
}
|
||||
if (sourceUrl !== event.sourceUrl) {
|
||||
note("repaired", "sourceUrl", `${JSON.stringify(String(event.sourceUrl).slice(0, 120))} → ${JSON.stringify(sourceUrl)}`);
|
||||
}
|
||||
|
||||
let id = event.id;
|
||||
if (title !== rawTitle && id === eventId(event.game, rawTitle, event.startsAt)) {
|
||||
id = eventId(event.game, title, event.startsAt);
|
||||
if (id !== event.id) {
|
||||
note("repaired", "id", `${event.id} → ${id} (title was sanitised)`);
|
||||
}
|
||||
}
|
||||
|
||||
return { event: { ...event, id, title, summary, sourceUrl }, notes };
|
||||
}
|
||||
|
||||
export interface SanitizeEventsResult {
|
||||
events: GachaEvent[];
|
||||
notes: SanitizeNote[];
|
||||
}
|
||||
|
||||
export interface SanitizeEventsOptions extends SanitizeEventOptions {
|
||||
/**
|
||||
* Where notes go. Defaults to `console.warn`, on purpose: a dropped event
|
||||
* must never be silent, and defaulting to a no-op would make silence the
|
||||
* behaviour a future caller gets for free.
|
||||
*/
|
||||
onNote?: (note: SanitizeNote) => void;
|
||||
}
|
||||
|
||||
/** Sanitise a parser's whole output. Order is preserved; drops are reported. */
|
||||
export function sanitizeEvents(
|
||||
events: readonly GachaEvent[],
|
||||
options: SanitizeEventsOptions = {},
|
||||
): SanitizeEventsResult {
|
||||
const onNote = options.onNote ?? defaultReporter;
|
||||
const kept: GachaEvent[] = [];
|
||||
const notes: SanitizeNote[] = [];
|
||||
|
||||
for (const event of events) {
|
||||
const result = sanitizeEvent(event, options);
|
||||
for (const n of result.notes) {
|
||||
notes.push(n);
|
||||
onNote(n);
|
||||
}
|
||||
if (result.event !== null) kept.push(result.event);
|
||||
}
|
||||
|
||||
return { events: kept, notes };
|
||||
}
|
||||
|
||||
function defaultReporter(note: SanitizeNote): void {
|
||||
const prefix = note.level === "dropped" ? "! dropped" : " repaired";
|
||||
console.warn(`${prefix} ${note.sourceId} ${note.field}: ${note.message}`);
|
||||
}
|
||||
Reference in New Issue
Block a user