feat: sanitise every string a source publishes
Scraped text becomes React content, JSON on disk and eventually SQLite rows, so it gets cleaned at one seam: the parse wrapper in toAdapter(). Every source passes through it, a source added tomorrow is covered without its author doing anything, and no parser can opt out — parsers stay pure readers of one site's markup. Removes script/style/comment content and residual tags, decodes entities to a fixed point so an encoded tag cannot resurrect in a later decoder, NFKC-normalises, strips control, zero-width and bidi-override characters (an RTL override visually spoofs a title), bounds each field to the cap the schema already declares, and requires sourceUrl to be absolute http(s). Three things it will not do: touch a date, drop an event it could clean instead, or repair in silence — every repair and drop is logged by default. Event IDs are localStorage keys, so an ID is recomputed only when a sanitised title actually changed and the ID was minted the standard way; all seven fixtures pass through unrepaired and byte-identical, which is the regression guard. Also stops decodeEntities throwing RangeError on an out-of-range numeric reference, which would have taken a whole source's events down. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
2b9338a8b0
commit
18b9652aed
+18
-4
@@ -23,13 +23,27 @@ const ENTITIES: Record<string, string> = {
|
||||
rdquo: "”",
|
||||
};
|
||||
|
||||
/**
|
||||
* A numeric reference the page may or may not mean literally.
|
||||
*
|
||||
* Out-of-range and non-finite code points come back as the original text rather
|
||||
* than throwing: `�` and `�` are junk a hostile or merely
|
||||
* broken page can emit, and `String.fromCodePoint` throws RangeError on both.
|
||||
* A parser crashing on one bad character would take a whole source's events
|
||||
* down with it.
|
||||
*/
|
||||
function fromCodePoint(value: number, original: string): string {
|
||||
if (!Number.isInteger(value) || value < 0 || value > 0x10ffff) return original;
|
||||
return String.fromCodePoint(value);
|
||||
}
|
||||
|
||||
export function decodeEntities(input: string): string {
|
||||
return input
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) =>
|
||||
String.fromCodePoint(parseInt(hex, 16)),
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (whole, hex: string) =>
|
||||
fromCodePoint(parseInt(hex, 16), whole),
|
||||
)
|
||||
.replace(/&#(\d+);/g, (_, dec: string) =>
|
||||
String.fromCodePoint(parseInt(dec, 10)),
|
||||
.replace(/&#(\d+);/g, (whole, dec: string) =>
|
||||
fromCodePoint(parseInt(dec, 10), whole),
|
||||
)
|
||||
.replace(/&([a-zA-Z]+);/g, (whole, name: string) => ENTITIES[name] ?? whole);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user