fix: keep sanitised text stable when it is truncated
The cut was marked with U+2026, which NFKC decomposes into three dots — so re-sanitising a truncated string grew it by two characters and re-cut it at a different word boundary. A title would quietly rewrite itself every time the event was re-ingested, and the module promises idempotency in its own docstring. Append what normalisation would produce instead. The existing corpus missed this because its only over-length entry has no space in its last 40%, so it happened to re-truncate to the identical string; the new test uses prose. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
02863ed008
commit
1bc7b0057c
+13
-2
@@ -149,16 +149,27 @@ function collapse(input: string): string {
|
||||
function truncate(input: string, max: number): string {
|
||||
if (input.length <= max) return input;
|
||||
|
||||
let cut = input.slice(0, max - 1);
|
||||
let cut = input.slice(0, max - ELLIPSIS.length);
|
||||
// Never end on half a surrogate pair.
|
||||
if (/[\uD800-\uDBFF]$/.test(cut)) cut = cut.slice(0, -1);
|
||||
|
||||
const lastSpace = cut.lastIndexOf(" ");
|
||||
if (lastSpace > max * 0.6) cut = cut.slice(0, lastSpace);
|
||||
|
||||
return `${cut.replace(/[\s,;:.–—-]+$/, "")}…`;
|
||||
return `${cut.replace(/[\s,;:.–—-]+$/, "")}${ELLIPSIS}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Three dots, not U+2026.
|
||||
*
|
||||
* NFKC decomposes the ellipsis character into these three anyway, so a "…"
|
||||
* appended here would grow by two characters on the next pass and re-cut the
|
||||
* string at a different word boundary — breaking the idempotency this module
|
||||
* promises. Writing what normalisation would produce keeps the second pass a
|
||||
* no-op.
|
||||
*/
|
||||
const ELLIPSIS = "...";
|
||||
|
||||
export interface SanitizeTextOptions {
|
||||
/** Hard cap; the result is never longer. Defaults to the summary cap. */
|
||||
maxLength?: number;
|
||||
|
||||
Reference in New Issue
Block a user