fix: harden the fetch pipeline for a real server
A review of code that has never made a live request. Each of these was reproduced before it was fixed. robots.txt matched a group by testing whether our whole User-Agent contained the group's name. Our contact URL carries "StereotypicalCat", so a wiki writing `User-agent: cat` matched us — and because a named group *replaces* the wildcard group, that silently discarded every rule under `User-agent: *`. Match the RFC 9309 product token instead. A 200 carrying an HTML "not found" page also parsed to zero rules and read as permission; it now fails closed, while a genuinely empty body still means "no restrictions". The response body was read outside the try that guarded the request, so one truncated body aborted the whole cycle: later sources were never fetched, and the failed source never recorded its check, meaning a re-dispatch would ask that wiki again minutes later. Bodies were decoded as UTF-8 unconditionally and stored re-encoded. A page served in a legacy charset became replacement characters with the original bytes gone — and mojibake in a title flows into slugify and moves every localStorage key for that source. Decode by the declared charset, keep the served bytes verbatim, and hash those. Unchanged bytes skipped the metadata write, so once a server rotated its ETag we sent a stale validator forever and it served full bodies instead of 304s. Also: snapshot writes go through a temp file and a rename, and `--only` with no value is an argument error rather than "all sources". Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5
Claude Opus 5
parent
99786f6fc9
commit
c85ec0b2d1
+69
-12
@@ -137,6 +137,26 @@ export function agentToken(userAgent: string): string {
|
||||
return first.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Does a `User-agent:` value in robots.txt name us?
|
||||
*
|
||||
* RFC 9309 § 2.2.1 matches the *product token* — the header up to the first
|
||||
* `/` — not the header text. Matching anywhere in the header is actively
|
||||
* dangerous here: our contact URL contains the string `StereotypicalCat`, so a
|
||||
* `User-agent: cat` group elsewhere in the file would be treated as naming us,
|
||||
* and because a named group replaces the `*` group outright, that unrelated
|
||||
* group's rules would *discard* every rule the site actually wrote for us.
|
||||
* Erring towards obeying more rules means never letting a coincidence take a
|
||||
* `*` group away.
|
||||
*
|
||||
* A robots.txt that names us with a version (`gacha-event-tracker/1.0`) is
|
||||
* still honoured: the group's own product token is compared too.
|
||||
*/
|
||||
function agentNames(agent: string, token: string): boolean {
|
||||
if (agent === "*") return false;
|
||||
return agent === token || agentToken(agent) === token;
|
||||
}
|
||||
|
||||
/**
|
||||
* The group that applies to a user agent, with every group naming the same
|
||||
* agent merged, as RFC 9309 requires.
|
||||
@@ -150,19 +170,11 @@ export function groupFor(
|
||||
userAgent: string,
|
||||
): RobotsGroup | null {
|
||||
const token = agentToken(userAgent);
|
||||
const full = userAgent.toLowerCase();
|
||||
|
||||
let bestName: string | null = null;
|
||||
for (const group of robots.groups) {
|
||||
for (const agent of group.agents) {
|
||||
if (agent === "*") continue;
|
||||
// Match on the product token first (the spec's rule); fall back to a
|
||||
// substring of the whole header so a group naming "gptbot" still binds a
|
||||
// header of "Mozilla/5.0 (compatible; GPTBot/1.2)". Erring towards
|
||||
// matching means erring towards obeying more rules, not fewer.
|
||||
const hit =
|
||||
token === agent || token.startsWith(agent) || full.includes(agent);
|
||||
if (!hit) continue;
|
||||
if (!agentNames(agent, token)) continue;
|
||||
if (bestName === null || agent.length > bestName.length) bestName = agent;
|
||||
}
|
||||
}
|
||||
@@ -280,8 +292,10 @@ const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
* One robots.txt fetch per host per run (cached 24h), reused by every source on
|
||||
* that host — six Game8 adapters must not mean six robots requests.
|
||||
*
|
||||
* Fails closed. A 5xx, a timeout or a network error means we do not know what
|
||||
* the site permits, and "unknown" is not permission.
|
||||
* Fails closed. A 5xx, a timeout, a network error, a body that dies mid-read
|
||||
* or a body that is plainly not robots.txt all mean we do not know what the
|
||||
* site permits, and "unknown" is not permission. Only two answers open the
|
||||
* host: a parsed robots.txt, and a 404/410 saying there is none.
|
||||
*/
|
||||
export class RobotsCache {
|
||||
private readonly entries = new Map<string, CacheEntry>();
|
||||
@@ -368,7 +382,50 @@ export class RobotsCache {
|
||||
};
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
// Reading the body is a second chance to fail: the connection can reset or
|
||||
// the timeout can fire mid-stream, long after the headers arrived. Outside
|
||||
// the try that rejection would escape as an exception rather than as "we
|
||||
// could not read robots.txt", which is the one answer this class exists to
|
||||
// give.
|
||||
let text: string;
|
||||
try {
|
||||
text = await response.text();
|
||||
} catch (error) {
|
||||
return {
|
||||
robots: ALLOW_ALL,
|
||||
usable: false,
|
||||
reason: `robots.txt body unreadable (${String(error)})`,
|
||||
at,
|
||||
};
|
||||
}
|
||||
|
||||
// A soft 404 — an HTML "not found" page served with status 200 — is the
|
||||
// commonest robots.txt misconfiguration there is, and it parses to zero
|
||||
// groups, which is indistinguishable from "everything is permitted". We do
|
||||
// not know what the site allows, and unknown is not permission.
|
||||
if (looksLikeHtml(text)) {
|
||||
return {
|
||||
robots: ALLOW_ALL,
|
||||
usable: false,
|
||||
reason: "robots.txt returned HTML, not a robots.txt (soft 404?)",
|
||||
at,
|
||||
};
|
||||
}
|
||||
|
||||
return { robots: parseRobots(text), usable: true, reason: "robots.txt ok", at };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Is this body markup rather than robots.txt?
|
||||
*
|
||||
* An empty body is *valid* robots.txt meaning "no restrictions", so emptiness
|
||||
* is deliberately not a failure. Only markup is — no robots.txt directive can
|
||||
* begin with `<`.
|
||||
*/
|
||||
export function looksLikeHtml(text: string): boolean {
|
||||
const head = text.trimStart().slice(0, 512).toLowerCase();
|
||||
if (head === "") return false;
|
||||
if (head.startsWith("<")) return true;
|
||||
return /<!doctype html|<html[\s>]|<head[\s>]|<body[\s>]/.test(head);
|
||||
}
|
||||
|
||||
+153
-20
@@ -18,14 +18,15 @@
|
||||
* the metadata, every cycle would produce a commit that says nothing, and
|
||||
* "commit only when something changed" would be unenforceable.
|
||||
*/
|
||||
import { mkdir, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdir, readdir, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
export interface SnapshotMeta {
|
||||
sourceId: string;
|
||||
url: string;
|
||||
/** sha256 of the body, hex. The parse stage skips work when it is unchanged. */
|
||||
/** sha256 of the served bytes, hex. The parse stage skips work when it is unchanged. */
|
||||
contentHash: string;
|
||||
/** The served length, in bytes — not the length of the decoded text. */
|
||||
bytes: number;
|
||||
etag: string | null;
|
||||
lastModified: string | null;
|
||||
@@ -33,6 +34,11 @@ export interface SnapshotMeta {
|
||||
contentChangedAt: string;
|
||||
/** Events the adapter yielded from this body, for drop detection. */
|
||||
eventCount: number | null;
|
||||
/**
|
||||
* The encoding the stored bytes are in, as the server declared it. Absent in
|
||||
* metadata written before charsets were handled, which is read as UTF-8.
|
||||
*/
|
||||
charset?: string;
|
||||
}
|
||||
|
||||
export interface SnapshotState {
|
||||
@@ -53,7 +59,15 @@ export interface Snapshot {
|
||||
|
||||
export interface SaveInput {
|
||||
url: string;
|
||||
html: string;
|
||||
/**
|
||||
* The body as served. Bytes are the honest unit: a page in Shift_JIS or
|
||||
* Latin-1 that we stored as re-encoded text could never be re-decoded, and
|
||||
* `snapshots/README.md` promises "the body verbatim". A string is accepted
|
||||
* as a convenience and is stored as UTF-8.
|
||||
*/
|
||||
body: Uint8Array | string;
|
||||
/** The encoding `body` is in. Defaults to UTF-8. */
|
||||
charset?: string;
|
||||
etag: string | null;
|
||||
lastModified: string | null;
|
||||
/** ISO timestamp of this fetch. */
|
||||
@@ -66,8 +80,101 @@ export interface SaveResult {
|
||||
meta: SnapshotMeta;
|
||||
}
|
||||
|
||||
export function hashBody(html: string): string {
|
||||
return new Bun.CryptoHasher("sha256").update(html).digest("hex");
|
||||
export function hashBody(body: Uint8Array | string): string {
|
||||
return new Bun.CryptoHasher("sha256").update(body).digest("hex");
|
||||
}
|
||||
|
||||
const DEFAULT_CHARSET = "utf-8";
|
||||
|
||||
/** The charset a `Content-Type` header declares, lowercased, or null. */
|
||||
export function charsetFromContentType(contentType: string | null): string | null {
|
||||
if (contentType === null) return null;
|
||||
const match = /;\s*charset\s*=\s*"?([^;"\s]+)"?/i.exec(contentType);
|
||||
return match?.[1]?.toLowerCase() ?? null;
|
||||
}
|
||||
|
||||
/** The charset a `<meta>` tag declares in the first bytes of a document. */
|
||||
export function sniffMetaCharset(bytes: Uint8Array): string | null {
|
||||
// Every encoding we might meet here is ASCII-compatible in its first 2 KiB,
|
||||
// so reading the head as Latin-1 cannot lose the declaration.
|
||||
const head = Buffer.from(
|
||||
bytes.buffer,
|
||||
bytes.byteOffset,
|
||||
Math.min(bytes.byteLength, 2048),
|
||||
).toString("latin1");
|
||||
const match = /<meta[^>]*?charset\s*=\s*["']?\s*([a-z0-9_\-:.]+)/i.exec(head);
|
||||
return match?.[1]?.toLowerCase() ?? null;
|
||||
}
|
||||
|
||||
export interface DecodedBody {
|
||||
text: string;
|
||||
/** The charset actually used, which is what gets recorded in the metadata. */
|
||||
charset: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a fetched body into text.
|
||||
*
|
||||
* `Response.text()` assumes UTF-8, so a page served in Shift_JIS or Latin-1
|
||||
* comes back as a field of U+FFFD. That is not merely ugly: mojibake in a title
|
||||
* flows through `slugify` into the event ID, which is a localStorage key, so a
|
||||
* mis-decoded fetch silently orphans every completion mark for that source
|
||||
* (CLAUDE.md § Event IDs are localStorage keys).
|
||||
*
|
||||
* Header first, then a `<meta charset>` sniff, then UTF-8. An encoding label
|
||||
* the runtime does not know falls back to UTF-8 rather than throwing — the raw
|
||||
* bytes are kept either way, so a wrong guess stays recoverable.
|
||||
*/
|
||||
export function decodeBody(
|
||||
bytes: Uint8Array,
|
||||
contentType: string | null,
|
||||
): DecodedBody {
|
||||
const declared =
|
||||
charsetFromContentType(contentType) ?? sniffMetaCharset(bytes) ?? DEFAULT_CHARSET;
|
||||
try {
|
||||
const decoder = new TextDecoder(declared);
|
||||
return { text: decoder.decode(bytes), charset: decoder.encoding };
|
||||
} catch {
|
||||
return {
|
||||
text: new TextDecoder(DEFAULT_CHARSET).decode(bytes),
|
||||
charset: DEFAULT_CHARSET,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function toBytes(body: Uint8Array | string): Uint8Array {
|
||||
return typeof body === "string" ? new TextEncoder().encode(body) : body;
|
||||
}
|
||||
|
||||
function decodeStored(bytes: Uint8Array, charset: string | undefined): string {
|
||||
try {
|
||||
return new TextDecoder(charset ?? DEFAULT_CHARSET).decode(bytes);
|
||||
} catch {
|
||||
return new TextDecoder(DEFAULT_CHARSET).decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a file by writing a sibling temp file and renaming it into place.
|
||||
*
|
||||
* `rename` within a directory is atomic, so a reader — the feed build, git,
|
||||
* the next refresh — sees either the whole old file or the whole new one, and
|
||||
* a crash mid-write leaves a stray temp file rather than a truncated snapshot.
|
||||
*/
|
||||
async function writeAtomic(path: string, data: Uint8Array | string): Promise<void> {
|
||||
const temp = `${path}.tmp-${process.pid.toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
try {
|
||||
await writeFile(temp, data);
|
||||
await rename(temp, path);
|
||||
} catch (error) {
|
||||
await rm(temp, { force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Would writing this metadata change the file on disk? */
|
||||
function sameMeta(a: SnapshotMeta, b: SnapshotMeta): boolean {
|
||||
return JSON.stringify(a) === JSON.stringify(b);
|
||||
}
|
||||
|
||||
function emptyState(sourceId: string): SnapshotState {
|
||||
@@ -125,7 +232,14 @@ export class SnapshotStore {
|
||||
const body = Bun.file(this.bodyPath(sourceId));
|
||||
if (!(await body.exists())) return null;
|
||||
|
||||
return { meta, state: await this.readState(sourceId), html: await body.text() };
|
||||
// Decoded with the charset the bytes were stored in, so a Shift_JIS page
|
||||
// reads back as the text the wiki published rather than as mojibake.
|
||||
const bytes = new Uint8Array(await body.arrayBuffer());
|
||||
return {
|
||||
meta,
|
||||
state: await this.readState(sourceId),
|
||||
html: decodeStored(bytes, meta.charset),
|
||||
};
|
||||
}
|
||||
|
||||
/** Sources with a stored snapshot, by id. */
|
||||
@@ -181,35 +295,54 @@ export class SnapshotStore {
|
||||
/**
|
||||
* Store a fetched body.
|
||||
*
|
||||
* Identical bytes are a no-op on disk: the metadata keeps its original
|
||||
* `contentChangedAt` and validators, so an unchanged source produces no diff
|
||||
* for the workflow to commit.
|
||||
* Identical bytes are a no-op on the body and keep the original
|
||||
* `contentChangedAt`, so an unchanged source does not look like a changed
|
||||
* one. The validators are the exception: a server is free to rotate an ETag
|
||||
* while serving the very same bytes, and keeping the old one would mean
|
||||
* sending a stale `If-None-Match` forever — every cycle costing the wiki a
|
||||
* full body where a 304 was the whole point (CLAUDE.md § Scraping conduct).
|
||||
* So the metadata is refreshed, and `changed` stays false.
|
||||
*/
|
||||
async save(sourceId: string, input: SaveInput): Promise<SaveResult> {
|
||||
const contentHash = hashBody(input.html);
|
||||
const bytes = toBytes(input.body);
|
||||
const charset = input.charset ?? DEFAULT_CHARSET;
|
||||
const contentHash = hashBody(bytes);
|
||||
const previous = await this.readMeta(sourceId);
|
||||
const bodyExists = await Bun.file(this.bodyPath(sourceId)).exists();
|
||||
const changed =
|
||||
previous === null || previous.contentHash !== contentHash || !bodyExists;
|
||||
|
||||
if (!changed && previous !== null) {
|
||||
return { changed: false, meta: previous };
|
||||
}
|
||||
|
||||
const meta: SnapshotMeta = {
|
||||
sourceId,
|
||||
url: input.url,
|
||||
contentHash,
|
||||
bytes: Buffer.byteLength(input.html),
|
||||
bytes: bytes.byteLength,
|
||||
etag: input.etag,
|
||||
lastModified: input.lastModified,
|
||||
contentChangedAt: input.at,
|
||||
eventCount: input.eventCount,
|
||||
contentChangedAt: changed ? input.at : (previous?.contentChangedAt ?? input.at),
|
||||
// New bytes mean the count they yielded, even when that is unknown; the
|
||||
// same bytes keep the count we already recorded for them.
|
||||
eventCount: changed
|
||||
? input.eventCount
|
||||
: (input.eventCount ?? previous?.eventCount ?? null),
|
||||
charset,
|
||||
};
|
||||
|
||||
await mkdir(this.root, { recursive: true });
|
||||
await writeFile(this.bodyPath(sourceId), input.html);
|
||||
await writeFile(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`);
|
||||
|
||||
if (!changed && previous !== null) {
|
||||
if (sameMeta(previous, meta)) return { changed: false, meta: previous };
|
||||
await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`);
|
||||
return { changed: false, meta };
|
||||
}
|
||||
|
||||
// Body first, metadata second, each renamed into place. A crash between
|
||||
// them leaves new bytes beside older metadata, whose stale hash makes the
|
||||
// next save rewrite both — the other order would leave metadata claiming a
|
||||
// hash for bytes that were never written, and the next save would believe
|
||||
// it and skip them.
|
||||
await writeAtomic(this.bodyPath(sourceId), bytes);
|
||||
await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`);
|
||||
|
||||
return { changed: true, meta };
|
||||
}
|
||||
@@ -229,7 +362,7 @@ export class SnapshotStore {
|
||||
};
|
||||
|
||||
await mkdir(this.root, { recursive: true });
|
||||
await writeFile(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`);
|
||||
await writeAtomic(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`);
|
||||
return state;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user