fix: harden the fetch pipeline for a real server
A review of code that has never made a live request. Each of these was reproduced before it was fixed. robots.txt matched a group by testing whether our whole User-Agent contained the group's name. Our contact URL carries "StereotypicalCat", so a wiki writing `User-agent: cat` matched us — and because a named group *replaces* the wildcard group, that silently discarded every rule under `User-agent: *`. Match the RFC 9309 product token instead. A 200 carrying an HTML "not found" page also parsed to zero rules and read as permission; it now fails closed, while a genuinely empty body still means "no restrictions". The response body was read outside the try that guarded the request, so one truncated body aborted the whole cycle: later sources were never fetched, and the failed source never recorded its check, meaning a re-dispatch would ask that wiki again minutes later. Bodies were decoded as UTF-8 unconditionally and stored re-encoded. A page served in a legacy charset became replacement characters with the original bytes gone — and mojibake in a title flows into slugify and moves every localStorage key for that source. Decode by the declared charset, keep the served bytes verbatim, and hash those. Unchanged bytes skipped the metadata write, so once a server rotated its ETag we sent a stale validator forever and it served full bodies instead of 304s. Also: snapshot writes go through a temp file and a rename, and `--only` with no value is an argument error rather than "all sources". Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5
Claude Opus 5
parent
99786f6fc9
commit
c85ec0b2d1
+202
-1
@@ -99,7 +99,7 @@ function options(
|
||||
async function seed(html: string, at: string, eventCount: number | null) {
|
||||
await store.save("genshin-game8-events", {
|
||||
url: "https://game8.co/games/Genshin-Impact/archives/301601",
|
||||
html,
|
||||
body: html,
|
||||
etag: 'W/"v1"',
|
||||
lastModified: "Fri, 14 Aug 2026 09:00:00 GMT",
|
||||
at,
|
||||
@@ -370,6 +370,79 @@ describe("a source being down never blanks the feed", () => {
|
||||
expect(summary.outcomes[0]?.note).toContain("down from 10");
|
||||
});
|
||||
|
||||
test("a body that dies mid-read is one source's failure, not the cycle's", async () => {
|
||||
// The headers arrive, then the connection resets. Read outside the try,
|
||||
// that rejection escaped refreshOne and took the whole run with it: the
|
||||
// sources after this one were never asked, no summary was printed, and
|
||||
// this source's lastCheckedAt was never written — so the six-hour floor
|
||||
// did not register a request we had already spent.
|
||||
const { opts, calls } = options({
|
||||
adapters: [
|
||||
adapter(),
|
||||
adapter({
|
||||
id: "nte-game8-events",
|
||||
game: "nte",
|
||||
url: "https://game8.co/games/Neverness-to-Everness/archives/592073",
|
||||
}),
|
||||
],
|
||||
responder: (call) => {
|
||||
if (!call.url.includes("Genshin")) {
|
||||
return new Response("<html><event></event></html>");
|
||||
}
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("<html><event>"));
|
||||
controller.error(new Error("ECONNRESET"));
|
||||
},
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const summary = await runRefresh(opts);
|
||||
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(summary.outcomes[0]?.result).toBe("failed");
|
||||
expect(summary.outcomes[0]?.note).toContain("body unreadable");
|
||||
expect(summary.outcomes[1]?.result).toBe("fetched");
|
||||
expect(summary.hardFailure).toBeNull();
|
||||
|
||||
// The request was spent, so it must be on the record.
|
||||
const state = await store.readState("genshin-game8-events");
|
||||
expect(state.lastCheckedAt).toBe(NOW.toISOString());
|
||||
expect(state.consecutiveFailures).toBe(1);
|
||||
});
|
||||
|
||||
test("an unforeseen error in one source does not abort the others", async () => {
|
||||
// The robots gate itself blowing up is not something refreshOne guards;
|
||||
// the loop's backstop is what keeps the remaining sources alive.
|
||||
const { opts } = options({
|
||||
adapters: [
|
||||
adapter(),
|
||||
adapter({
|
||||
id: "nte-game8-events",
|
||||
game: "nte",
|
||||
url: "https://game8.co/games/Neverness-to-Everness/archives/592073",
|
||||
}),
|
||||
],
|
||||
robots: {
|
||||
allows: async (url) => {
|
||||
if (url.includes("Genshin")) throw new Error("robots cache exploded");
|
||||
return { allowed: true, reason: "ok" };
|
||||
},
|
||||
},
|
||||
});
|
||||
const summary = await runRefresh(opts);
|
||||
|
||||
expect(summary.outcomes).toHaveLength(2);
|
||||
expect(summary.outcomes[0]?.result).toBe("failed");
|
||||
expect(summary.outcomes[0]?.note).toContain("unexpected error");
|
||||
expect(summary.outcomes[1]?.result).toBe("fetched");
|
||||
expect(summary.hardFailure).toBeNull();
|
||||
});
|
||||
|
||||
test("a feed that will not rebuild fails the run", async () => {
|
||||
const { opts } = options({
|
||||
rebuildFeed: async () => {
|
||||
@@ -381,6 +454,121 @@ describe("a source being down never blanks the feed", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("a page that is not UTF-8", () => {
|
||||
// "<html><event>イベント</event></html>" with the title in Shift_JIS.
|
||||
const SJIS_TITLE = [0x83, 0x43, 0x83, 0x78, 0x83, 0x93, 0x83, 0x67];
|
||||
const body = new Uint8Array([
|
||||
...new TextEncoder().encode("<html><event>"),
|
||||
...SJIS_TITLE,
|
||||
...new TextEncoder().encode("</event></html>"),
|
||||
]);
|
||||
|
||||
test("is decoded with the charset the server declared", async () => {
|
||||
const { opts } = options({
|
||||
responder: () =>
|
||||
new Response(body, {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/html; charset=shift_jis" },
|
||||
}),
|
||||
});
|
||||
const summary = await runRefresh(opts);
|
||||
expect(summary.outcomes[0]?.result).toBe("fetched");
|
||||
|
||||
const snapshot = await store.read("genshin-game8-events");
|
||||
// Read as UTF-8 this is U+FFFD soup, and mojibake in a title flows through
|
||||
// slugify into the event ID, which is a localStorage key.
|
||||
expect(snapshot?.html).toBe("<html><event>イベント</event></html>");
|
||||
expect(snapshot?.html).not.toContain("�");
|
||||
});
|
||||
|
||||
test("stores the bytes as served, so a re-decode is still possible", async () => {
|
||||
const { opts } = options({
|
||||
responder: () =>
|
||||
new Response(body, {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/html; charset=shift_jis" },
|
||||
}),
|
||||
});
|
||||
await runRefresh(opts);
|
||||
|
||||
const onDisk = new Uint8Array(
|
||||
await Bun.file(store.bodyPath("genshin-game8-events")).arrayBuffer(),
|
||||
);
|
||||
expect([...onDisk]).toEqual([...body]);
|
||||
// `bytes` is the served length, which is not the length of the decoded text.
|
||||
expect((await store.readMeta("genshin-game8-events"))?.bytes).toBe(
|
||||
body.byteLength,
|
||||
);
|
||||
});
|
||||
|
||||
test("falls back to the document's own meta charset", async () => {
|
||||
const withMeta = new Uint8Array([
|
||||
...new TextEncoder().encode('<html><head><meta charset="shift_jis"></head><event>'),
|
||||
...SJIS_TITLE,
|
||||
...new TextEncoder().encode("</event></html>"),
|
||||
]);
|
||||
const { opts } = options({
|
||||
responder: () =>
|
||||
new Response(withMeta, {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/html" },
|
||||
}),
|
||||
});
|
||||
await runRefresh(opts);
|
||||
expect((await store.read("genshin-game8-events"))?.html).toContain(
|
||||
"イベント",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the workflows that drive the refresh", () => {
|
||||
// These three defects live in YAML, and each one is silent: nothing fails,
|
||||
// the site just quietly carries wrong or stale data. Asserting on the file
|
||||
// is the only offline way to keep them fixed.
|
||||
const read = (name: string) =>
|
||||
Bun.file(new URL(`../.github/workflows/${name}`, import.meta.url)).text();
|
||||
|
||||
test("ci.yml restores the refresh bookkeeping before it builds the feed", async () => {
|
||||
// lastConfirmedAt lives only in the gitignored snapshots/*.state.json. If
|
||||
// the job that builds the deployed feed never restores that cache,
|
||||
// freshnessAt falls back to contentChangedAt and the UI calls a source
|
||||
// stale two days after its bytes last moved — which for a wiki page is
|
||||
// most of the time.
|
||||
const ci = await read("ci.yml");
|
||||
const restores = ci.split("actions/cache/restore@").length - 1;
|
||||
expect(restores).toBeGreaterThanOrEqual(2); // the check job and the build job
|
||||
expect(ci).toContain("snapshots/*.state.json");
|
||||
// Restore only: refresh.yml owns writing it.
|
||||
expect(ci).not.toContain("actions/cache/save@");
|
||||
|
||||
const buildJob = ci.slice(ci.indexOf(" build:"), ci.indexOf(" image:"));
|
||||
expect(buildJob).toContain("actions/cache/restore@");
|
||||
});
|
||||
|
||||
test("refresh.yml saves its cache under a key that changes per attempt", async () => {
|
||||
// github.run_id is stable across re-runs, so a re-run's save is skipped and
|
||||
// the next run restores bookkeeping from before it — losing the record of
|
||||
// requests we did make.
|
||||
const refresh = await read("refresh.yml");
|
||||
const saveKey = /key: (refresh-state-[^\n]*)\n/g;
|
||||
const keys = [...refresh.matchAll(saveKey)].map((m) => m[1] ?? "");
|
||||
const savedKey = keys.find((k) => k.includes("run_id"));
|
||||
expect(savedKey).toBeDefined();
|
||||
expect(savedKey).toContain("github.run_attempt");
|
||||
});
|
||||
|
||||
test("refresh.yml survives losing a push race without ever forcing", async () => {
|
||||
// A human push landing mid-job made the push non-fast-forward: the fetched
|
||||
// pages were thrown away while the bookkeeping had already spent their
|
||||
// six-hour budget.
|
||||
const refresh = await read("refresh.yml");
|
||||
expect(refresh).toContain("git rebase");
|
||||
expect(refresh).toMatch(/for attempt in/);
|
||||
expect(refresh).not.toContain("--force");
|
||||
expect(refresh).not.toContain("-f origin");
|
||||
});
|
||||
});
|
||||
|
||||
describe("flags", () => {
|
||||
test("--dry-run makes no requests and writes nothing", async () => {
|
||||
const { opts, calls, rebuilds } = options({ dryRun: true });
|
||||
@@ -436,4 +624,17 @@ describe("flags", () => {
|
||||
test("parseArgs rejects an unknown flag rather than ignoring it", () => {
|
||||
expect(() => parseArgs(["--force"])).toThrow("unknown flag");
|
||||
});
|
||||
|
||||
test("parseArgs rejects a flag whose value is missing", () => {
|
||||
// `--only` with nothing after it used to mean "every source": the operator
|
||||
// asked for one request and would have got seven.
|
||||
expect(() => parseArgs(["--only"])).toThrow("--only requires a value");
|
||||
expect(() => parseArgs(["--only", "--dry-run"])).toThrow(
|
||||
"--only requires a value",
|
||||
);
|
||||
expect(() => parseArgs(["--snapshots"])).toThrow("--snapshots requires a value");
|
||||
expect(() => parseArgs(["--user-agent"])).toThrow(
|
||||
"--user-agent requires a value",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -77,6 +77,48 @@ Disallow: /admin
|
||||
expect(isAllowed(robots, UA, "/games/x")).toBe(true);
|
||||
});
|
||||
|
||||
test("an unrelated named group cannot steal the wildcard group", () => {
|
||||
// Our contact URL contains "StereotypicalCat". Matching a group name
|
||||
// anywhere in the header made `User-agent: cat` look like our group, and
|
||||
// because a named group replaces `*`, that coincidence *discarded* the
|
||||
// rules the site actually wrote for everyone.
|
||||
const ua =
|
||||
"gacha-event-tracker/1.0 (+https://github.com/StereotypicalCat/gacha-event-tracker)";
|
||||
const robots = parseRobots(`
|
||||
User-agent: *
|
||||
Disallow: /games/
|
||||
|
||||
User-agent: cat
|
||||
Disallow: /litter
|
||||
`);
|
||||
expect(groupFor(robots, ua)?.agents).toEqual(["*"]);
|
||||
expect(isAllowed(robots, ua, "/games/x")).toBe(false);
|
||||
expect(isAllowed(robots, ua, "/litter")).toBe(true);
|
||||
});
|
||||
|
||||
test("a substring of the product token does not name us either", () => {
|
||||
const robots = parseRobots(`
|
||||
User-agent: *
|
||||
Disallow: /games/
|
||||
|
||||
User-agent: gacha
|
||||
Allow: /
|
||||
`);
|
||||
expect(isAllowed(robots, UA, "/games/x")).toBe(false);
|
||||
});
|
||||
|
||||
test("a group naming us with a version still binds", () => {
|
||||
const robots = parseRobots(`
|
||||
User-agent: *
|
||||
Disallow: /
|
||||
|
||||
User-agent: gacha-event-tracker/1.0
|
||||
Disallow: /admin
|
||||
`);
|
||||
expect(isAllowed(robots, UA, "/games/x")).toBe(true);
|
||||
expect(isAllowed(robots, UA, "/admin/panel")).toBe(false);
|
||||
});
|
||||
|
||||
test("the longest matching agent name wins", () => {
|
||||
const robots = parseRobots(`
|
||||
User-agent: googlebot
|
||||
@@ -227,6 +269,47 @@ describe("RobotsCache", () => {
|
||||
expect(decision.reason).toContain("503");
|
||||
});
|
||||
|
||||
test("an empty 200 body is a valid robots.txt that restricts nothing", async () => {
|
||||
const { cache } = cacheWith(() => new Response(" \n", { status: 200 }));
|
||||
const decision = await cache.allows("https://x.test/wiki/Event");
|
||||
expect(decision.allowed).toBe(true);
|
||||
expect(decision.reason).toBe("robots.txt ok");
|
||||
});
|
||||
|
||||
test("fails closed on a soft 404 — an HTML page served as 200", async () => {
|
||||
// The commonest robots.txt misconfiguration there is. It parses to zero
|
||||
// groups, which is indistinguishable from "nothing is restricted", so
|
||||
// reading it as permission is exactly the fail-open this class forbids.
|
||||
const { cache } = cacheWith(
|
||||
() =>
|
||||
new Response(
|
||||
"<!DOCTYPE html>\n<html><head><title>404 Not Found</title></head><body>Not found</body></html>",
|
||||
{ status: 200, headers: { "Content-Type": "text/html" } },
|
||||
),
|
||||
);
|
||||
const decision = await cache.allows("https://x.test/wiki/Event");
|
||||
expect(decision.allowed).toBe(false);
|
||||
expect(decision.reason).toContain("HTML");
|
||||
});
|
||||
|
||||
test("fails closed when the body dies mid-read", async () => {
|
||||
const { cache } = cacheWith(
|
||||
() =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("User-agent: *\n"));
|
||||
controller.error(new Error("ECONNRESET"));
|
||||
},
|
||||
}),
|
||||
{ status: 200 },
|
||||
),
|
||||
);
|
||||
const decision = await cache.allows("https://x.test/wiki/Event");
|
||||
expect(decision.allowed).toBe(false);
|
||||
expect(decision.reason).toContain("unreadable");
|
||||
});
|
||||
|
||||
test("fails closed when robots.txt is unreachable", async () => {
|
||||
const { cache } = cacheWith(() => {
|
||||
throw new Error("ECONNREFUSED");
|
||||
|
||||
+227
-6
@@ -1,10 +1,14 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { statSync } from "node:fs";
|
||||
import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
charsetFromContentType,
|
||||
decodeBody,
|
||||
freshnessAt,
|
||||
hashBody,
|
||||
sniffMetaCharset,
|
||||
SnapshotStore,
|
||||
type SnapshotMeta,
|
||||
} from "../src/ingest/snapshots.ts";
|
||||
@@ -25,10 +29,20 @@ afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function save(html: string, at: string, extra: Partial<{ etag: string | null; lastModified: string | null; eventCount: number | null }> = {}) {
|
||||
function save(
|
||||
body: Uint8Array | string,
|
||||
at: string,
|
||||
extra: Partial<{
|
||||
etag: string | null;
|
||||
lastModified: string | null;
|
||||
eventCount: number | null;
|
||||
charset: string;
|
||||
}> = {},
|
||||
) {
|
||||
return store.save("genshin-game8-events", {
|
||||
url: "https://game8.co/games/Genshin-Impact/archives/301601",
|
||||
html,
|
||||
body,
|
||||
...(extra.charset === undefined ? {} : { charset: extra.charset }),
|
||||
etag: extra.etag ?? null,
|
||||
lastModified: extra.lastModified ?? null,
|
||||
at,
|
||||
@@ -60,10 +74,10 @@ describe("SnapshotStore", () => {
|
||||
expect(await store.list()).toEqual(["genshin-game8-events"]);
|
||||
});
|
||||
|
||||
test("identical bytes are not a change and do not rewrite metadata", async () => {
|
||||
test("identical bytes are not a change", async () => {
|
||||
await save("<html>one</html>", T0, { etag: '"v1"', eventCount: 9 });
|
||||
const again = await save("<html>one</html>", T1, {
|
||||
etag: '"v2"',
|
||||
etag: '"v1"',
|
||||
eventCount: 9,
|
||||
});
|
||||
|
||||
@@ -71,7 +85,39 @@ describe("SnapshotStore", () => {
|
||||
// contentChangedAt still points at the fetch that produced these bytes,
|
||||
// which is what keeps an unchanged cycle out of the commit log.
|
||||
expect(again.meta.contentChangedAt).toBe(T0);
|
||||
expect(again.meta.etag).toBe('"v1"');
|
||||
});
|
||||
|
||||
test("identical bytes and identical validators rewrite nothing at all", async () => {
|
||||
await save("<html>one</html>", T0, { etag: '"v1"', eventCount: 9 });
|
||||
const before = await Bun.file(store.metaPath("genshin-game8-events")).text();
|
||||
await save("<html>one</html>", T1, { etag: '"v1"', eventCount: 9 });
|
||||
expect(await Bun.file(store.metaPath("genshin-game8-events")).text()).toBe(
|
||||
before,
|
||||
);
|
||||
});
|
||||
|
||||
test("a rotated ETag is stored even though the bytes did not change", async () => {
|
||||
// Servers rotate validators on identical bytes. Keeping the old one meant
|
||||
// sending a stale If-None-Match forever, so the wiki would serve the whole
|
||||
// page every cycle — the exact cost conditional requests exist to avoid.
|
||||
await save("<html>one</html>", T0, { etag: '"v1"', eventCount: 9 });
|
||||
const again = await save("<html>one</html>", T1, {
|
||||
etag: '"v2"',
|
||||
lastModified: "Sat, 15 Aug 2026 12:00:00 GMT",
|
||||
eventCount: 9,
|
||||
});
|
||||
|
||||
expect(again.changed).toBe(false);
|
||||
expect(again.meta.contentChangedAt).toBe(T0);
|
||||
expect(again.meta.etag).toBe('"v2"');
|
||||
|
||||
const persisted = await store.readMeta("genshin-game8-events");
|
||||
expect(persisted?.etag).toBe('"v2"');
|
||||
expect(persisted?.contentChangedAt).toBe(T0);
|
||||
expect(store.conditionalHeaders(persisted)).toEqual({
|
||||
"If-None-Match": '"v2"',
|
||||
"If-Modified-Since": "Sat, 15 Aug 2026 12:00:00 GMT",
|
||||
});
|
||||
});
|
||||
|
||||
test("different bytes are a change", async () => {
|
||||
@@ -99,6 +145,181 @@ describe("SnapshotStore", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("charset", () => {
|
||||
// "イベント" as Shift_JIS, and "Café" as Latin-1: both are mojibake if the
|
||||
// bytes are read as UTF-8.
|
||||
const SJIS = new Uint8Array([0x83, 0x43, 0x83, 0x78, 0x83, 0x93, 0x83, 0x67]);
|
||||
const LATIN1 = new Uint8Array([0x43, 0x61, 0x66, 0xe9]);
|
||||
|
||||
test("reads the charset out of a Content-Type header", () => {
|
||||
expect(charsetFromContentType('text/html; charset="Shift_JIS"')).toBe(
|
||||
"shift_jis",
|
||||
);
|
||||
expect(charsetFromContentType("text/html;charset=iso-8859-1")).toBe(
|
||||
"iso-8859-1",
|
||||
);
|
||||
expect(charsetFromContentType("text/html")).toBeNull();
|
||||
expect(charsetFromContentType(null)).toBeNull();
|
||||
});
|
||||
|
||||
test("falls back to the meta charset in the document head", () => {
|
||||
const html = new TextEncoder().encode(
|
||||
'<!DOCTYPE html><html><head><meta charset="shift_jis"><title>x</title></head>',
|
||||
);
|
||||
expect(sniffMetaCharset(html)).toBe("shift_jis");
|
||||
expect(
|
||||
sniffMetaCharset(
|
||||
new TextEncoder().encode(
|
||||
'<meta http-equiv="Content-Type" content="text/html; charset=EUC-JP">',
|
||||
),
|
||||
),
|
||||
).toBe("euc-jp");
|
||||
expect(sniffMetaCharset(new TextEncoder().encode("<html><body>"))).toBeNull();
|
||||
});
|
||||
|
||||
test("decodes with the declared charset, not with UTF-8", () => {
|
||||
expect(decodeBody(SJIS, "text/html; charset=shift_jis").text).toBe(
|
||||
"イベント",
|
||||
);
|
||||
expect(decodeBody(LATIN1, "text/html; charset=iso-8859-1").text).toBe("Café");
|
||||
expect(decodeBody(new TextEncoder().encode("ok"), null).text).toBe("ok");
|
||||
});
|
||||
|
||||
test("an encoding label we do not know falls back to UTF-8 rather than throwing", () => {
|
||||
const decoded = decodeBody(
|
||||
new TextEncoder().encode("ok"),
|
||||
"text/html; charset=x-nonesuch",
|
||||
);
|
||||
expect(decoded.text).toBe("ok");
|
||||
expect(decoded.charset).toBe("utf-8");
|
||||
});
|
||||
|
||||
test("stores the served bytes verbatim and reads them back as text", async () => {
|
||||
const { meta } = await save(SJIS, T0, { charset: "shift_jis", eventCount: 2 });
|
||||
|
||||
// The bytes on disk are the bytes the server sent; the hash and the size
|
||||
// describe those bytes, so a later re-decode is still possible.
|
||||
const onDisk = new Uint8Array(
|
||||
await Bun.file(store.bodyPath("genshin-game8-events")).arrayBuffer(),
|
||||
);
|
||||
expect([...onDisk]).toEqual([...SJIS]);
|
||||
expect(meta.bytes).toBe(SJIS.byteLength);
|
||||
expect(meta.contentHash).toBe(hashBody(SJIS));
|
||||
|
||||
const snapshot = await store.read("genshin-game8-events");
|
||||
expect(snapshot?.html).toBe("イベント");
|
||||
expect(snapshot?.html).not.toContain("�");
|
||||
});
|
||||
|
||||
test("metadata written before charsets were recorded reads as UTF-8", async () => {
|
||||
await save("<html>é</html>", T0);
|
||||
const meta = await store.readMeta("genshin-game8-events");
|
||||
const { charset: _dropped, ...legacy } = meta as SnapshotMeta;
|
||||
await writeFile(
|
||||
store.metaPath("genshin-game8-events"),
|
||||
`${JSON.stringify(legacy, null, 2)}\n`,
|
||||
);
|
||||
expect((await store.read("genshin-game8-events"))?.html).toBe("<html>é</html>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("writes land whole or not at all", () => {
|
||||
test("leaves no temp files behind", async () => {
|
||||
await save("<html>one</html>", T0);
|
||||
await store.recordCheck("genshin-game8-events", {
|
||||
at: T0,
|
||||
status: 200,
|
||||
ok: true,
|
||||
});
|
||||
const names = await readdir(root);
|
||||
expect(names.sort()).toEqual([
|
||||
"genshin-game8-events.html",
|
||||
"genshin-game8-events.meta.json",
|
||||
"genshin-game8-events.state.json",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a temp file left by a crashed run is not mistaken for a snapshot", async () => {
|
||||
await writeFile(
|
||||
join(root, "genshin-game8-events.meta.json.tmp-abc"),
|
||||
"{ half",
|
||||
);
|
||||
expect(await store.list()).toEqual([]);
|
||||
await save("<html>one</html>", T0);
|
||||
expect(await store.list()).toEqual(["genshin-game8-events"]);
|
||||
});
|
||||
|
||||
test("replaces each file rather than overwriting it in place", async () => {
|
||||
// Written in place, a snapshot exists in a truncated state for as long as
|
||||
// the write takes, and a crash there leaves it that way. Renaming a
|
||||
// complete temp file over it cannot: the replacement is one atomic step,
|
||||
// which shows up as a new inode.
|
||||
await save("<html>one</html>", T0, { etag: '"v1"' });
|
||||
const before = {
|
||||
body: statSync(store.bodyPath("genshin-game8-events")).ino,
|
||||
meta: statSync(store.metaPath("genshin-game8-events")).ino,
|
||||
};
|
||||
|
||||
await save("<html>two</html>", T1, { etag: '"v2"' });
|
||||
expect(statSync(store.bodyPath("genshin-game8-events")).ino).not.toBe(
|
||||
before.body,
|
||||
);
|
||||
expect(statSync(store.metaPath("genshin-game8-events")).ino).not.toBe(
|
||||
before.meta,
|
||||
);
|
||||
|
||||
// Including the metadata-only rewrite that a rotated validator triggers.
|
||||
const metaIno = statSync(store.metaPath("genshin-game8-events")).ino;
|
||||
await save("<html>two</html>", T1, { etag: '"v3"' });
|
||||
expect(statSync(store.metaPath("genshin-game8-events")).ino).not.toBe(
|
||||
metaIno,
|
||||
);
|
||||
|
||||
// And the state file, which is written on every single cycle.
|
||||
await store.recordCheck("genshin-game8-events", {
|
||||
at: T0,
|
||||
status: 200,
|
||||
ok: true,
|
||||
});
|
||||
const stateIno = statSync(store.statePath("genshin-game8-events")).ino;
|
||||
await store.recordCheck("genshin-game8-events", {
|
||||
at: T1,
|
||||
status: 200,
|
||||
ok: true,
|
||||
});
|
||||
expect(statSync(store.statePath("genshin-game8-events")).ino).not.toBe(
|
||||
stateIno,
|
||||
);
|
||||
});
|
||||
|
||||
test("a reader never sees a half-written body", async () => {
|
||||
const first = `<html>${"a".repeat(1_000_000)}</html>`;
|
||||
const second = `<html>${"b".repeat(1_000_000)}</html>`;
|
||||
await save(first, T0);
|
||||
|
||||
let stop = false;
|
||||
const seen = new Set<string>();
|
||||
const reader = (async () => {
|
||||
while (!stop) {
|
||||
const text = await Bun.file(store.bodyPath("genshin-game8-events")).text();
|
||||
seen.add(`${text.length}:${text.slice(-8)}`);
|
||||
await Bun.sleep(0);
|
||||
}
|
||||
})();
|
||||
|
||||
await save(second, T1);
|
||||
stop = true;
|
||||
await reader;
|
||||
|
||||
for (const observed of seen) {
|
||||
expect([
|
||||
`${first.length}:${first.slice(-8)}`,
|
||||
`${second.length}:${second.slice(-8)}`,
|
||||
]).toContain(observed);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("conditional requests", () => {
|
||||
test("emits both validators when both are known", async () => {
|
||||
const { meta } = await save("<html>one</html>", T0, {
|
||||
|
||||
Reference in New Issue
Block a user