#!/usr/bin/env node // Differential fuzzer: plays the same random SpellGrid level and script on the Rust and the // TypeScript reference CLIs of one module and compares what they print. // // node tools/diff-tracks.mjs [--seeds N] [--seed S] [--module NN] [--dir D] [--keep] // npm run diff:tracks -- --seeds 500 // // Stages both references of module NN (default: the highest module whose references have a // SpellGrid CLI) with their cards, as play-spellgrid does, into /rust and /ts (default D: // a new temp directory), builds the Rust CLI once and installs the TypeScript dependencies. // Then, for each seed (1..N, or only S), it generates a level (/cases/seed-S.json) and a // script of commands (/cases/seed-S.txt), pipes the script to both CLIs, and compares stdout. // Nothing is printed for a seed on which they agree. The first divergence prints the seed, the // level path, the script and a unified diff, and exits 1. A crash (panic, uncaught exception, // 10 s timeout) on one side is a divergence; on both sides it is reported as "both crashed" and // exits 1. A temp directory is removed at the end unless --keep is given or a case failed; a // directory given with --dir is always kept, and re-running there reuses the Rust build. import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs"; import { spawn, spawnSync } from "node:child_process"; import { availableParallelism, tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { fail, isMain, moduleDirs } from "./lib.mjs"; import { stagePlay } from "./play-spellgrid.mjs"; const TIMEOUT_MS = 10_000; const OPERATORS = ["IS", "AND", "NOT"]; const DIRS = ["w", "a", "s", "d"]; const MARKER = ".diff-tracks"; export function parseArgs(argv) { const opts = { seeds: 100, seed: null, module: null, dir: null, keep: false }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--seeds") opts.seeds = Number(argv[++i]); else if (a === "--seed") opts.seed = Number(argv[++i]); else if (a === "--module") opts.module = String(argv[++i]).padStart(2, "0"); else if (a === "--dir") opts.dir = argv[++i]; else if (a === "--keep") opts.keep = true; else fail(`diff-tracks: unknown argument ${a}\nusage: diff-tracks.mjs [--seeds N] [--seed S] [--module NN] [--dir D] [--keep]`); } if (!Number.isInteger(opts.seeds) || opts.seeds < 1) fail("diff-tracks: --seeds needs a positive whole number"); if (opts.seed !== null && !Number.isInteger(opts.seed)) fail("diff-tracks: --seed needs a whole number"); return opts; } /** A seeded PRNG (mulberry32): `rng(seed)()` is a number in [0, 1). The same seed gives the same sequence. */ export function rng(seed) { let a = seed >>> 0; return () => { a = (a + 0x6d2b79f5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; } /** * The words and cards a staged reference reads: `words` from its words.json, `cards` (by name, * a later directory replacing an earlier card of the same name) as { name, type, noun, chooses }, * where `chooses` marks a card whose effect asks the chooser (Assemble). */ export function loadVocab(wordsFile, cardsDirs) { const words = JSON.parse(readFileSync(wordsFile, "utf8")); const byName = new Map(); for (const dir of cardsDirs) { for (const f of readdirSync(dir).filter((f) => f.endsWith(".json")).sort()) { const def = JSON.parse(readFileSync(join(dir, f), "utf8")); const chooses = (def.text ?? []).some((t) => t.kind === "effect" && t.effect?.type === "assemble"); byName.set(def.name, { name: def.name, type: def.type, noun: def.noun ?? null, chooses }); } } const cards = [...byName.values()].sort((a, b) => a.name.localeCompare(b.name)); return { words, cards }; } /** One random case: a level in the course's level format and a script of CLI commands. Pure. */ export function generateCase(seed, vocab) { const r = rng(seed); const int = (lo, hi) => lo + Math.floor(r() * (hi - lo + 1)); const pick = (list) => list[Math.floor(r() * list.length)]; const chance = (p) => r() < p; const nouns = vocab.words.filter((w) => w.role === "noun").map((w) => w.name); const objectNouns = vocab.words.filter((w) => w.role === "noun" && !w.selects).map((w) => w.name); const properties = vocab.words.filter((w) => w.role === "property").map((w) => w.name); const conditions = vocab.words.filter((w) => w.role === "condition").map((w) => w.name); const placeable = vocab.cards.filter((c) => c.type === "minion" && c.noun); const holdable = vocab.cards.filter((c) => c.type === "spell" || (c.type === "minion" && c.noun)); const width = int(4, 8); const height = Math.min(8, Math.max(4, width + int(-2, 2))); const free = []; for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) free.push([x, y]); const takeFree = () => (free.length ? free.splice(Math.floor(r() * free.length), 1)[0] : null); const isFree = (x, y) => free.some(([fx, fy]) => fx === x && fy === y); const claim = (x, y) => free.splice(free.findIndex(([fx, fy]) => fx === x && fy === y), 1); const objects = []; const cards = []; // The cards on the grid first: their nouns are good sentence subjects. for (let i = int(0, 3); i > 0 && placeable.length; i--) { const at = takeFree(); if (at) cards.push({ card: pick(placeable).name, x: at[0], y: at[1] }); } const objectKinds = [pick(objectNouns), pick(objectNouns)]; const cardNouns = cards.map((c) => vocab.cards.find((v) => v.name === c.card).noun); const subjects = [...objectKinds, ...cardNouns]; // Whole sentences, in a row or a column: NOUN [AND NOUN] [CONDITION NOUN [AND NOUN]] IS [NOT] // (PROPERTY | NOUN) [AND ...]. A condition only with a vocabulary that has condition words // (Module 11 on), so the cases of earlier modules' vocabularies don't change. const sentence = (first) => { // The first is the player's: the first object noun IS YOU (that noun always has an object). if (first) return [objectKinds[0], "IS", "YOU"]; const line = [chance(0.7) ? pick(subjects) : pick(nouns)]; if (chance(0.15)) line.push("AND", pick(subjects)); if (conditions.length && chance(0.4)) { line.push(pick(conditions), pick(subjects)); if (chance(0.2)) line.push("AND", pick(nouns)); } line.push("IS"); if (chance(0.15)) line.push("NOT"); line.push(chance(0.75) ? pick(properties) : pick(nouns)); if (chance(0.2)) line.push("AND", pick(properties)); return line; }; const sentences = []; if (chance(0.9)) sentences.push(sentence(true)); for (let i = int(1, 2); i > 0; i--) sentences.push(sentence(false)); for (const line of sentences) { for (let tries = 0; tries < 50; tries++) { const across = chance(0.5); const [dx, dy] = across ? [1, 0] : [0, 1]; const [maxX, maxY] = [width - 1 - dx * (line.length - 1), height - 1 - dy * (line.length - 1)]; if (maxX < 0 || maxY < 0) continue; const [x0, y0] = [int(0, maxX), int(0, maxY)]; const cells = line.map((_, i) => [x0 + dx * i, y0 + dy * i]); if (!cells.every(([x, y]) => isFree(x, y))) continue; cells.forEach(([x, y], i) => { claim(x, y); objects.push({ kind: `text:${line[i]}`, x, y }); }); break; } } // Loose words, the objects of the nouns, and now and then one object on an occupied tile. const allWords = [...vocab.words.map((w) => w.name), ...OPERATORS]; for (let i = int(0, 3); i > 0; i--) { const at = takeFree(); if (at) objects.push({ kind: `text:${pick(allWords)}`, x: at[0], y: at[1] }); } for (let i = int(1, 6); i > 0; i--) { const at = takeFree(); if (at) objects.push({ kind: (i === 1 ? objectKinds[0] : pick(objectKinds)).toLowerCase(), x: at[0], y: at[1] }); } if (objects.length && chance(0.2)) { const { x, y } = pick(objects); objects.push({ kind: pick(objectKinds).toLowerCase(), x, y }); } const hand = []; for (let i = int(0, 3); i > 0 && holdable.length; i--) hand.push(pick(holdable).name); // The script: moves, casts and plays (mostly of cards in the hand), and the chooser's answers. const answer = () => { const k = r(); return k < 0.5 ? String(int(1, 3)) : k < 0.8 ? pick(vocab.cards).name : pick(["top", "bottom"]); }; const coords = () => { if (cards.length && chance(0.5)) { const c = pick(cards); return [c.x, c.y]; } return [int(0, width), int(0, height)]; }; const minions = vocab.cards.filter((c) => c.type === "minion"); const script = []; const length = int(5, 25); let base = null; // the base the last Assemble asked for by name, if any while (script.length < length) { const k = r(); if (k < 0.6) script.push(pick(DIRS)); else if (k < 0.93) { // After an Assemble, now and then play what it may have made: " (Assembled)". if (base && chance(0.4)) { script.push(`c ${chance(0.8) ? base : pick(minions).name} (Assembled) ${int(0, width - 1)} ${int(0, height - 1)}`); continue; } const name = hand.length && chance(0.75) ? pick(hand) : pick(vocab.cards).name; const card = vocab.cards.find((c) => c.name === name); const at = chance(card.type === "spell" ? 0.5 : 0.85) ? coords() : null; script.push(at ? `c ${card.name} ${at[0]} ${at[1]}` : `c ${card.name}`); if (card.chooses) { const answers = Array.from({ length: int(0, 3) }, answer); if (answers.length && minions.length && chance(0.5)) answers[0] = pick(minions).name; if (minions.some((m) => m.name === answers[0])) base = answers[0]; script.push(...answers); } } else script.push(answer()); } return { level: { width, height, objects, cards, hand }, script: script.slice(0, 25) }; } /** A unified diff of two texts, line by line, with 3 lines of context. */ export function unifiedDiff(a, b, nameA = "rust", nameB = "ts") { const x = a.split("\n"); const y = b.split("\n"); const lcs = Array.from({ length: x.length + 1 }, () => new Uint32Array(y.length + 1)); for (let i = x.length - 1; i >= 0; i--) { for (let j = y.length - 1; j >= 0; j--) { lcs[i][j] = x[i] === y[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]); } } // Each op: [sign, line, lines of a before it, lines of b before it]. const ops = []; for (let i = 0, j = 0; i < x.length || j < y.length;) { if (i < x.length && j < y.length && x[i] === y[j]) ops.push([" ", x[i], i++, j++]); else if (i < x.length && (j === y.length || lcs[i + 1][j] >= lcs[i][j + 1])) ops.push(["-", x[i], i++, j]); else ops.push(["+", y[j], i, j++]); } const out = [`--- ${nameA}`, `+++ ${nameB}`]; const changed = ops.flatMap((op, k) => (op[0] === " " ? [] : [k])); for (let k = 0; k < changed.length;) { const first = Math.max(0, changed[k] - 3); let last = changed[k]; while (k < changed.length && changed[k] <= last + 6) last = changed[k++]; const hunk = ops.slice(first, Math.min(ops.length, last + 4)); const aLen = hunk.filter((op) => op[0] !== "+").length; const bLen = hunk.filter((op) => op[0] !== "-").length; out.push(`@@ -${hunk[0][2] + (aLen ? 1 : 0)},${aLen} +${hunk[0][3] + (bLen ? 1 : 0)},${bLen} @@`); for (const [sign, line] of hunk) out.push(`${sign}${line}`); } return out.join("\n"); } /** Run `cmd args` in `cwd` with `input` on stdin; resolves to { out, err, code, signal }. */ function runCli(cmd, args, cwd, input) { return new Promise((done) => { const child = spawn(cmd, args, { cwd, timeout: TIMEOUT_MS, killSignal: "SIGKILL" }); let out = ""; let err = ""; child.stdout.on("data", (d) => (out += d)); child.stderr.on("data", (d) => (err += d)); child.stdin.on("error", () => {}); child.on("close", (code, signal) => done({ out, err, code, signal })); child.stdin.end(input); }); } /** Did a CLI run crash? Both CLIs exit 0 (won) or 1 (lost, quit) and print nothing on stderr otherwise. */ export const crashed = (r) => r.signal != null || (r.code !== 0 && r.code !== 1) || r.err.trim() !== ""; /** Compare a Rust run with a TypeScript run: "agree", "outputs differ", or which side crashed. */ export function verdict(rust, ts) { const [rc, tc] = [crashed(rust), crashed(ts)]; if (rc && tc) return "both crashed"; if (rc) return "the Rust CLI crashed"; if (tc) return "the TypeScript CLI crashed"; return rust.out === ts.out ? "agree" : "outputs differ"; } const why = (r) => r.signal === "SIGKILL" ? `timed out after ${TIMEOUT_MS / 1000} s` : r.signal ? `killed by ${r.signal}` : r.err.trim().split("\n").slice(0, 8).join("\n") || `exit code ${r.code}`; /** The highest module whose Rust and TypeScript references both have the SpellGrid CLI. */ function defaultModule() { const withCli = moduleDirs().filter((m) => existsSync(join(m.path, "reference", "ts", "src", "cli.ts")) && existsSync(join(m.path, "reference", "rust", "src", "cli.rs"))); if (!withCli.length) fail("diff-tracks: no module has a SpellGrid CLI in both references"); return withCli.at(-1).num; } async function main() { const opts = parseArgs(process.argv.slice(2)); const num = opts.module ?? defaultModule(); const base = opts.dir ? resolve(opts.dir) : mkdtempSync(join(tmpdir(), "diff-tracks-")); if (existsSync(base) && readdirSync(base).length && !existsSync(join(base, MARKER))) { fail(`diff-tracks: ${base} is not empty and was not staged by diff-tracks; give an empty or new directory`); } mkdirSync(base, { recursive: true }); writeFileSync(join(base, MARKER), `${num}\n`); const rust = stagePlay("rust", join(base, "rust"), num); const ts = stagePlay("ts", join(base, "ts"), num); const cases = join(base, "cases"); mkdirSync(cases, { recursive: true }); const build = spawnSync("cargo", ["build", "--quiet", "--bin", "spellgrid"], { cwd: rust, stdio: ["ignore", "inherit", "inherit"] }); if (build.status !== 0) fail(`diff-tracks: cargo build failed in ${rust}`); if (!existsSync(join(ts, "node_modules"))) { const r = spawnSync("npm", ["install", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: ts, stdio: ["ignore", "ignore", "inherit"] }); if (r.status !== 0) fail(`diff-tracks: npm install failed in ${ts}`); } const vocab = loadVocab(join(ts, "src", "content", "words.json"), [join(ts, "cards")]); const seeds = opts.seed !== null ? [opts.seed] : Array.from({ length: opts.seeds }, (_, i) => i + 1); const width = Math.max(1, Math.min(8, availableParallelism())); const runCase = async (seed) => { const { level, script } = generateCase(seed, vocab); const levelPath = join(cases, `seed-${seed}.json`); writeFileSync(levelPath, `${JSON.stringify(level, null, 2)}\n`); writeFileSync(join(cases, `seed-${seed}.txt`), `${script.join("\n")}\n`); const input = `${script.join("\n")}\n`; const [a, b] = await Promise.all([ runCli(join(rust, "target", "debug", "spellgrid"), [levelPath], rust, input), runCli(process.execPath, ["--import", "tsx", "src/cli.ts", levelPath], ts, input), ]); return { seed, levelPath, script, rust: a, ts: b }; }; let bad = null; for (let k = 0; k < seeds.length && !bad; k += width) { const results = await Promise.all(seeds.slice(k, k + width).map(runCase)); bad = results.find((c) => verdict(c.rust, c.ts) !== "agree") ?? null; } if (!bad) { console.log(`diff-tracks: module ${num}, ${seeds.length} seed(s), Rust and TypeScript agree`); if (!opts.dir && !opts.keep) rmSync(base, { recursive: true, force: true }); return; } console.log(`diff-tracks: module ${num}, seed ${bad.seed}: ${verdict(bad.rust, bad.ts)}`); console.log(`level: ${bad.levelPath}`); console.log(`script:\n${bad.script.map((l) => ` ${l}`).join("\n")}`); if (crashed(bad.rust)) console.log(`Rust:\n${why(bad.rust)}`); if (crashed(bad.ts)) console.log(`TypeScript:\n${why(bad.ts)}`); if (bad.rust.out !== bad.ts.out) console.log(unifiedDiff(bad.rust.out, bad.ts.out)); console.log(`(staged in ${base}; kept)`); process.exit(1); } if (isMain(import.meta.url)) await main();