systemic-game-design-course/tools/diff-tracks.mjs

#!/usr/bin/env node
// Differential fuzzer: plays the same random SpellGrid level and script on the Rust and the
// TypeScript reference CLIs of one module and compares what they print.
//
//   node tools/diff-tracks.mjs [--seeds N] [--seed S] [--module NN] [--dir D] [--keep]
//   npm run diff:tracks -- --seeds 500
//
// Stages both references of module NN (default: the highest module whose references have a
// SpellGrid CLI) with their cards, as play-spellgrid does, into <D>/rust and <D>/ts (default D:
// a new temp directory), builds the Rust CLI once and installs the TypeScript dependencies.
// Then, for each seed (1..N, or only S), it generates a level (<D>/cases/seed-S.json) and a
// script of commands (<D>/cases/seed-S.txt), pipes the script to both CLIs, and compares stdout.
// Nothing is printed for a seed on which they agree. The first divergence prints the seed, the
// level path, the script and a unified diff, and exits 1. A crash (panic, uncaught exception,
// 10 s timeout) on one side is a divergence; on both sides it is reported as "both crashed" and
// exits 1. A temp directory is removed at the end unless --keep is given or a case failed; a
// directory given with --dir is always kept, and re-running there reuses the Rust build.
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs";
import { spawn, spawnSync } from "node:child_process";
import { availableParallelism, tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { fail, isMain, moduleDirs } from "./lib.mjs";
import { stagePlay } from "./play-spellgrid.mjs";

const TIMEOUT_MS = 10_000;
const OPERATORS = ["IS", "AND", "NOT"];
const DIRS = ["w", "a", "s", "d"];
const MARKER = ".diff-tracks";

export function parseArgs(argv) {
  const opts = { seeds: 100, seed: null, module: null, dir: null, keep: false };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--seeds") opts.seeds = Number(argv[++i]);
    else if (a === "--seed") opts.seed = Number(argv[++i]);
    else if (a === "--module") opts.module = String(argv[++i]).padStart(2, "0");
    else if (a === "--dir") opts.dir = argv[++i];
    else if (a === "--keep") opts.keep = true;
    else fail(`diff-tracks: unknown argument ${a}\nusage: diff-tracks.mjs [--seeds N] [--seed S] [--module NN] [--dir D] [--keep]`);
  }
  if (!Number.isInteger(opts.seeds) || opts.seeds < 1) fail("diff-tracks: --seeds needs a positive whole number");
  if (opts.seed !== null && !Number.isInteger(opts.seed)) fail("diff-tracks: --seed needs a whole number");
  return opts;
}

/** A seeded PRNG (mulberry32): `rng(seed)()` is a number in [0, 1). The same seed gives the same sequence. */
export function rng(seed) {
  let a = seed >>> 0;
  return () => {
    a = (a + 0x6d2b79f5) >>> 0;
    let t = a;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

/**
 * The words and cards a staged reference reads: `words` from its words.json, `cards` (by name,
 * a later directory replacing an earlier card of the same name) as { name, type, noun, chooses },
 * where `chooses` marks a card whose effect asks the chooser (Assemble).
 */
export function loadVocab(wordsFile, cardsDirs) {
  const words = JSON.parse(readFileSync(wordsFile, "utf8"));
  const byName = new Map();
  for (const dir of cardsDirs) {
    for (const f of readdirSync(dir).filter((f) => f.endsWith(".json")).sort()) {
      const def = JSON.parse(readFileSync(join(dir, f), "utf8"));
      const chooses = (def.text ?? []).some((t) => t.kind === "effect" && t.effect?.type === "assemble");
      byName.set(def.name, { name: def.name, type: def.type, noun: def.noun ?? null, chooses });
    }
  }
  const cards = [...byName.values()].sort((a, b) => a.name.localeCompare(b.name));
  return { words, cards };
}

/** One random case: a level in the course's level format and a script of CLI commands. Pure. */
export function generateCase(seed, vocab) {
  const r = rng(seed);
  const int = (lo, hi) => lo + Math.floor(r() * (hi - lo + 1));
  const pick = (list) => list[Math.floor(r() * list.length)];
  const chance = (p) => r() < p;

  const nouns = vocab.words.filter((w) => w.role === "noun").map((w) => w.name);
  const objectNouns = vocab.words.filter((w) => w.role === "noun" && !w.selects).map((w) => w.name);
  const properties = vocab.words.filter((w) => w.role === "property").map((w) => w.name);
  const conditions = vocab.words.filter((w) => w.role === "condition").map((w) => w.name);
  const placeable = vocab.cards.filter((c) => c.type === "minion" && c.noun);
  const holdable = vocab.cards.filter((c) => c.type === "spell" || (c.type === "minion" && c.noun));

  const width = int(4, 8);
  const height = Math.min(8, Math.max(4, width + int(-2, 2)));
  const free = [];
  for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) free.push([x, y]);
  const takeFree = () => (free.length ? free.splice(Math.floor(r() * free.length), 1)[0] : null);
  const isFree = (x, y) => free.some(([fx, fy]) => fx === x && fy === y);
  const claim = (x, y) => free.splice(free.findIndex(([fx, fy]) => fx === x && fy === y), 1);

  const objects = [];
  const cards = [];
  // The cards on the grid first: their nouns are good sentence subjects.
  for (let i = int(0, 3); i > 0 && placeable.length; i--) {
    const at = takeFree();
    if (at) cards.push({ card: pick(placeable).name, x: at[0], y: at[1] });
  }
  const objectKinds = [pick(objectNouns), pick(objectNouns)];
  const cardNouns = cards.map((c) => vocab.cards.find((v) => v.name === c.card).noun);
  const subjects = [...objectKinds, ...cardNouns];

  // Whole sentences, in a row or a column: NOUN [AND NOUN] [CONDITION NOUN [AND NOUN]] IS [NOT]
  // (PROPERTY | NOUN) [AND ...]. A condition only with a vocabulary that has condition words
  // (Module 11 on), so the cases of earlier modules' vocabularies don't change.
  const sentence = (first) => {
    // The first is the player's: the first object noun IS YOU (that noun always has an object).
    if (first) return [objectKinds[0], "IS", "YOU"];
    const line = [chance(0.7) ? pick(subjects) : pick(nouns)];
    if (chance(0.15)) line.push("AND", pick(subjects));
    if (conditions.length && chance(0.4)) {
      line.push(pick(conditions), pick(subjects));
      if (chance(0.2)) line.push("AND", pick(nouns));
    }
    line.push("IS");
    if (chance(0.15)) line.push("NOT");
    line.push(chance(0.75) ? pick(properties) : pick(nouns));
    if (chance(0.2)) line.push("AND", pick(properties));
    return line;
  };
  const sentences = [];
  if (chance(0.9)) sentences.push(sentence(true));
  for (let i = int(1, 2); i > 0; i--) sentences.push(sentence(false));
  for (const line of sentences) {
    for (let tries = 0; tries < 50; tries++) {
      const across = chance(0.5);
      const [dx, dy] = across ? [1, 0] : [0, 1];
      const [maxX, maxY] = [width - 1 - dx * (line.length - 1), height - 1 - dy * (line.length - 1)];
      if (maxX < 0 || maxY < 0) continue;
      const [x0, y0] = [int(0, maxX), int(0, maxY)];
      const cells = line.map((_, i) => [x0 + dx * i, y0 + dy * i]);
      if (!cells.every(([x, y]) => isFree(x, y))) continue;
      cells.forEach(([x, y], i) => {
        claim(x, y);
        objects.push({ kind: `text:${line[i]}`, x, y });
      });
      break;
    }
  }
  // Loose words, the objects of the nouns, and now and then one object on an occupied tile.
  const allWords = [...vocab.words.map((w) => w.name), ...OPERATORS];
  for (let i = int(0, 3); i > 0; i--) {
    const at = takeFree();
    if (at) objects.push({ kind: `text:${pick(allWords)}`, x: at[0], y: at[1] });
  }
  for (let i = int(1, 6); i > 0; i--) {
    const at = takeFree();
    if (at) objects.push({ kind: (i === 1 ? objectKinds[0] : pick(objectKinds)).toLowerCase(), x: at[0], y: at[1] });
  }
  if (objects.length && chance(0.2)) {
    const { x, y } = pick(objects);
    objects.push({ kind: pick(objectKinds).toLowerCase(), x, y });
  }
  const hand = [];
  for (let i = int(0, 3); i > 0 && holdable.length; i--) hand.push(pick(holdable).name);

  // The script: moves, casts and plays (mostly of cards in the hand), and the chooser's answers.
  const answer = () => {
    const k = r();
    return k < 0.5 ? String(int(1, 3)) : k < 0.8 ? pick(vocab.cards).name : pick(["top", "bottom"]);
  };
  const coords = () => {
    if (cards.length && chance(0.5)) {
      const c = pick(cards);
      return [c.x, c.y];
    }
    return [int(0, width), int(0, height)];
  };
  const minions = vocab.cards.filter((c) => c.type === "minion");
  const script = [];
  const length = int(5, 25);
  let base = null; // the base the last Assemble asked for by name, if any
  while (script.length < length) {
    const k = r();
    if (k < 0.6) script.push(pick(DIRS));
    else if (k < 0.93) {
      // After an Assemble, now and then play what it may have made: "<minion> (Assembled)".
      if (base && chance(0.4)) {
        script.push(`c ${chance(0.8) ? base : pick(minions).name} (Assembled) ${int(0, width - 1)} ${int(0, height - 1)}`);
        continue;
      }
      const name = hand.length && chance(0.75) ? pick(hand) : pick(vocab.cards).name;
      const card = vocab.cards.find((c) => c.name === name);
      const at = chance(card.type === "spell" ? 0.5 : 0.85) ? coords() : null;
      script.push(at ? `c ${card.name} ${at[0]} ${at[1]}` : `c ${card.name}`);
      if (card.chooses) {
        const answers = Array.from({ length: int(0, 3) }, answer);
        if (answers.length && minions.length && chance(0.5)) answers[0] = pick(minions).name;
        if (minions.some((m) => m.name === answers[0])) base = answers[0];
        script.push(...answers);
      }
    } else script.push(answer());
  }
  return { level: { width, height, objects, cards, hand }, script: script.slice(0, 25) };
}

/** A unified diff of two texts, line by line, with 3 lines of context. */
export function unifiedDiff(a, b, nameA = "rust", nameB = "ts") {
  const x = a.split("\n");
  const y = b.split("\n");
  const lcs = Array.from({ length: x.length + 1 }, () => new Uint32Array(y.length + 1));
  for (let i = x.length - 1; i >= 0; i--) {
    for (let j = y.length - 1; j >= 0; j--) {
      lcs[i][j] = x[i] === y[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
    }
  }
  // Each op: [sign, line, lines of a before it, lines of b before it].
  const ops = [];
  for (let i = 0, j = 0; i < x.length || j < y.length;) {
    if (i < x.length && j < y.length && x[i] === y[j]) ops.push([" ", x[i], i++, j++]);
    else if (i < x.length && (j === y.length || lcs[i + 1][j] >= lcs[i][j + 1])) ops.push(["-", x[i], i++, j]);
    else ops.push(["+", y[j], i, j++]);
  }
  const out = [`--- ${nameA}`, `+++ ${nameB}`];
  const changed = ops.flatMap((op, k) => (op[0] === " " ? [] : [k]));
  for (let k = 0; k < changed.length;) {
    const first = Math.max(0, changed[k] - 3);
    let last = changed[k];
    while (k < changed.length && changed[k] <= last + 6) last = changed[k++];
    const hunk = ops.slice(first, Math.min(ops.length, last + 4));
    const aLen = hunk.filter((op) => op[0] !== "+").length;
    const bLen = hunk.filter((op) => op[0] !== "-").length;
    out.push(`@@ -${hunk[0][2] + (aLen ? 1 : 0)},${aLen} +${hunk[0][3] + (bLen ? 1 : 0)},${bLen} @@`);
    for (const [sign, line] of hunk) out.push(`${sign}${line}`);
  }
  return out.join("\n");
}

/** Run `cmd args` in `cwd` with `input` on stdin; resolves to { out, err, code, signal }. */
function runCli(cmd, args, cwd, input) {
  return new Promise((done) => {
    const child = spawn(cmd, args, { cwd, timeout: TIMEOUT_MS, killSignal: "SIGKILL" });
    let out = "";
    let err = "";
    child.stdout.on("data", (d) => (out += d));
    child.stderr.on("data", (d) => (err += d));
    child.stdin.on("error", () => {});
    child.on("close", (code, signal) => done({ out, err, code, signal }));
    child.stdin.end(input);
  });
}

/** Did a CLI run crash? Both CLIs exit 0 (won) or 1 (lost, quit) and print nothing on stderr otherwise. */
export const crashed = (r) => r.signal != null || (r.code !== 0 && r.code !== 1) || r.err.trim() !== "";

/** Compare a Rust run with a TypeScript run: "agree", "outputs differ", or which side crashed. */
export function verdict(rust, ts) {
  const [rc, tc] = [crashed(rust), crashed(ts)];
  if (rc && tc) return "both crashed";
  if (rc) return "the Rust CLI crashed";
  if (tc) return "the TypeScript CLI crashed";
  return rust.out === ts.out ? "agree" : "outputs differ";
}

const why = (r) =>
  r.signal === "SIGKILL" ? `timed out after ${TIMEOUT_MS / 1000} s`
  : r.signal ? `killed by ${r.signal}`
  : r.err.trim().split("\n").slice(0, 8).join("\n") || `exit code ${r.code}`;

/** The highest module whose Rust and TypeScript references both have the SpellGrid CLI. */
function defaultModule() {
  const withCli = moduleDirs().filter((m) =>
    existsSync(join(m.path, "reference", "ts", "src", "cli.ts")) && existsSync(join(m.path, "reference", "rust", "src", "cli.rs")));
  if (!withCli.length) fail("diff-tracks: no module has a SpellGrid CLI in both references");
  return withCli.at(-1).num;
}

async function main() {
  const opts = parseArgs(process.argv.slice(2));
  const num = opts.module ?? defaultModule();
  const base = opts.dir ? resolve(opts.dir) : mkdtempSync(join(tmpdir(), "diff-tracks-"));
  if (existsSync(base) && readdirSync(base).length && !existsSync(join(base, MARKER))) {
    fail(`diff-tracks: ${base} is not empty and was not staged by diff-tracks; give an empty or new directory`);
  }
  mkdirSync(base, { recursive: true });
  writeFileSync(join(base, MARKER), `${num}\n`);
  const rust = stagePlay("rust", join(base, "rust"), num);
  const ts = stagePlay("ts", join(base, "ts"), num);
  const cases = join(base, "cases");
  mkdirSync(cases, { recursive: true });

  const build = spawnSync("cargo", ["build", "--quiet", "--bin", "spellgrid"], { cwd: rust, stdio: ["ignore", "inherit", "inherit"] });
  if (build.status !== 0) fail(`diff-tracks: cargo build failed in ${rust}`);
  if (!existsSync(join(ts, "node_modules"))) {
    const r = spawnSync("npm", ["install", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: ts, stdio: ["ignore", "ignore", "inherit"] });
    if (r.status !== 0) fail(`diff-tracks: npm install failed in ${ts}`);
  }
  const vocab = loadVocab(join(ts, "src", "content", "words.json"), [join(ts, "cards")]);

  const seeds = opts.seed !== null ? [opts.seed] : Array.from({ length: opts.seeds }, (_, i) => i + 1);
  const width = Math.max(1, Math.min(8, availableParallelism()));
  const runCase = async (seed) => {
    const { level, script } = generateCase(seed, vocab);
    const levelPath = join(cases, `seed-${seed}.json`);
    writeFileSync(levelPath, `${JSON.stringify(level, null, 2)}\n`);
    writeFileSync(join(cases, `seed-${seed}.txt`), `${script.join("\n")}\n`);
    const input = `${script.join("\n")}\n`;
    const [a, b] = await Promise.all([
      runCli(join(rust, "target", "debug", "spellgrid"), [levelPath], rust, input),
      runCli(process.execPath, ["--import", "tsx", "src/cli.ts", levelPath], ts, input),
    ]);
    return { seed, levelPath, script, rust: a, ts: b };
  };

  let bad = null;
  for (let k = 0; k < seeds.length && !bad; k += width) {
    const results = await Promise.all(seeds.slice(k, k + width).map(runCase));
    bad = results.find((c) => verdict(c.rust, c.ts) !== "agree") ?? null;
  }

  if (!bad) {
    console.log(`diff-tracks: module ${num}, ${seeds.length} seed(s), Rust and TypeScript agree`);
    if (!opts.dir && !opts.keep) rmSync(base, { recursive: true, force: true });
    return;
  }
  console.log(`diff-tracks: module ${num}, seed ${bad.seed}: ${verdict(bad.rust, bad.ts)}`);
  console.log(`level: ${bad.levelPath}`);
  console.log(`script:\n${bad.script.map((l) => `  ${l}`).join("\n")}`);
  if (crashed(bad.rust)) console.log(`Rust:\n${why(bad.rust)}`);
  if (crashed(bad.ts)) console.log(`TypeScript:\n${why(bad.ts)}`);
  if (bad.rust.out !== bad.ts.out) console.log(unifiedDiff(bad.rust.out, bad.ts.out));
  console.log(`(staged in ${base}; kept)`);
  process.exit(1);
}

if (isMain(import.meta.url)) await main();