systemic-game-design-course/tools/test/verify-levels.test.mjs

import { test } from "node:test";
import assert from "node:assert/strict";
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { toolsDir } from "./helpers.mjs";
import { checkProof, describe, EXHAUST_MAX, parseArgs, proofProblems, solverRuns } from "../verify-levels.mjs";

const levelsDir = join(toolsDir, "..", "course", "10-spellgrid", "levels");

// What the solver prints (tools/solver/solve.ts), cut down to the lines verify-levels reads.
const SOLVED_IN = (n) => ({ code: 0, out: `1 input: 3 new positions, 4 in all\nsolution in ${n}: right; cast Whirlwind\n` });
const EXHAUSTED = { code: 1, out: "2 inputs: 0 new positions, 9 in all\nno solution: every reachable position was searched\n" };
const CUT_OFF = { code: 1, out: `no solution within ${EXHAUST_MAX} inputs (the search was cut off, not exhausted)\n` };

/** A fake solver: answers each run from `answer(args)` and records the arguments it was given. */
function fakeSolver(answer) {
  const calls = [];
  const run = async (args) => {
    calls.push(args);
    return answer(args);
  };
  return { run, calls };
}

test("verify_levels_parses_args", () => {
  assert.deepEqual(parseArgs([]), { levels: [], proofs: null, dir: null });
  assert.deepEqual(parseArgs(["l4", "--proofs", "p.json", "l5-x", "--dir", "d"]), {
    levels: ["l4", "l5-x"],
    proofs: "p.json",
    dir: "d",
  });
});

test("verify_levels_runs_the_solver_once_per_claim", () => {
  assert.deepEqual(solverRuns({ level: "l2", shortest: 4 }).map((r) => r.args), [["--max", "4"]]);
  assert.deepEqual(
    solverRuns({ level: "l1", shortest: 10, neverRule: "DEATHRATTLE IS DOUBLE" }).map((r) => r.args),
    [["--max", "10"], ["--max", "60", "--never-rule", "DEATHRATTLE IS DOUBLE"]],
  );
  assert.deepEqual(
    solverRuns({ level: "l3", shortest: 6, naiveUnsolvable: true }).map((r) => r.args),
    [["--max", "6"], ["--max", "60", "--naive"]],
  );
  assert.equal(describe({ level: "l3", shortest: 6, naiveUnsolvable: true }), "shortest 6; none under --naive");
});

test("verify_levels_accepts_a_proof_that_holds", async () => {
  const proof = { level: "l1", shortest: 10, neverRule: "DEATHRATTLE IS DOUBLE" };
  const solver = fakeSolver((args) => (args.includes("--never-rule") ? EXHAUSTED : SOLVED_IN(10)));
  assert.deepEqual(await checkProof(proof, solver.run), []);
  assert.equal(solver.calls.length, 2);
});

test("verify_levels_reports_a_proof_that_fails", async () => {
  // The fixture proof claims 5 inputs; the solver finds 4, so the claim "none shorter" is false.
  const shorter = await checkProof({ level: "l3", shortest: 5 }, fakeSolver(() => SOLVED_IN(4)).run);
  assert.equal(shorter.length, 1);
  assert.match(shorter[0], /a solution in 4 inputs, shorter than 5/);

  const none = await checkProof({ level: "l3", shortest: 5 }, fakeSolver(() => EXHAUSTED).run);
  assert.deepEqual(none, ["no solution within 5 inputs"]);

  const cooked = await checkProof(
    { level: "l4", shortest: 7, neverRule: "DEATHRATTLE IS DOUBLE" },
    fakeSolver((args) => (args.includes("--never-rule") ? SOLVED_IN(9) : SOLVED_IN(7))).run,
  );
  assert.equal(cooked.length, 1);
  assert.match(cooked[0], /solved without DEATHRATTLE IS DOUBLE in 9 inputs/);

  const cutOff = await checkProof(
    { level: "l3", shortest: 6, naiveUnsolvable: true },
    fakeSolver((args) => (args.includes("--naive") ? CUT_OFF : SOLVED_IN(6))).run,
  );
  assert.equal(cutOff.length, 1);
  assert.match(cutOff[0], /under --naive was cut off at 60 inputs, not exhausted/);
});

test("verify_levels_rejects_malformed_proofs", async () => {
  assert.deepEqual(proofProblems({ level: "l1", shortest: 10 }), []);
  assert.ok(proofProblems({ level: "l1" }).some((p) => /shortest/.test(p)));
  assert.ok(proofProblems({ level: "l1", shortest: 3, neverRule: "double" }).some((p) => /neverRule/.test(p)));
  assert.ok(proofProblems({ level: "l1", shortest: 3, naive: true }).some((p) => /unknown key "naive"/.test(p)));
  const solver = fakeSolver(() => SOLVED_IN(3));
  assert.notDeepEqual(await checkProof({ level: "l1", shortest: 0 }, solver.run), []);
  assert.equal(solver.calls.length, 0);
});

test("verify_levels_every_level_has_one_proof", () => {
  const levels = readdirSync(levelsDir).filter((f) => /^l\d+-.*\.json$/.test(f)).map((f) => f.replace(/\.json$/, "")).sort();
  const proofs = JSON.parse(readFileSync(join(levelsDir, "proofs.json"), "utf8"));
  for (const proof of proofs) assert.deepEqual(proofProblems(proof), [], proof.level);
  assert.deepEqual(proofs.map((p) => p.level).sort(), levels);
  // Each proof says what the level is for: a rule it can't be solved without, or the naive engine.
  for (const proof of proofs.filter((p) => p.level !== "l2-spliced-corpse")) {
    assert.ok(proof.neverRule !== undefined || proof.naiveUnsolvable, `${proof.level} proves nothing about its idea`);
  }
});