systemic-game-design-course/tools/test/mutation-check.test.mjs

import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fixtures, run, toolsDir } from "./helpers.mjs";
import { applyMutant, judge, mutantProblems, parseArgs, readReport, targetModule } from "../mutation-check.mjs";

// A course with one module whose TypeScript reference has one list (src/order.mjs) and a
// stand-in test runner (run-tests.mjs) that writes a vitest-shaped report.
const env = { COURSE_ROOT: join(fixtures, "course-mutation"), MUTATION_CMD: "node run-tests.mjs" };
const SWAP = {
  name: "Order swapped",
  file: "src/order.mjs",
  find: 'export const ORDER = ["first", "second"];',
  replace: 'export const ORDER = ["second", "first"];',
  expect: ["test_01_first_comes_first"],
};

/** Run the tool on the fixture course with these mutants. */
function check(mutants, ...args) {
  const file = join(mkdtempSync(join(tmpdir(), "sdc-mut-")), "mutants.json");
  writeFileSync(file, JSON.stringify(mutants));
  return run("mutation-check.mjs", ["--mutants", file, ...args], env);
}

test("mutation_check_parses_args", () => {
  assert.deepEqual(parseArgs([]), { names: [], mutants: null, module: null });
  assert.deepEqual(parseArgs(["A", "--mutants", "m.json", "--module", "9", "B"]), { names: ["A", "B"], mutants: "m.json", module: "09" });
});

test("mutation_check_kills_a_fixture_mutant", () => {
  const r = check([SWAP]);
  assert.equal(r.code, 0, r.out);
  assert.match(r.out, /unmutated: 2 tests pass/);
  assert.match(r.out, /killed {3}Order swapped: test_01_first_comes_first\n/);
  assert.match(r.out, /1 mutants killed/);
  // The file is restored: the staged copy is gone, and the course's copy was never touched.
  assert.match(readFileSync(join(env.COURSE_ROOT, "01-m01", "reference", "ts", "src", "order.mjs"), "utf8"), /\["first", "second"\]/);
});

test("mutation_check_fails_on_a_surviving_mutant", () => {
  // Renaming the second step changes nothing the fixture's tests look at.
  const survivor = { ...SWAP, name: "Second renamed", replace: 'export const ORDER = ["first", "later"];' };
  const r = check([SWAP, survivor]);
  assert.equal(r.code, 1, r.out);
  assert.match(r.out, /killed {3}Order swapped/);
  assert.match(r.out, /SURVIVED Second renamed: test_01_first_comes_first passed; failed: none/);
  assert.match(r.out, /1 of 2 mutants not killed/);
});

test("mutation_check_fails_on_a_stale_find", () => {
  const stale = { ...SWAP, name: "Stale", find: 'export const ORDER = ["first", "third"];' };
  const r = check([stale]);
  assert.equal(r.code, 1, r.out);
  assert.match(r.out, /STALE {4}Stale: stale: `find` occurs 0 times in src\/order\.mjs, not once/);
  const missing = check([{ ...SWAP, name: "Gone", file: "src/gone.mjs" }]);
  assert.equal(missing.code, 1);
  assert.match(missing.out, /STALE {4}Gone: no file src\/gone\.mjs/);
});

test("mutation_check_fails_on_a_run_that_is_not_sane", () => {
  // A mutant that breaks the module kills every test, but by not loading: that isn't a kill.
  const r = check([{ ...SWAP, name: "Broken", replace: "export const ORDER = [" }]);
  assert.equal(r.code, 1, r.out);
  assert.match(r.out, /INSANE {3}Broken: a test file did not run: order: /);
});

test("mutation_check_runs_only_the_named_mutants", () => {
  const r = check([SWAP, { ...SWAP, name: "Stale", find: "nowhere" }], "Order swapped");
  assert.equal(r.code, 0, r.out);
  assert.doesNotMatch(r.out, /Stale/);
});

test("mutation_check_applies_exactly_one_replacement", () => {
  assert.equal(applyMutant("a [x] b", { file: "f", find: "[x]", replace: "$&y" }), "a $&y b");
  assert.throws(() => applyMutant("x x", { file: "f", find: "x", replace: "y" }), /occurs 2 times in f/);
});

test("mutation_check_judges_from_the_report", () => {
  const report = (tests, extra = {}) => ({ testResults: [{ name: "t", status: "failed", assertionResults: tests.map(([title, status]) => ({ title, status })), ...extra }] });
  const baseline = readReport(report([["test_a", "passed"], ["test_b", "passed"]]));
  const m = { expect: ["test_a"] };
  assert.equal(judge(m, baseline, readReport(report([["test_a", "failed"], ["test_b", "passed"]]))).killed, true);
  assert.deepEqual(judge(m, baseline, readReport(report([["test_a", "passed"], ["test_b", "failed"]]))).survivors, ["test_a"]);
  const lost = judge(m, baseline, readReport(report([["test_a", "failed"]])));
  assert.equal(lost.sane, false);
  assert.match(lost.problems[0], /1 tests did not run, e\.g\. test_b/);
  assert.match(judge({ expect: ["test_zz"] }, baseline, baseline).problems[0], /doesn't have: test_zz/);
});

test("mutation_check_rejects_malformed_mutants", () => {
  assert.deepEqual(mutantProblems(SWAP), []);
  assert.ok(mutantProblems({ ...SWAP, expect: [] }).some((p) => /expect/.test(p)));
  assert.ok(mutantProblems({ ...SWAP, replace: SWAP.find }).some((p) => /nothing would change/.test(p)));
  assert.ok(mutantProblems({ ...SWAP, why: "x" }).some((p) => /unknown key "why"/.test(p)));
});

test("mutation_check_course_mutants_are_well_formed", () => {
  const mutants = JSON.parse(readFileSync(join(toolsDir, "mutants.json"), "utf8"));
  for (const m of mutants) assert.deepEqual(mutantProblems(m), [], m.name);
  assert.equal(new Set(mutants.map((m) => m.name)).size, mutants.length, "names are unique");
  // Each mutant's find occurs once in the highest module's TypeScript reference (the full run checks it too).
  const ref = join(targetModule(null).path, "reference", "ts");
  for (const m of mutants) assert.doesNotThrow(() => applyMutant(readFileSync(join(ref, m.file), "utf8"), m), m.name);
});