systemic-game-design-course/tools/mutation-check.mjs

#!/usr/bin/env node
// Mutation check: are the course's ordering decisions pinned by tests?
//
//   node tools/mutation-check.mjs [name ...] [--mutants <file>] [--module NN]
//   npm run mutation:check
//
// Reads tools/mutants.json (or --mutants), a list of mutants
//   { "name": "Hazards in reverse order", "file": "src/content/hazards.ts",
//     "find": "return [sink, hotMelt, defeat];", "replace": "return [defeat, hotMelt, sink];",
//     "expect": ["test_09_hazard_order_melt_before_defeat"] }
// and checks each one against the TypeScript reference of the highest module that has one
// (or --module NN). The reference is staged once with the tests of every module up to it, as
// verify-references does, and its dependencies are installed. The unmutated reference must pass.
// Then, for each mutant (or only the ones named), the tool replaces `find` in `file` with
// `replace` (`find` must occur exactly once), runs the tests, and restores the file. A mutant
// is killed when every test in `expect` fails and the run is otherwise sane: every test of the
// unmutated run ran again (no test file failed to load). Prints one line per mutant with the
// tests that failed; exits 1 if any mutant survives, any `find` is stale, or any run is not sane.
//
// MUTATION_CMD replaces the test command, and then nothing is installed (the tool's tests use it).
// Whatever runs must write a vitest JSON report to `.mutation-report.json` in the staged directory.
import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { fail, isMain, moduleDirs } from "./lib.mjs";
import { planVerification, stage } from "./verify-references.mjs";

const REPORT = ".mutation-report.json";
const DEFAULT_CMD = `npx vitest run --reporter=json --outputFile=${REPORT}`;
const KEYS = ["name", "file", "find", "replace", "expect"];

export function parseArgs(argv) {
  const opts = { names: [], mutants: null, module: null };
  for (let i = 0; i < argv.length; i++) {
    if (argv[i] === "--mutants") opts.mutants = argv[++i];
    else if (argv[i] === "--module") opts.module = String(argv[++i]).padStart(2, "0");
    else if (argv[i].startsWith("--")) fail(`mutation-check: unknown option ${argv[i]}\nusage: mutation-check.mjs [name ...] [--mutants <file>] [--module NN]`);
    else opts.names.push(argv[i]);
  }
  return opts;
}

/** What is wrong with a mutant entry's shape, or [] if nothing. */
export function mutantProblems(m) {
  if (typeof m !== "object" || m === null || Array.isArray(m)) return ["a mutant must be an object"];
  const problems = [];
  for (const key of Object.keys(m)) if (!KEYS.includes(key)) problems.push(`unknown key "${key}"`);
  for (const key of ["name", "file", "find"]) {
    if (typeof m[key] !== "string" || m[key] === "") problems.push(`${key} must be a non-empty string`);
  }
  if (typeof m.replace !== "string") problems.push("replace must be a string");
  else if (m.replace === m.find) problems.push("replace is the same as find: nothing would change");
  if (!Array.isArray(m.expect) || m.expect.length === 0 || !m.expect.every((t) => typeof t === "string" && /^test_\w+$/.test(t))) {
    problems.push("expect must list at least one test name (test_...)");
  }
  return problems;
}

/** `text` with the mutant applied. Throws if `find` does not occur exactly once (a stale mutant). */
export function applyMutant(text, mutant) {
  const count = text.split(mutant.find).length - 1;
  if (count !== 1) {
    throw new Error(`stale: \`find\` occurs ${count} times in ${mutant.file}, not once`);
  }
  return text.replace(mutant.find, () => mutant.replace);
}

/**
 * Read a vitest JSON report: the names of the tests that passed and failed, and the test files
 * that failed without a failing test (a file that did not load, or an error outside a test).
 */
export function readReport(json) {
  const passed = new Set();
  const failed = new Set();
  const broken = [];
  for (const file of json.testResults ?? []) {
    const results = file.assertionResults ?? [];
    for (const t of results) {
      if (t.status === "passed") passed.add(t.title);
      else if (t.status === "failed") failed.add(t.title);
    }
    if (file.status === "failed" && !results.some((t) => t.status === "failed")) {
      broken.push(`${file.name}: ${String(file.message ?? "").split("\n")[0]}`);
    }
  }
  return { passed, failed, broken };
}

/**
 * The verdict on one mutant run, given the unmutated run (`baseline`) and this one (`run`), both
 * from readReport: { killed, sane, failed: [names], problems: [text] }.
 */
export function judge(mutant, baseline, run) {
  const problems = [];
  const unknown = mutant.expect.filter((t) => !baseline.passed.has(t));
  if (unknown.length) problems.push(`expects tests the reference doesn't have: ${unknown.join(", ")}`);
  for (const b of run.broken) problems.push(`a test file did not run: ${b}`);
  const missing = [...baseline.passed].filter((t) => !run.passed.has(t) && !run.failed.has(t));
  if (missing.length && run.broken.length === 0) problems.push(`${missing.length} tests did not run, e.g. ${missing.slice(0, 3).join(", ")}`);
  const sane = problems.length === 0;
  const survivors = mutant.expect.filter((t) => !run.failed.has(t));
  return { killed: sane && survivors.length === 0, sane, survivors, failed: [...run.failed].sort(), problems };
}

/** Run the tests in `dir`; returns readReport of what they wrote, or throws. */
function runTests(dir, cmd) {
  rmSync(join(dir, REPORT), { force: true });
  const r = spawnSync(cmd, { cwd: dir, shell: true, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
  const report = join(dir, REPORT);
  if (!existsSync(report)) throw new Error(`the tests wrote no report (exit ${r.status}):\n${(r.stdout || "") + (r.stderr || "")}`);
  return readReport(JSON.parse(readFileSync(report, "utf8")));
}

/** The plan entry (as verify-references makes it) for module `num`, or the highest with a TS reference. */
export function targetModule(num) {
  const plans = planVerification(moduleDirs(), 99).filter((p) => p.langs.includes("ts"));
  const plan = num === null ? plans.at(-1) : plans.find((p) => p.num === num);
  if (!plan) fail(num === null ? "mutation-check: no module has a TypeScript reference" : `mutation-check: module ${num} has no TypeScript reference`);
  return plan;
}

function main() {
  const opts = parseArgs(process.argv.slice(2));
  const file = opts.mutants ?? fileURLToPath(new URL("./mutants.json", import.meta.url));
  let mutants = JSON.parse(readFileSync(file, "utf8"));
  if (!Array.isArray(mutants)) fail(`${file}: expected a list of mutants`);
  const shapes = mutants.flatMap((m, i) => mutantProblems(m).map((p) => `mutant ${i + 1} (${m?.name ?? "no name"}): ${p}`));
  if (shapes.length) fail(shapes.join("\n"));
  if (opts.names.length) {
    mutants = mutants.filter((m) => opts.names.includes(m.name));
    if (mutants.length === 0) fail(`no mutant named ${opts.names.join(", ")} in ${file}`);
  }

  const plan = targetModule(opts.module);
  const dir = stage(plan, "ts");
  const cmd = process.env.MUTATION_CMD || DEFAULT_CMD;
  let bad = 0;
  try {
    if (existsSync(join(dir, "package.json")) && !process.env.MUTATION_CMD) {
      const r = spawnSync("npm", ["install", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" });
      if (r.status !== 0) throw new Error(`npm install failed:\n${r.stdout}${r.stderr}`);
    }
    console.log(`Module ${plan.num} TypeScript reference, tests of Modules ${plan.testModules[0].num}-${plan.num}`);
    const baseline = runTests(dir, cmd);
    if (baseline.failed.size || baseline.broken.length || baseline.passed.size === 0) {
      throw new Error(`the unmutated reference does not pass: ${[...baseline.failed, ...baseline.broken].join(", ") || "no tests ran"}`);
    }
    console.log(`unmutated: ${baseline.passed.size} tests pass`);
    for (const mutant of mutants) {
      const path = join(dir, mutant.file);
      if (!existsSync(path)) {
        bad++;
        console.log(`STALE    ${mutant.name}: no file ${mutant.file}`);
        continue;
      }
      const original = readFileSync(path, "utf8");
      let mutated;
      try {
        mutated = applyMutant(original, mutant);
      } catch (e) {
        bad++;
        console.log(`STALE    ${mutant.name}: ${e.message}`);
        continue;
      }
      writeFileSync(path, mutated);
      let verdict;
      try {
        verdict = judge(mutant, baseline, runTests(dir, cmd));
      } catch (e) {
        verdict = { killed: false, sane: false, survivors: [], failed: [], problems: [e.message] };
      } finally {
        writeFileSync(path, original);
      }
      const failed = verdict.failed.join(", ") || "none";
      if (verdict.killed) console.log(`killed   ${mutant.name}: ${failed}`);
      else {
        bad++;
        if (!verdict.sane) console.log(`INSANE   ${mutant.name}: ${verdict.problems.join("; ")}`);
        else console.log(`SURVIVED ${mutant.name}: ${verdict.survivors.join(", ")} passed; failed: ${failed}`);
      }
    }
  } catch (e) {
    console.error(`mutation-check: ${e.message}`);
    process.exitCode = 1;
    return;
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
  console.log(bad === 0 ? `${mutants.length} mutants killed` : `${bad} of ${mutants.length} mutants not killed`);
  process.exit(bad === 0 ? 0 : 1);
}

if (isMain(import.meta.url)) main();