#!/usr/bin/env node // Usage: node scripts/mutate.mjs [--jobs N] [--skip-failing] [file...] // // Mutation analysis of the JS reference solutions: how many small bugs would the JS test suite notice? // Each mutant changes one token on one line of one file in js/solutions (e.g. `<=` -> `<`, `===` -> `!==`, // `+` -> `-`, `1` -> `2`, dropping a `!`). The whole JS suite runs against each mutant in a temp copy of the repo // whose js/src holds the solutions. A mutant is "killed" when some test fails (or the run times out), and // "survives" when every test still passes: either the suite has a gap, or the mutant is equivalent (it cannot // change behaviour, e.g. `x >= 0` -> `x > 0` where x is never 0). // // Files: paths relative to js/solutions (e.g. `game.js pong/rules.js`); default: every .js file there except the // browser glue (browser.js, *main.js) and todo.js. // Allow-list: scripts/mutate-allow.txt lists known equivalent mutants, one per line, as // : => # // (lines trimmed, no line number, so the list survives edits elsewhere in the file). The exact line to paste is // printed for each survivor. // Exit: 0 when every survivor is allow-listed, 1 when there are others, 2 on a setup error (including a suite that // already fails on the unmutated solutions; --skip-failing then leaves out the failing test files, with a warning). import { readFileSync, writeFileSync, cpSync, rmSync, mkdtempSync, readdirSync, existsSync } from 'node:fs' import { spawn, spawnSync } from 'node:child_process' import { join, dirname, relative, basename } from 'node:path' import { tmpdir, availableParallelism } from 'node:os' import { fileURLToPath } from 'node:url' const repo = join(dirname(fileURLToPath(import.meta.url)), '..') const allowPath = join(repo, 'scripts', 'mutate-allow.txt') const TIMEOUT_MS = 60000 const argv = process.argv.slice(2) const flag = (name) => argv.includes(name) const jobsAt = argv.indexOf('--jobs') const jobs = jobsAt >= 0 ? Number(argv[jobsAt + 1]) : Math.max(1, availableParallelism() - 1) const fileArgs = argv.filter((a, i) => !a.startsWith('--') && !(jobsAt >= 0 && i === jobsAt + 1)) if (!Number.isInteger(jobs) || jobs < 1) { console.error('mutate: --jobs needs a positive integer'); process.exit(2) } const listJs = (dir) => readdirSync(dir, { withFileTypes: true }).flatMap((e) => e.isDirectory() ? listJs(join(dir, e.name)) : e.name.endsWith('.js') ? [join(dir, e.name)] : []) const isGlue = (f) => basename(f) === 'browser.js' || basename(f).endsWith('main.js') || basename(f) === 'todo.js' const solutions = join(repo, 'js', 'solutions') const files = fileArgs.length > 0 ? fileArgs.map((f) => f.replace(/^js\/solutions\//, '')) : listJs(solutions).map((f) => relative(solutions, f)).filter((f) => !isGlue(f)).sort() const missing = files.filter((f) => !existsSync(join(solutions, f))) if (missing.length > 0) { console.error(`mutate: not in js/solutions: ${missing.join(', ')}`); process.exit(2) } // Single-token rules. Each match of each rule on a code line is one mutant. const rules = [ [/<=/g, '<'], [/>=/g, '>'], [/(?!])<(?![=<])/g, '<='], [/(?-])>(?![=>])/g, '>='], [/===/g, '!=='], [/!==/g, '==='], [/&&/g, '||'], [/\|\|/g, '&&'], [/ \+ /g, ' - '], [/ - /g, ' + '], [/ % /g, ' * '], [/\b0\b/g, '1'], [/\b1\b/g, '2'], [/\b2\b/g, '3'], [/\b3\b/g, '4'], [/\b8\b/g, '7'], [/!(?=[a-z(])/g, ''], ] const isCodeLine = (line) => !/^\s*(\/\/|\*|\/\*|import\b)/.test(line) const stripComment = (line) => line.replace(/\/\/.*$/, '') const mutantsOf = (file) => { const lines = readFileSync(join(solutions, file), 'utf8').split('\n') return lines.flatMap((line, li) => { if (!isCodeLine(line)) return [] const code = stripComment(line) return rules.flatMap(([re, rep]) => [...code.matchAll(re)].map((m) => { const mutated = line.slice(0, m.index) + rep + line.slice(m.index + m[0].length) return { file, line: li + 1, original: line.trim(), mutated: mutated.trim(), source: [...lines.slice(0, li), mutated, ...lines.slice(li + 1)].join('\n') } })) }) } const keyOf = (m) => `${m.file}: ${m.original} => ${m.mutated}` // The allow-list: `key # reason`; a line without a reason is an error, so nobody adds one without thinking. const readAllow = () => { if (!existsSync(allowPath)) return new Map() const entries = readFileSync(allowPath, 'utf8').split('\n') .map((l, i) => [l.trim(), i + 1]).filter(([l]) => l !== '' && !l.startsWith('#')) return new Map(entries.map(([l, n]) => { const at = l.lastIndexOf(' # ') if (at < 0 || l.slice(at + 4).trim() === '') { console.error(`mutate: ${relative(repo, allowPath)}:${n}: missing " # reason"`); process.exit(2) } return [l.slice(0, at).trim(), l.slice(at + 4).trim()] })) } // A temp copy of the repo parts the JS suite needs, with js/src replaced by the solutions. const makeCopy = () => { const work = mkdtempSync(join(tmpdir(), 'pure-snake-mutate-')) cpSync(join(repo, 'js'), join(work, 'js'), { recursive: true, filter: (s) => !s.includes('node_modules') }) cpSync(join(repo, 'fixtures'), join(work, 'fixtures'), { recursive: true }) rmSync(join(work, 'js', 'src'), { recursive: true }) cpSync(join(work, 'js', 'solutions'), join(work, 'js', 'src'), { recursive: true }) return work } const runSuite = (work, args) => new Promise((resolve) => { const child = spawn('node', ['--test', '--test-concurrency=1', ...args], { cwd: join(work, 'js'), stdio: 'ignore' }) const timer = setTimeout(() => { child.kill('SIGKILL'); resolve('timeout') }, TIMEOUT_MS) child.on('exit', (code) => { clearTimeout(timer); resolve(code === 0 ? 'pass' : 'fail') }) }) // Baseline: the suite must pass on the unmutated solutions, or every mutant would look "killed". // With --skip-failing, tests that already fail are skipped by name (a file that fails to load is left out whole). const baseline = makeCopy() const allTests = readdirSync(join(baseline, 'js', 'test')).filter((f) => f.endsWith('.test.js')).sort() .map((f) => `test/${f}`) const failingIn = (t) => { const r = spawnSync('node', ['--test', '--test-reporter=tap', t], { cwd: join(baseline, 'js'), encoding: 'utf8', timeout: TIMEOUT_MS }) if (r.status === 0) return null const names = [...r.stdout.matchAll(/^\s*not ok \d+ - (.*)$/gm)].map((m) => m[1].trim()).filter((n) => n !== t) return { file: t, names } } const failing = allTests.map(failingIn).filter((f) => f !== null) rmSync(baseline, { recursive: true }) const skipFiles = failing.filter((f) => f.names.length === 0).map((f) => f.file) const skipNames = failing.flatMap((f) => f.names) if (failing.length > 0) { console.error('mutate: the suite already fails on the unmutated solutions:') failing.forEach((f) => console.error(` ${f.file}: ${f.names.length === 0 ? '(the whole file)' : f.names.join('; ')}`)) if (!flag('--skip-failing')) { console.error('mutate: fix them first, or pass --skip-failing to leave them out'); process.exit(2) } console.error('mutate: WARNING: running without them; some mutants may survive only because of that') } const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') const suiteArgs = [...skipNames.map((n) => `--test-skip-pattern=^${escapeRe(n)}$`), ...allTests.filter((t) => !skipFiles.includes(t))] const tests = allTests.filter((t) => !skipFiles.includes(t)) const allow = readAllow() // read before the long run, so a malformed line fails fast const mutants = files.flatMap(mutantsOf) console.log(`mutate: ${mutants.length} mutants in ${files.length} files, ${tests.length} test files, ${jobs} jobs`) // A pool of workers, each with its own copy; a worker writes the mutant, runs the suite, restores the file. const results = new Array(mutants.length) const worker = async (queue) => { const work = makeCopy() try { for (const i of queue) { const m = mutants[i] const path = join(work, 'js', 'src', m.file) const original = readFileSync(path, 'utf8') writeFileSync(path, m.source) results[i] = await runSuite(work, suiteArgs) writeFileSync(path, original) } } finally { rmSync(work, { recursive: true, force: true }) } } const queues = Array.from({ length: Math.min(jobs, mutants.length) }, (_, w) => mutants.map((_, i) => i).filter((i) => i % jobs === w)) await Promise.all(queues.map(worker)) const survivors = mutants.filter((_, i) => results[i] === 'pass') const timeouts = results.filter((r) => r === 'timeout').length const unlisted = survivors.filter((m) => !allow.has(keyOf(m))) const listed = survivors.filter((m) => allow.has(keyOf(m))) const survivorKeys = new Set(survivors.map(keyOf)) const stale = [...allow.keys()].filter((k) => files.includes(k.slice(0, k.indexOf(': '))) && !survivorKeys.has(k)) console.log(`killed ${mutants.length - survivors.length} (${timeouts} by timeout), survived ${survivors.length}` + ` (${listed.length} allow-listed as equivalent, ${unlisted.length} not)`) if (unlisted.length > 0) { console.log('\nSurvivors not in scripts/mutate-allow.txt (a test gap, or an equivalent mutant to allow-list with a reason):') unlisted.forEach((m) => console.log(` js/solutions/${m.file}:${m.line}\n - ${m.original}\n + ${m.mutated}\n allow-list line: ${keyOf(m)} # `)) } if (stale.length > 0) { console.log('\nAllow-list entries for these files that no longer occur as survivors (remove or update them):') stale.forEach((k) => console.log(` ${k}`)) } process.exit(unlisted.length > 0 ? 1 : failing.length > 0 ? 2 : 0)