feat(course): audit fixes and twelve extensions (Modules 7b, 12, 13)
Audit pass: - Frame rule counts frames since the last move, so every step lasts exactly movesEvery(score) frames, also right after a speed-up - clampTime + MAX_GAP_MS: a hidden tab no longer replays the whole gap - Pong Restart keeps held keys - Fact corrections (mulberry32 output coverage, Haskell IO, uniform) - New fixture cases from a mutation analysis Extensions (all three tracks, shared fixtures, course text): - Turn queue (Exercise 10.6), persisted high score (10.7), replay viewer (10.8), play() end-to-end helper (9.5) - Optional State (4.7) and optional Module 7b, stack-safe IO - Module 12, property-based testing with generators, shrinking, invariants and a model-based oracle - Module 13, Conway's Life with a Click event - JS type checking via jsconfig, JSDoc and types.d.ts - Maintainer tools: scripts/mutate.mjs, scripts/typecheck-js.sh, Playwright end-to-end suite (e2e/, scripts/e2e.sh) Docs: docs/contract.md is the canonical contract; specs and build log under docs/superpowers. Verified: JS 728, Scala 689, F# 691 unit tests; stubs and progression M01-M13 and M07b; 176 end-to-end tests on Chromium and WebKit. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
1 parent
876f5aabb1
commit
edd57f0148
253 files changed
+21753
-503
No files matched your search
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env node
|
||||
// Usage: node scripts/mutate.mjs [--jobs N] [--skip-failing] [file...]
|
||||
//
|
||||
// Mutation analysis of the JS reference solutions: how many small bugs would the JS test suite notice?
|
||||
// Each mutant changes one token on one line of one file in js/solutions (e.g. `<=` -> `<`, `===` -> `!==`,
|
||||
// `+` -> `-`, `1` -> `2`, dropping a `!`). The whole JS suite runs against each mutant in a temp copy of the repo
|
||||
// whose js/src holds the solutions. A mutant is "killed" when some test fails (or the run times out), and
|
||||
// "survives" when every test still passes: either the suite has a gap, or the mutant is equivalent (it cannot
|
||||
// change behaviour, e.g. `x >= 0` -> `x > 0` where x is never 0).
|
||||
//
|
||||
// Files: paths relative to js/solutions (e.g. `game.js pong/rules.js`); default: every .js file there except the
|
||||
// browser glue (browser.js, *main.js) and todo.js.
|
||||
// Allow-list: scripts/mutate-allow.txt lists known equivalent mutants, one per line, as
|
||||
// <file>: <original line> => <mutated line> # <why it is equivalent>
|
||||
// (lines trimmed, no line number, so the list survives edits elsewhere in the file). The exact line to paste is
|
||||
// printed for each survivor.
|
||||
// Exit: 0 when every survivor is allow-listed, 1 when there are others, 2 on a setup error (including a suite that
|
||||
// already fails on the unmutated solutions; --skip-failing then leaves out the failing test files, with a warning).
|
||||
import { readFileSync, writeFileSync, cpSync, rmSync, mkdtempSync, readdirSync, existsSync } from 'node:fs'
|
||||
import { spawn, spawnSync } from 'node:child_process'
|
||||
import { join, dirname, relative, basename } from 'node:path'
|
||||
import { tmpdir, availableParallelism } from 'node:os'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const repo = join(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const allowPath = join(repo, 'scripts', 'mutate-allow.txt')
|
||||
const TIMEOUT_MS = 60000
|
||||
|
||||
const argv = process.argv.slice(2)
|
||||
const flag = (name) => argv.includes(name)
|
||||
const jobsAt = argv.indexOf('--jobs')
|
||||
const jobs = jobsAt >= 0 ? Number(argv[jobsAt + 1]) : Math.max(1, availableParallelism() - 1)
|
||||
const fileArgs = argv.filter((a, i) => !a.startsWith('--') && !(jobsAt >= 0 && i === jobsAt + 1))
|
||||
if (!Number.isInteger(jobs) || jobs < 1) { console.error('mutate: --jobs needs a positive integer'); process.exit(2) }
|
||||
|
||||
const listJs = (dir) => readdirSync(dir, { withFileTypes: true }).flatMap((e) =>
|
||||
e.isDirectory() ? listJs(join(dir, e.name)) : e.name.endsWith('.js') ? [join(dir, e.name)] : [])
|
||||
const isGlue = (f) => basename(f) === 'browser.js' || basename(f).endsWith('main.js') || basename(f) === 'todo.js'
|
||||
const solutions = join(repo, 'js', 'solutions')
|
||||
const files = fileArgs.length > 0
|
||||
? fileArgs.map((f) => f.replace(/^js\/solutions\//, ''))
|
||||
: listJs(solutions).map((f) => relative(solutions, f)).filter((f) => !isGlue(f)).sort()
|
||||
const missing = files.filter((f) => !existsSync(join(solutions, f)))
|
||||
if (missing.length > 0) { console.error(`mutate: not in js/solutions: ${missing.join(', ')}`); process.exit(2) }
|
||||
|
||||
// Single-token rules. Each match of each rule on a code line is one mutant.
|
||||
const rules = [
|
||||
[/<=/g, '<'], [/>=/g, '>'], [/(?<![<=>!])<(?![=<])/g, '<='], [/(?<![=>-])>(?![=>])/g, '>='],
|
||||
[/===/g, '!=='], [/!==/g, '==='], [/&&/g, '||'], [/\|\|/g, '&&'],
|
||||
[/ \+ /g, ' - '], [/ - /g, ' + '], [/ % /g, ' * '],
|
||||
[/\b0\b/g, '1'], [/\b1\b/g, '2'], [/\b2\b/g, '3'], [/\b3\b/g, '4'], [/\b8\b/g, '7'],
|
||||
[/!(?=[a-z(])/g, ''],
|
||||
]
|
||||
const isCodeLine = (line) => !/^\s*(\/\/|\*|\/\*|import\b)/.test(line)
|
||||
const stripComment = (line) => line.replace(/\/\/.*$/, '')
|
||||
|
||||
const mutantsOf = (file) => {
|
||||
const lines = readFileSync(join(solutions, file), 'utf8').split('\n')
|
||||
return lines.flatMap((line, li) => {
|
||||
if (!isCodeLine(line)) return []
|
||||
const code = stripComment(line)
|
||||
return rules.flatMap(([re, rep]) => [...code.matchAll(re)].map((m) => {
|
||||
const mutated = line.slice(0, m.index) + rep + line.slice(m.index + m[0].length)
|
||||
return { file, line: li + 1, original: line.trim(), mutated: mutated.trim(),
|
||||
source: [...lines.slice(0, li), mutated, ...lines.slice(li + 1)].join('\n') }
|
||||
}))
|
||||
})
|
||||
}
|
||||
const keyOf = (m) => `${m.file}: ${m.original} => ${m.mutated}`
|
||||
|
||||
// The allow-list: `key # reason`; a line without a reason is an error, so nobody adds one without thinking.
|
||||
const readAllow = () => {
|
||||
if (!existsSync(allowPath)) return new Map()
|
||||
const entries = readFileSync(allowPath, 'utf8').split('\n')
|
||||
.map((l, i) => [l.trim(), i + 1]).filter(([l]) => l !== '' && !l.startsWith('#'))
|
||||
return new Map(entries.map(([l, n]) => {
|
||||
const at = l.lastIndexOf(' # ')
|
||||
if (at < 0 || l.slice(at + 4).trim() === '') {
|
||||
console.error(`mutate: ${relative(repo, allowPath)}:${n}: missing " # reason"`); process.exit(2)
|
||||
}
|
||||
return [l.slice(0, at).trim(), l.slice(at + 4).trim()]
|
||||
}))
|
||||
}
|
||||
|
||||
// A temp copy of the repo parts the JS suite needs, with js/src replaced by the solutions.
|
||||
const makeCopy = () => {
|
||||
const work = mkdtempSync(join(tmpdir(), 'pure-snake-mutate-'))
|
||||
cpSync(join(repo, 'js'), join(work, 'js'), { recursive: true, filter: (s) => !s.includes('node_modules') })
|
||||
cpSync(join(repo, 'fixtures'), join(work, 'fixtures'), { recursive: true })
|
||||
rmSync(join(work, 'js', 'src'), { recursive: true })
|
||||
cpSync(join(work, 'js', 'solutions'), join(work, 'js', 'src'), { recursive: true })
|
||||
return work
|
||||
}
|
||||
|
||||
const runSuite = (work, args) => new Promise((resolve) => {
|
||||
const child = spawn('node', ['--test', '--test-concurrency=1', ...args], { cwd: join(work, 'js'), stdio: 'ignore' })
|
||||
const timer = setTimeout(() => { child.kill('SIGKILL'); resolve('timeout') }, TIMEOUT_MS)
|
||||
child.on('exit', (code) => { clearTimeout(timer); resolve(code === 0 ? 'pass' : 'fail') })
|
||||
})
|
||||
|
||||
// Baseline: the suite must pass on the unmutated solutions, or every mutant would look "killed".
|
||||
// With --skip-failing, tests that already fail are skipped by name (a file that fails to load is left out whole).
|
||||
const baseline = makeCopy()
|
||||
const allTests = readdirSync(join(baseline, 'js', 'test')).filter((f) => f.endsWith('.test.js')).sort()
|
||||
.map((f) => `test/${f}`)
|
||||
const failingIn = (t) => {
|
||||
const r = spawnSync('node', ['--test', '--test-reporter=tap', t],
|
||||
{ cwd: join(baseline, 'js'), encoding: 'utf8', timeout: TIMEOUT_MS })
|
||||
if (r.status === 0) return null
|
||||
const names = [...r.stdout.matchAll(/^\s*not ok \d+ - (.*)$/gm)].map((m) => m[1].trim()).filter((n) => n !== t)
|
||||
return { file: t, names }
|
||||
}
|
||||
const failing = allTests.map(failingIn).filter((f) => f !== null)
|
||||
rmSync(baseline, { recursive: true })
|
||||
const skipFiles = failing.filter((f) => f.names.length === 0).map((f) => f.file)
|
||||
const skipNames = failing.flatMap((f) => f.names)
|
||||
if (failing.length > 0) {
|
||||
console.error('mutate: the suite already fails on the unmutated solutions:')
|
||||
failing.forEach((f) => console.error(` ${f.file}: ${f.names.length === 0 ? '(the whole file)' : f.names.join('; ')}`))
|
||||
if (!flag('--skip-failing')) { console.error('mutate: fix them first, or pass --skip-failing to leave them out'); process.exit(2) }
|
||||
console.error('mutate: WARNING: running without them; some mutants may survive only because of that')
|
||||
}
|
||||
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
const suiteArgs = [...skipNames.map((n) => `--test-skip-pattern=^${escapeRe(n)}$`),
|
||||
...allTests.filter((t) => !skipFiles.includes(t))]
|
||||
const tests = allTests.filter((t) => !skipFiles.includes(t))
|
||||
|
||||
const allow = readAllow() // read before the long run, so a malformed line fails fast
|
||||
const mutants = files.flatMap(mutantsOf)
|
||||
console.log(`mutate: ${mutants.length} mutants in ${files.length} files, ${tests.length} test files, ${jobs} jobs`)
|
||||
|
||||
// A pool of workers, each with its own copy; a worker writes the mutant, runs the suite, restores the file.
|
||||
const results = new Array(mutants.length)
|
||||
const worker = async (queue) => {
|
||||
const work = makeCopy()
|
||||
try {
|
||||
for (const i of queue) {
|
||||
const m = mutants[i]
|
||||
const path = join(work, 'js', 'src', m.file)
|
||||
const original = readFileSync(path, 'utf8')
|
||||
writeFileSync(path, m.source)
|
||||
results[i] = await runSuite(work, suiteArgs)
|
||||
writeFileSync(path, original)
|
||||
}
|
||||
} finally { rmSync(work, { recursive: true, force: true }) }
|
||||
}
|
||||
const queues = Array.from({ length: Math.min(jobs, mutants.length) }, (_, w) =>
|
||||
mutants.map((_, i) => i).filter((i) => i % jobs === w))
|
||||
await Promise.all(queues.map(worker))
|
||||
|
||||
const survivors = mutants.filter((_, i) => results[i] === 'pass')
|
||||
const timeouts = results.filter((r) => r === 'timeout').length
|
||||
const unlisted = survivors.filter((m) => !allow.has(keyOf(m)))
|
||||
const listed = survivors.filter((m) => allow.has(keyOf(m)))
|
||||
const survivorKeys = new Set(survivors.map(keyOf))
|
||||
const stale = [...allow.keys()].filter((k) => files.includes(k.slice(0, k.indexOf(': '))) && !survivorKeys.has(k))
|
||||
|
||||
console.log(`killed ${mutants.length - survivors.length} (${timeouts} by timeout), survived ${survivors.length}` +
|
||||
` (${listed.length} allow-listed as equivalent, ${unlisted.length} not)`)
|
||||
if (unlisted.length > 0) {
|
||||
console.log('\nSurvivors not in scripts/mutate-allow.txt (a test gap, or an equivalent mutant to allow-list with a reason):')
|
||||
unlisted.forEach((m) => console.log(` js/solutions/${m.file}:${m.line}\n - ${m.original}\n + ${m.mutated}\n allow-list line: ${keyOf(m)} # <reason>`))
|
||||
}
|
||||
if (stale.length > 0) {
|
||||
console.log('\nAllow-list entries for these files that no longer occur as survivors (remove or update them):')
|
||||
stale.forEach((k) => console.log(` ${k}`))
|
||||
}
|
||||
process.exit(unlisted.length > 0 ? 1 : failing.length > 0 ? 2 : 0)
|
||||
Reference in new issue
Block a user