chore: link checker
Zero-dependency scripts/check-links.mjs: HEAD/GET every http(s) link in the course Markdown (skipping localhost), plus relative links and anchors. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
a23fd07906
commit
1a80d57499
@@ -0,0 +1,148 @@
|
||||
// Link checker for the course's Markdown. Tooling, not course code. Zero dependencies (Node 22+).
|
||||
// Usage: node scripts/check-links.mjs [--offline]
|
||||
// - every http(s) link (except localhost) in every *.md file (except docs/superpowers/, .superpowers/, node_modules/, build output)
|
||||
// is requested: HEAD first, GET as a fallback, redirects followed, with a timeout and a small concurrency limit;
|
||||
// - every relative Markdown link ([text](path) / [text](path#anchor)) must point to a file or folder in the repo;
|
||||
// anchors into Markdown files must match a heading of that file.
|
||||
// Prints a table of failures and exits 1 if there are any, 0 otherwise. --offline skips the http(s) checks.
|
||||
import { readdir, readFile, stat } from 'node:fs/promises'
|
||||
import { dirname, join, relative, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const root = resolve(fileURLToPath(new URL('..', import.meta.url)))
|
||||
const offline = process.argv.includes('--offline')
|
||||
const SKIP_DIRS = new Set(['.git', 'node_modules', '.superpowers', 'bin', 'obj', '.scala-build', '.bsp', 'dist', 'fable_modules'])
|
||||
const SKIP_PATHS = [join('docs', 'superpowers')]
|
||||
const TIMEOUT_MS = 20000
|
||||
const CONCURRENCY = 6
|
||||
const USER_AGENT =
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'
|
||||
|
||||
const walk = async (dir) => {
|
||||
const entries = await readdir(dir, { withFileTypes: true })
|
||||
const nested = await Promise.all(entries.map(async (e) => {
|
||||
const full = join(dir, e.name)
|
||||
const rel = relative(root, full)
|
||||
if (e.isDirectory()) return SKIP_DIRS.has(e.name) || SKIP_PATHS.includes(rel) ? [] : walk(full)
|
||||
return e.name.endsWith('.md') ? [full] : []
|
||||
}))
|
||||
return nested.flat()
|
||||
}
|
||||
|
||||
// Remove fenced code blocks and inline code: links inside code are examples, not references.
|
||||
const stripFences = (md) => md.replace(/^(```|~~~)[\s\S]*?^\1/gm, '')
|
||||
const stripCode = (md) => stripFences(md).replace(/`[^`\n]*`/g, '')
|
||||
|
||||
// GitHub-style heading slugs: lower case, drop punctuation except - and _, spaces become -.
|
||||
const slug = (h) => h.trim().toLowerCase().replace(/<[^>]*>/g, '').replace(/[^\p{L}\p{N}\s_-]/gu, '').replace(/\s/g, '-')
|
||||
const headingSlugs = (md) => {
|
||||
const seen = new Map()
|
||||
return new Set(stripFences(md).split('\n').filter((l) => /^#{1,6}\s/.test(l)).map((l) => {
|
||||
const base = slug(l.replace(/^#+\s*/, '').replace(/`/g, ''))
|
||||
const n = seen.get(base) ?? 0
|
||||
seen.set(base, n + 1)
|
||||
return n === 0 ? base : `${base}-${n}`
|
||||
}))
|
||||
}
|
||||
|
||||
// Links to the local dev server (http://localhost:8000/js/) are instructions for the reader, not references.
|
||||
const isLocal = (u) => /^https?:\/\/(localhost|127\.0\.0\.1)(:\d+)?(\/|$)/i.test(u)
|
||||
|
||||
const extract = (md) => {
|
||||
const text = stripCode(md)
|
||||
const http = new Set()
|
||||
const rel = new Set()
|
||||
// [text](target) and [text](target "title"); target may be wrapped in <>.
|
||||
for (const m of text.matchAll(/\]\(\s*<?([^)\s>]+)>?(?:\s+"[^"]*")?\s*\)/g)) {
|
||||
const t = m[1]
|
||||
if (/^https?:\/\//i.test(t)) http.add(t)
|
||||
else if (!/^(mailto:|tel:|data:)/i.test(t)) rel.add(t)
|
||||
}
|
||||
// Bare URLs and autolinks <https://...>.
|
||||
for (const m of text.matchAll(/(?<![(<\w])https?:\/\/[^\s<>()\]`"']+/g)) http.add(m[0].replace(/[.,;:!?]+$/, ''))
|
||||
for (const m of text.matchAll(/<(https?:\/\/[^>\s]+)>/g)) http.add(m[1])
|
||||
return { http, rel }
|
||||
}
|
||||
|
||||
const checkUrl = async (url) => {
|
||||
const attempt = async (method) => {
|
||||
const ctrl = new AbortController()
|
||||
const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS)
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method, redirect: 'follow', signal: ctrl.signal,
|
||||
headers: { 'user-agent': USER_AGENT, accept: 'text/html,application/xhtml+xml,application/pdf,*/*;q=0.8', 'accept-language': 'en' },
|
||||
})
|
||||
if (method === 'GET') res.body?.cancel().catch(() => {})
|
||||
return { status: res.status, ok: res.ok }
|
||||
} catch (e) {
|
||||
return { status: 0, ok: false, error: e.name === 'AbortError' ? 'timeout' : (e.cause?.code ?? e.message) }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
const head = await attempt('HEAD')
|
||||
if (head.ok) return head
|
||||
const get = await attempt('GET')
|
||||
if (get.ok || get.status !== 0) return get
|
||||
// One retry for network-level failures (flaky DNS, resets).
|
||||
return attempt('GET')
|
||||
}
|
||||
|
||||
const pool = async (items, n, f) => {
|
||||
const results = new Array(items.length)
|
||||
const worker = async (start) => {
|
||||
for (let i = start; i < items.length; i += n) results[i] = await f(items[i])
|
||||
}
|
||||
await Promise.all(Array.from({ length: n }, (_, k) => worker(k)))
|
||||
return results
|
||||
}
|
||||
|
||||
const files = (await walk(root)).sort()
|
||||
const failures = []
|
||||
const urls = new Map() // url -> [files]
|
||||
const slugCache = new Map()
|
||||
|
||||
for (const file of files) {
|
||||
const md = await readFile(file, 'utf8')
|
||||
const { http, rel } = extract(md)
|
||||
for (const u of http) if (!isLocal(u)) urls.set(u, [...(urls.get(u) ?? []), relative(root, file)])
|
||||
for (const r of rel) {
|
||||
const [pathPart, anchor] = r.split('#')
|
||||
const target = pathPart === '' ? file : resolve(dirname(file), decodeURIComponent(pathPart))
|
||||
const where = relative(root, file)
|
||||
if (!target.startsWith(root)) { failures.push({ where, link: r, problem: 'points outside the repository' }); continue }
|
||||
const st = await stat(target).catch(() => null)
|
||||
if (!st) { failures.push({ where, link: r, problem: 'file not found' }); continue }
|
||||
if (anchor && st.isFile() && target.endsWith('.md')) {
|
||||
if (!slugCache.has(target)) slugCache.set(target, headingSlugs(await readFile(target, 'utf8')))
|
||||
if (!slugCache.get(target).has(anchor.toLowerCase())) failures.push({ where, link: r, problem: 'no such heading' })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const relCount = failures.length
|
||||
let httpChecked = 0
|
||||
if (!offline) {
|
||||
const list = [...urls.keys()].sort()
|
||||
const results = await pool(list, CONCURRENCY, checkUrl)
|
||||
httpChecked = list.length
|
||||
list.forEach((u, i) => {
|
||||
const r = results[i]
|
||||
if (!r.ok) failures.push({ where: urls.get(u).join(', '), link: u, problem: r.status ? `HTTP ${r.status}` : r.error })
|
||||
})
|
||||
}
|
||||
|
||||
console.log(`Checked ${files.length} Markdown files: ${httpChecked} distinct http(s) links${offline ? ' (skipped: --offline)' : ''}, relative links ${relCount ? 'with problems' : 'all ok'}.`)
|
||||
if (failures.length === 0) {
|
||||
console.log('All links OK.')
|
||||
process.exit(0)
|
||||
}
|
||||
const rows = [['Problem', 'Link', 'Found in'], ...failures.map((f) => [f.problem, f.link, f.where])]
|
||||
const widths = [0, 1].map((c) => Math.min(70, Math.max(...rows.map((r) => r[c].length))))
|
||||
const line = (r) => `${r[0].padEnd(widths[0])} ${r[1].padEnd(widths[1])} ${r[2]}`
|
||||
console.log(`\n${failures.length} failing link(s):\n`)
|
||||
console.log(line(rows[0]))
|
||||
console.log(line(['-'.repeat(widths[0]), '-'.repeat(widths[1]), '--------']))
|
||||
rows.slice(1).forEach((r) => console.log(line(r)))
|
||||
process.exit(1)
|
||||
Reference in New Issue
Block a user