systemic-game-design-course/tools/check-links.mjs

#!/usr/bin/env node
// Report-only external link check. Lists every external URL found in course/,
// docs/research/ and README.md with its HTTP status. Always exits 0.
//
//   node tools/check-links.mjs            (or: npm run check:links)
//
// Reddit URLs are listed as "unverifiable": reddit blocks automated fetches,
// and the course already excludes reddit as a source.
import { readdirSync, readFileSync, statSync } from "node:fs";
import { join, relative, extname, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const SKIP_DIRS = new Set(["node_modules", "target", "dist", ".astro", ".git"]);
const EXTS = new Set([".md", ".json", ".rs", ".ts", ".mjs"]);
const TIMEOUT_MS = 20000;
const CONCURRENCY = 6;
const HOST_GAP_MS = 1000;

/** Extract http(s) URLs from text. Handles markdown links and trailing punctuation. */
export function extractUrls(text) {
  const out = [];
  for (const m of text.matchAll(/https?:\/\/[^\s<>"'`\]]+/g)) {
    let url = m[0];
    // Drop trailing sentence punctuation.
    url = url.replace(/[.,;:!?*]+$/, "");
    // Drop closing parens that aren't balanced inside the URL (markdown "(url)").
    while (url.endsWith(")") && count(url, "(") < count(url, ")")) {
      url = url.slice(0, -1).replace(/[.,;:!?*]+$/, "");
    }
    out.push(url);
  }
  return out;
}

const count = (s, ch) => s.split(ch).length - 1;

function* walk(path) {
  const st = statSync(path);
  if (st.isFile()) {
    if (EXTS.has(extname(path))) yield path;
    return;
  }
  for (const e of readdirSync(path)) {
    if (SKIP_DIRS.has(e)) continue;
    yield* walk(join(path, e));
  }
}

/** Hostname of `url`, or null when `url` does not parse (reported as "invalid URL"). */
export function hostOf(url) {
  try {
    return new URL(url).hostname;
  } catch {
    return null;
  }
}

export function classifyUnverifiable(url) {
  const host = hostOf(url);
  if (host === null) return "invalid URL";
  if (/(^|\.)reddit\.com$/.test(host)) return "unverifiable (reddit blocks fetch tools; excluded as a source)";
  return null;
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function get(url, method) {
  return fetch(url, { method, redirect: "follow", signal: AbortSignal.timeout(TIMEOUT_MS) });
}

// Fandom answers 403 to scripts. Its public MediaWiki API still says whether the page exists.
async function fandomExists(url) {
  const u = new URL(url);
  const m = u.hostname.endsWith(".fandom.com") && u.pathname.match(/^\/wiki\/(.+)$/);
  if (!m) return null;
  const api = `${u.origin}/api.php?action=query&format=json&titles=${m[1]}`;
  try {
    const pages = (await (await get(api, "GET")).json()).query?.pages ?? {};
    return Object.keys(pages).some((k) => k !== "-1") ? "page exists per MediaWiki API" : "page MISSING per MediaWiki API";
  } catch {
    return null;
  }
}

async function status(url) {
  try {
    let r = await get(url, "HEAD");
    if (r.status >= 400) r = await get(url, "GET");
    for (let tries = 0; r.status === 429 && tries < 3; tries++) {
      const wait = Math.min(Number(r.headers.get("retry-after")) || 10, 30);
      await sleep(wait * 1000);
      r = await get(url, "GET");
    }
    if (r.status === 403 || r.status === 402) {
      const note = await fandomExists(url);
      if (note) return `${r.status} (${note})`;
    }
    return String(r.status);
  } catch (e) {
    return `error (${e.cause?.code ?? e.name})`;
  }
}

async function main() {
  const root = join(fileURLToPath(import.meta.url), "..", "..");
  const seen = new Map(); // url -> Set(file)
  for (const top of ["course", "docs/research", "README.md"]) {
    for (const f of walk(join(root, top))) {
      for (const url of extractUrls(readFileSync(f, "utf8"))) {
        if (!seen.has(url)) seen.set(url, new Set());
        seen.get(url).add(relative(root, f));
      }
    }
  }
  const urls = [...seen.keys()].sort();
  // One request at a time per host (wikis rate-limit), up to CONCURRENCY hosts at once.
  const byHost = new Map();
  for (const url of urls) {
    const host = hostOf(url) ?? "(invalid)";
    if (!byHost.has(host)) byHost.set(host, []);
    byHost.get(host).push(url);
  }
  const queues = [...byHost.values()];
  const results = new Map();
  await Promise.all(
    Array.from({ length: CONCURRENCY }, async () => {
      for (let q = queues.shift(); q; q = queues.shift()) {
        for (const url of q) {
          results.set(url, classifyUnverifiable(url) ?? (await status(url)));
          await sleep(HOST_GAP_MS);
        }
      }
    }),
  );
  const tally = {};
  for (const url of urls) {
    const s = results.get(url);
    const key = /^\d/.test(s) ? s : s.split(" ")[0];
    tally[key] = (tally[key] ?? 0) + 1;
    console.log(`${s}\t${url}\t${[...seen.get(url)].join(", ")}`);
  }
  console.log(`\n${urls.length} unique external URLs. By status: ${JSON.stringify(tally)}`);
  console.log("Report only: exit code is always 0.");
}

if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
  await main();
}