mangabaka-seeder/src/core/history/wayback.ts

import { formatMonth, mergeByUrl } from "../covers";
import type { CoverCandidate, Granularity, SourceSeries } from "../model";
import type { Source } from "../sources/source";
import type { Transport } from "../transport";

export const WAYBACK = "https://web.archive.org";

export interface CdxRow {
  timestamp: string;
  original: string;
}

export interface HistoryOptions {
  every: Granularity;
  transport: Transport;
  warn(message: string): void;
  onProgress?(done: number, total: number): void;
  signal?: AbortSignal;
}

/** Every month with a capture (collapse=timestamp:6) — one cheap request per prefix (spec §4.3). */
export function cdxUrl(prefix: string): string {
  const q = new URLSearchParams({
    url: prefix, matchType: "prefix", filter: "statuscode:200", collapse: "timestamp:6", fl: "timestamp,original",
  });
  return `${WAYBACK}/cdx/search/cdx?${q}`;
}

export function parseCdx(text: string): CdxRow[] {
  const rows: CdxRow[] = [];
  for (const line of text.split("\n")) {
    const [timestamp, original] = line.trim().split(/\s+/);
    if (timestamp && original && /^\d{14}$/.test(timestamp)) rows.push({ timestamp, original });
  }
  return rows;
}

const monthIndex = (ts: string) => Number(ts.slice(0, 4)) * 12 + Number(ts.slice(4, 6)) - 1;
const hostIs = (row: CdxRow, host: string) => {
  try {
    return new URL(row.original).hostname === host;
  } catch {
    return false;
  }
};

/** Buckets of `every` months from the first capture; first capture per bucket, preferring `preferHost`. */
export function pickSnapshots(rows: readonly CdxRow[], every: Granularity, preferHost: string): CdxRow[] {
  const sorted = [...rows].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
  if (!sorted.length) return [];
  const first = monthIndex(sorted[0]!.timestamp);
  const buckets = new Map<number, CdxRow>();
  for (const row of sorted) {
    const bucket = Math.floor((monthIndex(row.timestamp) - first) / every);
    const current = buckets.get(bucket);
    if (!current || (!hostIs(current, preferHost) && hostIs(row, preferHost))) buckets.set(bucket, row);
  }
  return [...buckets.values()].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
}

export const snapshotUrl = (row: CdxRow): string => `${WAYBACK}/web/${row.timestamp}id_/${row.original}`;
export const archivedImageUrl = (timestamp: string, originalImageUrl: string): string =>
  `${WAYBACK}/web/${timestamp}id_/${originalImageUrl}`;

const message = (err: unknown) => (err instanceof Error ? err.message : String(err));

export async function waybackHistory(source: Source, series: SourceSeries, opts: HistoryOptions): Promise<CoverCandidate[]> {
  if (!source.extractCovers) return [];
  const rows: CdxRow[] = [];
  for (const prefix of series.archiveUrls) {
    try {
      const res = await opts.transport.fetch(cdxUrl(prefix), { signal: opts.signal });
      if (!res.ok) {
        await res.body?.cancel();
        opts.warn(`The Wayback index didn't answer (HTTP ${res.status}); no history from ${prefix}.`);
        continue;
      }
      // matchType=prefix on title_no=95 also matches title_no=950 (spec §4.3).
      rows.push(...parseCdx(await res.text()).filter((r) => source.isSameSeries(r.original, series)));
    } catch (err) {
      if (opts.signal?.aborted) return [];
      opts.warn(`The Wayback index didn't answer (${message(err)}); no history from ${prefix}.`);
    }
  }

  const preferHost = series.archiveUrls[0]?.split("/")[0] ?? "";
  const picks = pickSnapshots(rows, opts.every, preferHost);
  const found: CoverCandidate[] = [];
  let done = 0;
  for (const row of picks) {
    if (opts.signal?.aborted) break;
    let res: Response;
    try {
      res = await opts.transport.fetch(snapshotUrl(row), { signal: opts.signal });
    } catch (err) {
      if (opts.signal?.aborted) break;
      opts.warn(`Wayback stopped answering (${message(err)}); history stops at ${formatMonth(row.timestamp)}.`);
      break;
    }
    if (res.status === 429 || res.status >= 500) {
      await res.body?.cancel();
      opts.warn(`Wayback is refusing requests (HTTP ${res.status}); history stops at ${formatMonth(row.timestamp)}.`);
      break;
    }
    if (res.ok) {
      try {
        const pageUrl = new URL(row.original);
        for (const c of source.extractCovers(await res.text(), pageUrl)) {
          const candidate: CoverCandidate = { ...c, current: false, seenAt: row.timestamp, lastSeenAt: row.timestamp };
          if (c.originalUrl) candidate.archivedUrl = archivedImageUrl(row.timestamp, c.originalUrl);
          found.push(candidate);
        }
      } catch (err) {
        // One unreadable snapshot never costs the rest of the history (spec §4.3).
        if (opts.signal?.aborted) break;
        opts.warn(`Snapshot ${formatMonth(row.timestamp)} couldn't be read (${message(err)}).`);
      }
    } else {
      await res.body?.cancel();
      opts.warn(`Snapshot ${formatMonth(row.timestamp)} is unavailable (HTTP ${res.status}).`);
    }
    opts.onProgress?.(++done, picks.length);
  }
  return mergeByUrl(found);
}