diff --git a/internal/check-link-rot-sample.mjs b/internal/check-link-rot-sample.mjs index d667c8d6d..4e389454a 100644 --- a/internal/check-link-rot-sample.mjs +++ b/internal/check-link-rot-sample.mjs @@ -6,24 +6,48 @@ * keep offering it indefinitely, still labelled "link checked". Clause L of the coverage gate * guards the freshness of the RECORD; it cannot notice that the web moved underneath it. * - * WHY A SAMPLE. 8,285 verified links is a lot of traffic to aim at seven ministries for a - * maintenance check. A stratified sample — up to 25 per host, taken at a fixed stride so the + * WHY A SAMPLE. 8,394 verified links is a lot of traffic to aim at a handful of ministries for + * a maintenance check. A stratified sample — up to 25 per host, taken at a fixed stride so the * same run is repeatable — is enough to tell "nothing is rotting" from "a publisher has * reorganised", which is the decision this check exists to inform. A full re-verify is * scripts/verify-gospel-links.mjs and is the right tool once this finds something. * - * ONLY HOSTS WHERE A STATUS MEANS SOMETHING — the part worth keeping. - * Four of the eleven hosts NEVER 404: - * - jesusfilm.org and live.bible.is return an identical SPA shell for any path; - * - knowgod.com does the same; - * - fivefish.org answers a nonsense path with a plausible geo-located page, HTTP 200. - * Re-checking those by status would report perfect health forever, which is worse than not - * checking them at all, because it would look like evidence. They are excluded here and are - * verified by their publisher's API or by page title in the main verifier instead. + * ───────────────────────────────────────────────────────────────────────────────────────── + * WHAT THIS FILE USED TO SAY, AND WHY IT WAS WRONG. The first version excluded four hosts on + * the grounds that they "NEVER 404" and so could not be re-checked by status. That belief was + * inherited, not measured, and it quietly exempted 5,760 of 8,394 shipped links — 69% of the + * corpus, including the two largest hosts on the map. Measured on 2026-09-25, on the exact + * URL SHAPE the map actually ships: * - * IT IS SHOWN TO BE ABLE TO FAIL. Three URLs that must not exist are requested in band every - * run. If any of them answers 2xx the check reports that it proved nothing and exits non-zero - * rather than reporting a pass — the same rule applied to every other gate in this bundle. + * jesusfilm.org /watch/jesus/ real 200 · three different bogus slugs ALL 404, + * by HEAD as well as GET. It does 404. The exclusion was simply false, and + * 2,049 links sat outside every check because of it. + * live.bible.is /bible/ always 200, but a bogus fileset returns an empty + * SPA shell of 7,860–7,865 bytes while real filesets return 13.5–85 KB. + * Status is useless; SIZE separates them cleanly, at better than 1.7×. + * This is not a new trick — scripts/fetch-fcbh-index.mjs already builds the + * index this way and records `shellBytes` for it. Only this check had not + * learned it. 2,636 links. + * fivefish.org / always 200; found and not-found differ by page + * TITLE, which is how scripts/fetch-grn-index.mjs classifies them. Real, + * but their robots.txt asks Crawl-delay: 10, so a 25-link sample is four + * minutes of polite waiting and belongs in its own slower band, not here. + * 982 links, still unchecked, deliberately and on the record. + * knowgod.com / byte-identical 3,864-byte "Know God" page for a real + * path and a nonsense one. This one really is unverifiable by fetching it. + * 94 links, and saying so is the honest answer. + * + * The lesson is the one this repo keeps relearning: a claim about the outside world decays, + * and an exclusion is a claim. Each host below carries the measurement that justifies its + * band, so the next person can re-measure rather than re-inherit. + * ───────────────────────────────────────────────────────────────────────────────────────── + * + * IT IS SHOWN TO BE ABLE TO FAIL. Every band runs its controls in band, every run: + * - status band: URLs that must not exist, one per host shape. Any 2xx and the check + * reports that it proved nothing and exits non-zero rather than reporting a pass. + * - shell band: a bogus fileset must read as shell (or the check cannot see death) AND a + * fixed real fileset must read as substantive (or the check would call the whole host + * dead the day its page shape changes). */ import { readFileSync } from 'node:fs' import { resolve, dirname } from 'node:path' @@ -36,34 +60,61 @@ const TABLE = resolve(REPO, 'data/tables/gospel-link-verification.json') const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' -/** Hosts whose HTTP status is real evidence. Everything else lies; see the header. */ +/** Hosts whose HTTP status is real evidence. Measured, not assumed — see the header. */ const STATUS_MEANINGFUL = new Set([ 'ebible.org', 'alkitab.pw', 'png.bible', 'git.door43.org', 'scriptureearth.org', 'zume.training', 'arc.gt', + 'jesusfilm.org', 'www.jesusfilm.org', ]) +/** Hosts that answer 200 for anything but whose EMPTY page has a measurable size. */ +const SHELL_MEASURED = { + 'live.bible.is': { + /* Measured at run time, never hard-coded: the shell grows when they ship a bundle. */ + shellProbe: 'https://live.bible.is/bible/ZZZQQQ', + /* Must come back substantive, or the size rule has stopped separating anything. */ + livingControl: 'https://live.bible.is/bible/ENGKJV', + /* 7,860 shell vs 13,547 smallest real page on 2026-09-25 — 1.72×. 1.25 is slack. */ + aliveRatio: 1.25, + }, +} + +/** Hosts left out, with the reason, so the exclusion stays a measurement and not a habit. */ +const UNCHECKABLE = { + 'fivefish.org': 'found/not-found differ by title only, and Crawl-delay: 10 needs its own band', + 'knowgod.com': 'byte-identical page for a real and a nonsense path — unverifiable by fetch', +} + /** Must not exist. If one of these passes, the check cannot see death and says so. */ const CONTROLS = [ 'https://ebible.org/zzz-no-such-translation-xyzzy/', 'https://scriptureearth.org/data/zzz/PDF/nope.pdf', 'https://git.door43.org/unfoldingWord/no-such-repo-xyzzy', + /* The shipped shape, not a plausible-looking one: /watch/jesus/. */ + 'https://jesusfilm.org/watch/jesus/zzz-no-such-language-xyzzy', ] const PER_HOST = Number(process.env.ROT_PER_HOST || 25) -async function status(url) { +async function get(url, method = 'GET') { try { const r = await fetch(url, { + method, headers: { 'user-agent': UA }, redirect: 'follow', signal: AbortSignal.timeout(20000), }) - return r.status + const body = method === 'GET' ? await r.text() : '' + return { status: r.status, len: body.length } } catch { - return 0 + return { status: 0, len: 0 } } } +/* A 808 KB film page times 25 is 20 MB of somebody else's bandwidth to learn one number that + HEAD already carries. Status hosts are asked with HEAD; the shell band needs the body. */ +const status = (url) => get(url, 'HEAD').then((r) => r.status) + let table try { table = JSON.parse(readFileSync(TABLE, 'utf8')) } catch { console.error('LINK-ROT: NOT MEASURED — no verification table to sample.') @@ -75,10 +126,11 @@ const verified = Object.entries(table.links || {}) .map(([u]) => u) const byHost = new Map() +let skipped = 0 for (const u of verified) { let h try { h = new URL(u).host } catch { continue } - if (!STATUS_MEANINGFUL.has(h)) continue + if (!STATUS_MEANINGFUL.has(h) && !SHELL_MEASURED[h]) { skipped++; continue } if (!byHost.has(h)) byHost.set(h, []) byHost.get(h).push(u) } @@ -86,15 +138,15 @@ for (const u of verified) { /* Fixed stride, sorted input: the same sample every run, so a change is a change in the web rather than a change in which links we happened to pick. */ const sample = [] -for (const [, urls] of [...byHost].sort((a, b) => a[0].localeCompare(b[0]))) { +for (const [host, urls] of [...byHost].sort((a, b) => a[0].localeCompare(b[0]))) { urls.sort() const step = Math.max(1, Math.floor(urls.length / PER_HOST)) - const take = [] - for (let i = 0; i < urls.length && take.length < PER_HOST; i += step) take.push(urls[i]) - sample.push(...take) + for (let i = 0; i < urls.length && sample.filter((s) => s.host === host).length < PER_HOST; i += step) { + sample.push({ host, url: urls[i] }) + } } -/* ── controls first ─────────────────────────────────────────────────────────────────────── */ +/* ── controls: the status band ──────────────────────────────────────────────────────────── */ const ctl = [] for (const u of CONTROLS) ctl.push([u, await status(u)]) const blind = ctl.filter(([, s]) => s >= 200 && s < 300) @@ -105,21 +157,55 @@ if (blind.length) { console.error(' trusting any pass it reports.') process.exit(1) } -console.log(`LINK-ROT: controls pass — ${ctl.length} impossible URL(s), none answered 2xx.`) +console.log(`LINK-ROT: status controls pass — ${ctl.length} impossible URL(s), none answered 2xx.`) +/* ── controls: the shell band, both directions ─────────────────────────────────────────── */ +const shells = {} +for (const [host, cfg] of Object.entries(SHELL_MEASURED)) { + if (!byHost.has(host)) continue + const empty = await get(cfg.shellProbe) + const living = await get(cfg.livingControl) + const floor = empty.len * cfg.aliveRatio + if (!empty.len) { + console.error(`LINK-ROT: PROVED NOTHING — ${host} shell probe returned nothing to measure.`) + process.exit(1) + } + if (living.len <= floor) { + console.error( + `LINK-ROT: PROVED NOTHING — ${host} answered its KNOWN-GOOD control with ${living.len}b, ` + + `no bigger than the ${empty.len}b empty shell.`, + ) + console.error(' The size rule has stopped separating a real page from an empty one, so a') + console.error(' pass here would mean nothing and a fail would libel every link on the host.') + process.exit(1) + } + shells[host] = floor + console.log( + `LINK-ROT: ${host} shell measured at ${empty.len}b; known-good page ${living.len}b ` + + `(${(living.len / empty.len).toFixed(1)}×). Alive means over ${Math.round(floor)}b.`, + ) +} + +/* ── the sample ─────────────────────────────────────────────────────────────────────────── */ const dead = [] -for (const u of sample) { - const s = await status(u) - if (!(s >= 200 && s < 300)) dead.push([u, s]) +for (const { host, url } of sample) { + if (shells[host] !== undefined) { + const r = await get(url) + if (r.len <= shells[host]) dead.push([url, `${r.len}b — empty shell`]) + continue + } + const s = await status(url) + if (!(s >= 200 && s < 300)) dead.push([url, s || 'no response']) } console.log( `LINK-ROT: re-checked ${sample.length} verified link(s) across ${byHost.size} host(s) ` + - `where a status is meaningful (${verified.length - sample.length} not sampled).`, + `(${skipped} link(s) on hosts nothing can check: ` + + `${Object.keys(UNCHECKABLE).join(', ')}).`, ) if (dead.length) { console.error(`LINK-ROT: FAIL — ${dead.length} link(s) we call "checked" no longer resolve:`) - for (const [u, s] of dead.slice(0, 10)) console.error(` ${s || 'no response'} ${u}`) + for (const [u, why] of dead.slice(0, 10)) console.error(` ${why} ${u}`) console.error(' Re-run scripts/verify-gospel-links.mjs to re-test the whole corpus; the') console.error(' build drops anything it proves dead.') process.exit(1)