262 lines
14 KiB
JavaScript
262 lines
14 KiB
JavaScript
/**
|
||
* check-link-rot-sample.mjs — do the links we already verified still work?
|
||
*
|
||
* THE RISK. The verification table records that a URL resolved ON THE DAY IT WAS CHECKED.
|
||
* Nothing re-tests a link once it passes, so a publisher can retire a page and the map will
|
||
* keep offering it indefinitely, still labelled "link checked". Clause L of the coverage gate
|
||
* guards the freshness of the RECORD; it cannot notice that the web moved underneath it.
|
||
*
|
||
* WHY A SAMPLE. 8,394 verified links is a lot of traffic to aim at a handful of ministries for
|
||
* a maintenance check. A stratified sample — up to 25 per host, taken at a fixed stride so the
|
||
* same run is repeatable — is enough to tell "nothing is rotting" from "a publisher has
|
||
* reorganised", which is the decision this check exists to inform. A full re-verify is
|
||
* scripts/verify-gospel-links.mjs and is the right tool once this finds something.
|
||
*
|
||
* ─────────────────────────────────────────────────────────────────────────────────────────
|
||
* WHAT THIS FILE USED TO SAY, AND WHY IT WAS WRONG. The first version excluded four hosts on
|
||
* the grounds that they "NEVER 404" and so could not be re-checked by status. That belief was
|
||
* inherited, not measured, and it quietly exempted 5,760 of 8,394 shipped links — 69% of the
|
||
* corpus, including the two largest hosts on the map — while the bundle's own verifier had
|
||
* ALREADY solved three of the four. Measured on 2026-09-25, on the exact URL SHAPE the map
|
||
* actually ships, and reconciled against scripts/verify-gospel-links.mjs, which is the
|
||
* authority on per-host method and should be read before adding a host here:
|
||
*
|
||
* jesusfilm.org /watch/jesus/<slug> real 200 · three different bogus slugs ALL 404,
|
||
* by HEAD as well as GET. It does 404. The exclusion here was inherited
|
||
* from the OLD /watch/?languageId=N shape, which really was byte-identical
|
||
* for a valid and a bogus id — scripts/verify-gospel-links.mjs migrated to
|
||
* the Arclight shareUrl path and has been checking these for some time.
|
||
* Only this file never learned. 2,049 links.
|
||
* live.bible.is /bible/<filesetId> the page always answers 200, so it cannot be judged
|
||
* at all. Its API can: /api/bibles/<filesetId> returns 200 with the fileset
|
||
* for a real id and a 75-byte 404 for a bogus one. Same discriminator the
|
||
* verifier uses, same in-band abort. 2,636 links.
|
||
* fivefish.org /<iso639-3> always 200, but the REDIRECT discriminates and
|
||
* does it on a HEAD: a published code lands on its numbered language page
|
||
* (/cpx -> /421?language=Chinese,%20Puxian), one they do not carry is swept
|
||
* to /regions. Same rule scripts/verify-gospel-links.mjs now classifies
|
||
* them by. Their robots.txt asks Crawl-delay: 10, so this band is SMALL on
|
||
* purpose — six links, a minute — and the full re-read lives in the
|
||
* verifier behind VERIFY_SLOW_HOSTS=1. 1,022 links.
|
||
* knowgod.com /<path> byte-identical 3,864-byte page for a real path and
|
||
* a nonsense one, so the PAGE proves nothing. Cru's own API does prove it —
|
||
* a (language, tool) pair exists when the resource's latest translations
|
||
* intersect that language's — and the verifier already checks them that
|
||
* way. Not reproduced here yet. 94 links.
|
||
*
|
||
* The lesson is the one this repo keeps relearning: a claim about the outside world decays,
|
||
* and an exclusion is a claim. Each host below carries the measurement that justifies its
|
||
* band, so the next person can re-measure rather than re-inherit.
|
||
* ─────────────────────────────────────────────────────────────────────────────────────────
|
||
*
|
||
* IT IS SHOWN TO BE ABLE TO FAIL. Every band runs its controls in band, every run:
|
||
* - status band: URLs that must not exist, one per host shape. Any 2xx and the check
|
||
* reports that it proved nothing and exits non-zero rather than reporting a pass.
|
||
* - mapped band: a real fileset must answer 200 from the API AND an impossible one must not
|
||
* (or the check is either a false green across the host or a libel on every link in it).
|
||
* - landing band: a published code must land on its language page AND one they do not carry
|
||
* must be swept away, or the redirect has stopped separating them.
|
||
*/
|
||
import { readFileSync } from 'node:fs'
|
||
import { resolve, dirname } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
|
||
const HERE = dirname(fileURLToPath(import.meta.url))
|
||
const REPO = resolve(HERE, '..')
|
||
const TABLE = resolve(REPO, 'data/tables/gospel-link-verification.json')
|
||
|
||
const UA =
|
||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'
|
||
|
||
/** Hosts whose HTTP status is real evidence. Measured, not assumed — see the header. */
|
||
const STATUS_MEANINGFUL = new Set([
|
||
'ebible.org', 'alkitab.pw', 'png.bible', 'git.door43.org',
|
||
'scriptureearth.org', 'zume.training', 'arc.gt',
|
||
'jesusfilm.org', 'www.jesusfilm.org',
|
||
])
|
||
|
||
/**
|
||
* Hosts where the PAGE cannot be judged but a sibling URL carries an honest status.
|
||
*
|
||
* This is not a heuristic on a number the site controls. An earlier draft of this band
|
||
* compared the page's body size against an empty shell measured at run time; it separated
|
||
* cleanly the day it was written (7,860b shell against 13.5–85 KB of real page) and it is
|
||
* still the technique scripts/fetch-fcbh-index.mjs used to BUILD the index. But the verifier
|
||
* had already abandoned it for this host, and says why: their WAF started answering HEAD with
|
||
* a 403 whose body was 520 bytes for real and bogus alike, and the size rule silently agreed
|
||
* that everything was dead. A rule that survives only while the front door behaves is not a
|
||
* rule. The API 404s properly, costs a tenth of the page, and describes exactly the fileset
|
||
* the shipped page renders.
|
||
*/
|
||
const URL_MAPPED = {
|
||
'live.bible.is': {
|
||
/* /bible/ENGKJV -> /api/bibles/ENGKJV */
|
||
map: (u) => u.replace('/bible/', '/api/bibles/'),
|
||
mustExist: 'https://live.bible.is/api/bibles/ENGKJV',
|
||
mustNotExist: 'https://live.bible.is/api/bibles/ZZZZZZ',
|
||
},
|
||
}
|
||
|
||
/**
|
||
* Hosts that answer 200 for everything but send a request they do not serve somewhere else.
|
||
* The verdict is read from where the request ENDED, never from the status.
|
||
*/
|
||
const LANDING = {
|
||
'fivefish.org': {
|
||
alive: (final) => /\/\d+\?language=/.test(String(final || '')),
|
||
dead: (final) => /\/regions(\?|#|$)/.test(String(final || '')),
|
||
mustExist: 'https://fivefish.org/cpx',
|
||
mustNotExist: 'https://fivefish.org/zzq',
|
||
/* Their robots.txt asks Crawl-delay: 10. Six links plus two controls is eighty seconds. */
|
||
perHost: Number(process.env.ROT_FIVEFISH || 6),
|
||
gapMs: 10000,
|
||
},
|
||
}
|
||
|
||
/** Hosts left out, with the reason, so the exclusion stays a measurement and not a habit. */
|
||
const NOT_CHECKED_HERE = {
|
||
'knowgod.com': "page bytes are identical; the verifier proves these via Cru's API, not reproduced here",
|
||
}
|
||
|
||
/** Must not exist. If one of these passes, the check cannot see death and says so. */
|
||
const CONTROLS = [
|
||
'https://ebible.org/zzz-no-such-translation-xyzzy/',
|
||
'https://scriptureearth.org/data/zzz/PDF/nope.pdf',
|
||
'https://git.door43.org/unfoldingWord/no-such-repo-xyzzy',
|
||
/* The shipped shape, not a plausible-looking one: /watch/jesus/<language-slug>. */
|
||
'https://jesusfilm.org/watch/jesus/zzz-no-such-language-xyzzy',
|
||
]
|
||
|
||
const PER_HOST = Number(process.env.ROT_PER_HOST || 25)
|
||
|
||
async function get(url, method = 'GET') {
|
||
try {
|
||
const r = await fetch(url, {
|
||
method,
|
||
headers: { 'user-agent': UA },
|
||
redirect: 'follow',
|
||
signal: AbortSignal.timeout(20000),
|
||
})
|
||
const body = method === 'GET' ? await r.text() : ''
|
||
return { status: r.status, len: body.length, final: r.url }
|
||
} catch {
|
||
return { status: 0, len: 0, final: '' }
|
||
}
|
||
}
|
||
|
||
/* A 808 KB film page times 25 is 20 MB of somebody else's bandwidth to learn one number that
|
||
HEAD already carries. Status hosts are asked with HEAD; the shell band needs the body. */
|
||
const status = (url, method = 'HEAD') => get(url, method).then((r) => r.status)
|
||
|
||
let table
|
||
try { table = JSON.parse(readFileSync(TABLE, 'utf8')) } catch {
|
||
console.error('LINK-ROT: NOT MEASURED — no verification table to sample.')
|
||
process.exit(1)
|
||
}
|
||
|
||
const verified = Object.entries(table.links || {})
|
||
.filter(([, r]) => r && r.state === 'verified')
|
||
.map(([u]) => u)
|
||
|
||
const byHost = new Map()
|
||
let skipped = 0
|
||
for (const u of verified) {
|
||
let h
|
||
try { h = new URL(u).host } catch { continue }
|
||
if (!STATUS_MEANINGFUL.has(h) && !URL_MAPPED[h] && !LANDING[h]) { skipped++; continue }
|
||
if (!byHost.has(h)) byHost.set(h, [])
|
||
byHost.get(h).push(u)
|
||
}
|
||
|
||
/* Fixed stride, sorted input: the same sample every run, so a change is a change in the web
|
||
rather than a change in which links we happened to pick. */
|
||
const sample = []
|
||
for (const [host, urls] of [...byHost].sort((a, b) => a[0].localeCompare(b[0]))) {
|
||
urls.sort()
|
||
const want = (LANDING[host] && LANDING[host].perHost) || PER_HOST
|
||
const step = Math.max(1, Math.floor(urls.length / want))
|
||
for (let i = 0; i < urls.length && sample.filter((s) => s.host === host).length < want; i += step) {
|
||
sample.push({ host, url: urls[i] })
|
||
}
|
||
}
|
||
|
||
/* ── controls: the status band ──────────────────────────────────────────────────────────── */
|
||
const ctl = []
|
||
for (const u of CONTROLS) ctl.push([u, await status(u)])
|
||
const blind = ctl.filter(([, s]) => s >= 200 && s < 300)
|
||
if (blind.length) {
|
||
console.error('LINK-ROT: PROVED NOTHING — a URL that cannot exist answered 2xx:')
|
||
for (const [u, s] of blind) console.error(` ${s} ${u}`)
|
||
console.error(' This check cannot tell a live link from a dead one today. Fix it before')
|
||
console.error(' trusting any pass it reports.')
|
||
process.exit(1)
|
||
}
|
||
console.log(`LINK-ROT: status controls pass — ${ctl.length} impossible URL(s), none answered 2xx.`)
|
||
|
||
/* ── controls: the mapped band, both directions ────────────────────────────────────────── */
|
||
for (const [host, cfg] of Object.entries(URL_MAPPED)) {
|
||
if (!byHost.has(host)) continue
|
||
const yes = await status(cfg.mustExist, 'GET')
|
||
const no = await status(cfg.mustNotExist, 'GET')
|
||
if (!(yes >= 200 && yes < 300) || no === 200) {
|
||
console.error(
|
||
`LINK-ROT: PROVED NOTHING — ${host} answered ${yes} for a fileset that exists and ` +
|
||
`${no} for one that cannot. It is not discriminating.`,
|
||
)
|
||
console.error(' A pass here would be a false green across every link on the host, and a')
|
||
console.error(' fail would condemn links that are fine. Neither is worth reporting.')
|
||
process.exit(1)
|
||
}
|
||
console.log(`LINK-ROT: ${host} controls pass — real fileset ${yes}, impossible fileset ${no}.`)
|
||
}
|
||
|
||
/* ── controls: the landing band, both directions ──────────────────────────────────────── */
|
||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
||
for (const [host, cfg] of Object.entries(LANDING)) {
|
||
if (!byHost.has(host)) continue
|
||
const yes = await get(cfg.mustExist, 'HEAD')
|
||
await sleep(cfg.gapMs)
|
||
const no = await get(cfg.mustNotExist, 'HEAD')
|
||
await sleep(cfg.gapMs)
|
||
if (!cfg.alive(yes.final) || !cfg.dead(no.final)) {
|
||
console.error(
|
||
`LINK-ROT: PROVED NOTHING — ${host} sent a real code to ${yes.final || '(nowhere)'} and ` +
|
||
`an impossible one to ${no.final || '(nowhere)'}. The redirect no longer separates them.`,
|
||
)
|
||
console.error(' Every link on this host would be decided by a 200 that means nothing.')
|
||
process.exit(1)
|
||
}
|
||
console.log(`LINK-ROT: ${host} controls pass — real code landed on its language page, impossible one swept away.`)
|
||
}
|
||
|
||
/* ── the sample ─────────────────────────────────────────────────────────────────────────── */
|
||
const dead = []
|
||
for (const { host, url } of sample) {
|
||
const land = LANDING[host]
|
||
if (land) {
|
||
const r = await get(url, 'HEAD')
|
||
await sleep(land.gapMs)
|
||
if (!land.alive(r.final)) dead.push([url, `landed on ${r.final || '(nowhere)'}`])
|
||
continue
|
||
}
|
||
const cfg = URL_MAPPED[host]
|
||
/* GET on the mapped band: their WAF blocks HEAD, which is how the old size rule broke. */
|
||
const st = cfg ? await status(cfg.map(url), 'GET') : await status(url)
|
||
if (!(st >= 200 && st < 300)) {
|
||
dead.push([url, cfg ? `${st} from ${cfg.map(url)}` : st || 'no response'])
|
||
}
|
||
}
|
||
|
||
console.log(
|
||
`LINK-ROT: re-checked ${sample.length} verified link(s) across ${byHost.size} host(s) ` +
|
||
`(${skipped} link(s) not sampled here: ` +
|
||
`${Object.entries(NOT_CHECKED_HERE).map(([h, why]) => `${h} — ${why}`).join('; ')}).`,
|
||
)
|
||
if (dead.length) {
|
||
console.error(`LINK-ROT: FAIL — ${dead.length} link(s) we call "checked" no longer resolve:`)
|
||
for (const [u, why] of dead.slice(0, 10)) console.error(` ${why} ${u}`)
|
||
console.error(' Re-run scripts/verify-gospel-links.mjs to re-test the whole corpus; the')
|
||
console.error(' build drops anything it proves dead.')
|
||
process.exit(1)
|
||
}
|
||
console.log('LINK-ROT: PASS — every sampled link still resolves.')
|