Check 5fish links too, and say why YouVersion stays out

The rot sampler now covers 5fish by where its redirect LANDS — a published code
goes to its numbered language page, one they do not carry is swept to /regions —
the same rule the verifier uses, with both arms proved in band. Six links a run,
ten seconds apart, because their robots.txt asks for it. That leaves 94 of 8,510
shipped links outside every re-check, down from 5,760 this morning.

YouVersion is the largest catalogue the brief names and it is now recorded as
rejected, with the measurement: nodejs.bible.com serves it without a key but its
robots.txt is Disallow: / , www.bible.com answers even robots.txt with a bot
wall, and the unauthenticated slice returns 2 English versions against the dozens
that exist — so taking it would be both discourteous and wrong. It is named in
the unchecked list on 5,413 codes instead of being silently absent, because a
reader deciding that a group has nothing is entitled to know which pools nobody
searched.

internal/check-link-rot-sample.mjs

@@ -31,11 +31,13 @@
  *                   at all. Its API can: /api/bibles/<filesetId> returns 200 with the fileset
  *                   for a real id and a 75-byte 404 for a bogus one. Same discriminator the
  *                   verifier uses, same in-band abort. 2,636 links.
- *   fivefish.org    /<iso639-3>           always 200; found and not-found differ by page
+ *   fivefish.org    /<iso639-3>           always 200, but the REDIRECT discriminates and
- *                   TITLE, which is how scripts/fetch-grn-index.mjs classifies them. Real,
+ *                   does it on a HEAD: a published code lands on its numbered language page
- *                   but their robots.txt asks Crawl-delay: 10, so a 25-link sample is four
+ *                   (/cpx -> /421?language=Chinese,%20Puxian), one they do not carry is swept
- *                   minutes of polite waiting and belongs in its own slower band, not here.
+ *                   to /regions. Same rule scripts/verify-gospel-links.mjs now classifies
- *                   982 links, still unchecked here, deliberately and on the record.
+ *                   them by. Their robots.txt asks Crawl-delay: 10, so this band is SMALL on
+ *                   purpose — six links, a minute — and the full re-read lives in the
+ *                   verifier behind VERIFY_SLOW_HOSTS=1. 1,022 links.
  *   knowgod.com     /<path>               byte-identical 3,864-byte page for a real path and
  *                   a nonsense one, so the PAGE proves nothing. Cru's own API does prove it —
  *                   a (language, tool) pair exists when the resource's latest translations
@@ -52,6 +54,8 @@
  *     reports that it proved nothing and exits non-zero rather than reporting a pass.
  *   - mapped band: a real fileset must answer 200 from the API AND an impossible one must not
  *     (or the check is either a false green across the host or a libel on every link in it).
+ *   - landing band: a published code must land on its language page AND one they do not carry
+ *     must be swept away, or the redirect has stopped separating them.
  */
 import { readFileSync } from 'node:fs'
 import { resolve, dirname } from 'node:path'
@@ -93,9 +97,24 @@ const URL_MAPPED = {
   },
 }
 
+/**
+ * Hosts that answer 200 for everything but send a request they do not serve somewhere else.
+ * The verdict is read from where the request ENDED, never from the status.
+ */
+const LANDING = {
+  'fivefish.org': {
+    alive: (final) => /\/\d+\?language=/.test(String(final || '')),
+    dead: (final) => /\/regions(\?|#|$)/.test(String(final || '')),
+    mustExist: 'https://fivefish.org/cpx',
+    mustNotExist: 'https://fivefish.org/zzq',
+    /* Their robots.txt asks Crawl-delay: 10. Six links plus two controls is eighty seconds. */
+    perHost: Number(process.env.ROT_FIVEFISH || 6),
+    gapMs: 10000,
+  },
+}
+
 /** Hosts left out, with the reason, so the exclusion stays a measurement and not a habit. */
 const NOT_CHECKED_HERE = {
-  'fivefish.org': 'title-only signal behind Crawl-delay: 10 — needs its own slower band',
   'knowgod.com': "page bytes are identical; the verifier proves these via Cru's API, not reproduced here",
 }
 
@@ -119,9 +138,9 @@ async function get(url, method = 'GET') {
       signal: AbortSignal.timeout(20000),
     })
     const body = method === 'GET' ? await r.text() : ''
-    return { status: r.status, len: body.length }
+    return { status: r.status, len: body.length, final: r.url }
   } catch {
-    return { status: 0, len: 0 }
+    return { status: 0, len: 0, final: '' }
   }
 }
 
@@ -144,7 +163,7 @@ let skipped = 0
 for (const u of verified) {
   let h
   try { h = new URL(u).host } catch { continue }
-  if (!STATUS_MEANINGFUL.has(h) && !URL_MAPPED[h]) { skipped++; continue }
+  if (!STATUS_MEANINGFUL.has(h) && !URL_MAPPED[h] && !LANDING[h]) { skipped++; continue }
   if (!byHost.has(h)) byHost.set(h, [])
   byHost.get(h).push(u)
 }
@@ -154,8 +173,9 @@ for (const u of verified) {
 const sample = []
 for (const [host, urls] of [...byHost].sort((a, b) => a[0].localeCompare(b[0]))) {
   urls.sort()
-  const step = Math.max(1, Math.floor(urls.length / PER_HOST))
+  const want = (LANDING[host] && LANDING[host].perHost) || PER_HOST
-  for (let i = 0; i < urls.length && sample.filter((s) => s.host === host).length < PER_HOST; i += step) {
+  const step = Math.max(1, Math.floor(urls.length / want))
+  for (let i = 0; i < urls.length && sample.filter((s) => s.host === host).length < want; i += step) {
     sample.push({ host, url: urls[i] })
   }
 }
@@ -190,9 +210,35 @@ for (const [host, cfg] of Object.entries(URL_MAPPED)) {
   console.log(`LINK-ROT: ${host} controls pass — real fileset ${yes}, impossible fileset ${no}.`)
 }
 
+/* ── controls: the landing band, both directions ──────────────────────────────────────── */
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
+for (const [host, cfg] of Object.entries(LANDING)) {
+  if (!byHost.has(host)) continue
+  const yes = await get(cfg.mustExist, 'HEAD')
+  await sleep(cfg.gapMs)
+  const no = await get(cfg.mustNotExist, 'HEAD')
+  await sleep(cfg.gapMs)
+  if (!cfg.alive(yes.final) || !cfg.dead(no.final)) {
+    console.error(
+      `LINK-ROT: PROVED NOTHING — ${host} sent a real code to ${yes.final || '(nowhere)'} and ` +
+        `an impossible one to ${no.final || '(nowhere)'}. The redirect no longer separates them.`,
+    )
+    console.error('  Every link on this host would be decided by a 200 that means nothing.')
+    process.exit(1)
+  }
+  console.log(`LINK-ROT: ${host} controls pass — real code landed on its language page, impossible one swept away.`)
+}
+
 /* ── the sample ─────────────────────────────────────────────────────────────────────────── */
 const dead = []
 for (const { host, url } of sample) {
+  const land = LANDING[host]
+  if (land) {
+    const r = await get(url, 'HEAD')
+    await sleep(land.gapMs)
+    if (!land.alive(r.final)) dead.push([url, `landed on ${r.final || '(nowhere)'}`])
+    continue
+  }
   const cfg = URL_MAPPED[host]
   /* GET on the mapped band: their WAF blocks HEAD, which is how the old size rule broke. */
   const st = cfg ? await status(cfg.map(url), 'GET') : await status(url)

plugin-designs/gospel-resources/scripts/build-people-group-pins.mjs

@@ -336,7 +336,7 @@ for (const [c, v] of Object.entries(codes)) {
 // should ever name. A reader deciding where to work needs "we looked and it is not there"
 // to be distinguishable from "nobody looked".
 const SCANNED = 'ebible, arclight, door43, fcbh, scriptureearth, grn'
-const NOT_SCANNED = 'api_bible, dbl, progress_bible'
+const NOT_SCANNED = 'api_bible, dbl, progress_bible, youversion'
 
 // POOLS CONSIDERED AND REJECTED, with the measurement, so nobody spends another tick on
 // them. Each was reachable and doctrinally fine — the reason is always coverage or the
@@ -351,6 +351,22 @@ const NOT_SCANNED = 'api_bible, dbl, progress_bible'
 //                    even a perfect ISO join would add no coverage.
 //   - Davar Partners a ministry site, not a catalogue. No per-language page and therefore
 //                    no URL a worker could be sent to.
+//   - YouVersion /   the brief names it and it is the largest catalogue of the lot, so it is
+//     Bible.com      worth recording exactly why it stays out. Measured 2026-09-25:
+//                    * nodejs.bible.com serves versions by language over plain HTTP with no
+//                      key — and its robots.txt is `User-agent: * / Disallow: /`, allowing
+//                      only /assets/. This repo obeys 5fish's Crawl-delay and reached GRN
+//                      through their own public site rather than around their shield; a
+//                      blanket Disallow gets the same respect. Not ours to take.
+//                    * www.bible.com answers even /robots.txt with a "Client Challenge" bot
+//                      wall, so there is no public catalogue page to read either.
+//                    * and the unauthenticated slice would be wrong anyway: that endpoint
+//                      returns 2 versions for eng and 3 for deu, against the dozens each
+//                      really has. Shipping it would UNDERSTATE coverage while looking
+//                      authoritative, which is worse than leaving it out.
+//                    The honest route is a YouVersion API key, which is a Driver task and
+//                    not something an agent should work around. Until then this pool is
+//                    named in the unchecked list, never in the checked one.
 // GRN was the opposite case and is worth the contrast: also name-shy at first glance, but
 // their 5fish site resolves by ISO 639-3 directly, which is why it is in and these are not.
 // SIGN LANGUAGES GET THEIR OWN PROVENANCE, because the generic list is actively