diff --git a/plugin-designs/gospel-resources/scripts/fetch-grn-index.mjs b/plugin-designs/gospel-resources/scripts/fetch-grn-index.mjs index 25d9fa6db..e0e388b3e 100644 --- a/plugin-designs/gospel-resources/scripts/fetch-grn-index.mjs +++ b/plugin-designs/gospel-resources/scripts/fetch-grn-index.mjs @@ -58,13 +58,39 @@ const titleOf = (html) => { return m ? m[1].trim() : '' } +/** + * ═══ WHERE THE REQUEST ENDED DECIDES; THE TITLE ONLY NAMES ══════════════════════════════ + * + * This classified by TITLE alone, and its last rule was a catch-all: any title that matched + * neither known pattern was recorded as "none". That is a default, not a finding, and it was + * wrong often enough to matter. Caught by cross-checking IMB's own access flags against this + * index: of six Doxa codes where IMB said recordings exist and this index said none, FOUR + * had them — alk Alak, nos Nisu Eastern, jle Ngile, lma Limba East — each landing on its own + * numbered 5fish language page. + * + * The redirect is the real discriminator and was already proved in band twice elsewhere in + * this bundle, in verify-gospel-links.mjs and the rot sampler. A published code lands on + * /?language=; a code they do not carry is swept to /regions. So the LANDING + * decides state, and the title is demoted to what it is actually good at — supplying the + * language's name. A page that lands on its language page is a hit even if its title is + * phrased in a way nobody anticipated, which is exactly the case the catch-all was losing. + */ +const LANDED = (final) => /\/\d+\?language=/.test(String(final || '')) +const SWEPT = (final) => /\/regions(\?|#|$)/.test(String(final || '')) + /** @returns {'hit'|'none'|'geo'|'error'} and the language name when it is a hit. */ -function classify(title) { - if (/Learn about Christian beliefs in\s+(.+)$/i.test(title)) { - return { state: 'hit', name: /Learn about Christian beliefs in\s+(.+)$/i.exec(title)[1].trim() } +function classify(title, final) { + if (LANDED(final)) { + const m = /Learn about Christian beliefs in\s+(.+)$/i.exec(title) + let name = m ? m[1].trim() : '' + if (!name) { + try { name = decodeURIComponent(String(final).split('language=')[1] || '').trim() } catch { name = '' } + } + return { state: 'hit', name } } + if (SWEPT(final)) return { state: 'none' } + /* Neither landing shape: do NOT default to "none" — that is the bug this replaced. */ if (/Languages Spoken in/i.test(title)) return { state: 'geo' } - if (title) return { state: 'none' } return { state: 'error' } } @@ -76,7 +102,8 @@ async function probe(iso) { signal: AbortSignal.timeout(30000), }) if (!r.ok) return { state: 'error' } - return classify(titleOf(await r.text())) + const html = await r.text() + return classify(titleOf(html), r.url) } catch { return { state: 'error' } } @@ -112,12 +139,27 @@ for (const r of pins.pins) { pop[c] = (pop[c] || 0) + (r[5] || 0) } } -const todo = [...gap].filter((c) => !decided.has(c)).sort((a, b) => pop[b] - pop[a]) -console.log(`GRN: ${gap.size} gap code(s); ${decided.size} already decided; ${todo.length} left.`) +/* GRN_RECHECK=1 re-asks the codes this index recorded as "none" under the old title + catch-all. They were not measured, they were defaulted, and the landing rule disagrees with + enough of them to be worth the traffic. Scoped to a code list on GRN_CODES when given, so a + run can be pointed at Doxa's roster rather than the whole corpus. */ +const RECHECK = process.env.GRN_RECHECK === '1' +const ONLY = (process.env.GRN_CODES || '').split(/[,\s]+/).filter(Boolean) +let todo +if (RECHECK) { + const pool = ONLY.length ? ONLY : [...(prev.none || [])] + todo = pool.filter((c) => (prev.none || []).includes(c)).sort((a, b) => (pop[b] || 0) - (pop[a] || 0)) + console.log(`GRN: RECHECK — re-asking ${todo.length} code(s) previously recorded "none" under the title rule.`) +} else { + todo = [...gap].filter((c) => !decided.has(c)).sort((a, b) => pop[b] - pop[a]) + console.log(`GRN: ${gap.size} gap code(s); ${decided.size} already decided; ${todo.length} left.`) +} const batch = todo.slice(0, LIMIT_SMOKE ? 3 : MAX) const byIso = { ...(prev.byIso || {}) } -const none = [...(prev.none || [])] +let none = [...(prev.none || [])] +/* A recheck that could only ever ADD would be unable to correct the thing it exists for. */ +const clearNone = (c) => { none = none.filter((x) => x !== c) } let hits = 0 let errors = 0 @@ -127,11 +169,15 @@ for (let i = 0; i < batch.length; i++) { const r = await probe(iso) if (r.state === 'hit') { byIso[iso] = { u: `${ORIGIN}/${iso}`, n: r.name } + clearNone(iso) // a recheck must be able to take a code back out of "none" hits++ } else if (r.state === 'none' || r.state === 'geo') { /* A geo fallback means the path did not resolve to a language, same as "none" for us. */ - none.push(iso) + if (!none.includes(iso)) none.push(iso) } else { + /* An error leaves the code UNDECIDED rather than "none". Recording a failed request as + an absence is how the old catch-all produced false negatives in the first place. */ + clearNone(iso) errors++ } if ((i + 1) % 25 === 0) console.log(` ${i + 1}/${batch.length} (${hits} hit)`)