170 lines
11 KiB
JavaScript
170 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* refresh-legacy-resource-flags.mjs — re-derive the resource booleans in the two curated
|
|
* tables from the pin index, so they stop contradicting the map beside them.
|
|
*
|
|
* THE PROBLEM. gaps-and-opportunities and people-group-resources predate everything this
|
|
* flight added. Their curated columns — the bridge-language analysis, internet access,
|
|
* messaging channels — are the reason they exist and are NOT touched here. But they also
|
|
* carry resource booleans, and those had drifted into saying things the map disproves:
|
|
*
|
|
* 80 rows heart_has_any=false while the index has a resource (6M people)
|
|
* 317 rows "no scripture" while the index has TEXT Scripture
|
|
* 5 rows "no film" while the index has a film
|
|
* 2,106 rows has_audio = the string "unknown", written when no audio pool had been
|
|
* searched. Faith Comes By Hearing has been searched since.
|
|
*
|
|
* WHY REFRESH RATHER THAN REBUILD. The bridge analysis cannot be regenerated from the pins —
|
|
* it is curated work over a narrower, hand-checked set. Rebuilding would destroy it to fix a
|
|
* few columns. So only the facts the index is authoritative for are rewritten, and the
|
|
* fields DERIVED from them are recomputed in the same pass, because leaving `gap` or
|
|
* `opportunity` stale would just move the contradiction one column to the right.
|
|
*
|
|
* has_audio STAYS THREE-STATE. true where FCBH publishes audio; "unknown" otherwise — never
|
|
* false. GRN is the main audio pool and still answers 403 to everything, so "no audio exists"
|
|
* is a claim this build cannot make.
|
|
*/
|
|
import { readFileSync } from 'node:fs'
|
|
import { resolve, dirname } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { writeSnapshot } from './_snapshot-guard.mjs'
|
|
|
|
const HERE = dirname(fileURLToPath(import.meta.url))
|
|
const REPO = resolve(HERE, '../../..')
|
|
const PINS = resolve(REPO, 'data/tables/people-group-pins.json')
|
|
const { codes } = JSON.parse(readFileSync(PINS, 'utf8'))
|
|
|
|
const v = (c) => codes[c] || null
|
|
const hasText = (c) => { const x = v(c); return !!(x && (x.s || (x.au && /text/.test(x.au.m || '')))) }
|
|
const hasFilm = (c) => { const x = v(c); return !!(x && x.f) }
|
|
const hasAudio = (c) => { const x = v(c); return !!(x && x.au && /audio/.test(x.au.m || '')) }
|
|
// Scripture Earth and GRN belong here like every other pool. Without them this predicate
|
|
// reported "nothing to change" while 122 curated rows — 8.9 million people — still said
|
|
// heart_has_any=false for groups the index can now reach. A reconciler that cannot see a
|
|
// source is not reconciling, it is confirming.
|
|
//
|
|
// `gap` and `opportunity` below are derived from this, so widening it moves them too — which
|
|
// is CORRECT: a group that can now hear the gospel in its own language is no longer a gap,
|
|
// and saying otherwise would send a team somewhere already reached.
|
|
const hasAny = (c) => { const x = v(c); return !!(x && (x.s || x.f || x.au || x.bv || x.se || x.ga || (x.d && x.d.length))) }
|
|
|
|
const load = (name) => JSON.parse(readFileSync(resolve(REPO, `data/tables/${name}.json`), 'utf8'))
|
|
const changed = {}
|
|
const bump = (k) => { changed[k] = (changed[k] || 0) + 1 }
|
|
|
|
// ── gaps-and-opportunities ────────────────────────────────────────────────────────────
|
|
{
|
|
const j = load('gaps-and-opportunities')
|
|
for (const r of j.rows || []) {
|
|
const c = r.heart_iso639_3
|
|
if (!c) continue
|
|
const s = hasText(c), f = hasFilm(c), any = hasAny(c)
|
|
if (r.heart_has_scripture !== s) { r.heart_has_scripture = s; bump('heart_has_scripture') }
|
|
if (r.heart_has_film !== f) { r.heart_has_film = f; bump('heart_has_film') }
|
|
if (r.heart_has_any !== any) { r.heart_has_any = any; bump('heart_has_any') }
|
|
const b = r.bridge_iso639_3
|
|
if (b) {
|
|
const bs = hasText(b), bf = hasFilm(b)
|
|
if (r.bridge_has_scripture !== bs) { r.bridge_has_scripture = bs; bump('bridge_has_scripture') }
|
|
if (r.bridge_has_film !== bf) { r.bridge_has_film = bf; bump('bridge_has_film') }
|
|
}
|
|
// DERIVED FIELDS, RECOMPUTED FROM THE TABLE'S OWN DOCUMENTED DEFINITIONS — not from a
|
|
// definition invented here. Each formula below was verified to reproduce the existing
|
|
// column on all 2,106 rows before anything was rewritten, because a refresh that
|
|
// silently redefines a curated column is worse than the drift it set out to fix. A
|
|
// first attempt defined `gap` as simply !heart_has_any, which would have changed 416
|
|
// rows and quietly discarded the "and no in-country bridge either" half of its meaning.
|
|
//
|
|
// gap nothing in the heart language AND no in-country bridge carries anything
|
|
// opportunity nothing in the heart language, but a bridge spoken by MORE THAN ONE
|
|
// group in the same country does have something
|
|
// aligned an opportunity where the environment lines up too: internet yes/partial
|
|
// and a known channel
|
|
const bridgeHas = !!(r.bridge_has_scripture || r.bridge_has_film)
|
|
const gap = !any && !bridgeHas
|
|
if (r.gap !== gap) { r.gap = gap; bump('gap') }
|
|
const opp = !any && bridgeHas && (r.bridge_groups_in_country || 0) > 1
|
|
if (r.opportunity !== opp) { r.opportunity = opp; bump('opportunity') }
|
|
const net = r.internet_access === 'yes' || r.internet_access === 'partial'
|
|
const aligned = opp && net && !!r.top_channel
|
|
if (r.aligned !== aligned) { r.aligned = aligned; bump('aligned') }
|
|
const status = any ? 'reached in heart language'
|
|
: opp ? 'opportunity — bridge language has a resource'
|
|
: 'gap — nothing found in heart or bridge language'
|
|
if (r.status !== status) { r.status = status; bump('status') }
|
|
}
|
|
j.meta = { ...(j.meta || {}), resource_flags_refreshed: new Date().toISOString().slice(0, 10), resource_flags_source: 'data/tables/people-group-pins.json — the same index the map draws. Curated bridge, internet and channel columns are untouched.' }
|
|
writeSnapshot(resolve(REPO, 'data/tables/gaps-and-opportunities.json'), `${JSON.stringify(j, null, 1)}\n`,
|
|
(x) => (x.rows || []).length, 'gaps-and-opportunities rows')
|
|
}
|
|
|
|
// ── people-group-resources ────────────────────────────────────────────────────────────
|
|
{
|
|
const j = load('people-group-resources')
|
|
for (const r of j.rows || []) {
|
|
const c = r.iso639_3
|
|
if (!c) continue
|
|
const s = hasText(c), f = hasFilm(c)
|
|
if (r.has_scripture !== s) { r.has_scripture = s; bump('has_scripture') }
|
|
if (r.has_film !== f) { r.has_film = f; bump('has_film') }
|
|
if ('jesus_film' in r && r.jesus_film !== f) { r.jesus_film = f; bump('jesus_film') }
|
|
// THREE-STATE, never false. See the header.
|
|
const a = hasAudio(c) ? true : 'unknown'
|
|
if (r.has_audio !== a) { r.has_audio = a; bump('has_audio') }
|
|
// THE PAIR NOBODY WAS REFRESHING. These two columns say whether the group has anything
|
|
// at all, and no pass had ever touched them — so 194 rows still read
|
|
// any_resource_found=false for groups the index can now reach. They are a complementary
|
|
// pair and must move together, or the table contradicts itself row by row.
|
|
const any = hasAny(c)
|
|
if (r.any_resource_found !== any) { r.any_resource_found = any; bump('any_resource_found') }
|
|
if (r.no_resource_found !== !any) { r.no_resource_found = !any; bump('no_resource_found') }
|
|
}
|
|
// THE POOL LIST IS A CLAIM TOO. This table still told readers GRN was "not included:
|
|
// requires an API key" — false on both halves. GRN is included, 583 codes of it, and it
|
|
// needed no key: their 5fish site resolves by ISO 639-3 and asks only for a crawl delay.
|
|
// A caveat that names a pool as missing is exactly as misleading as a stale number when
|
|
// the pool is in fact there.
|
|
const notIncluded = (j.meta && j.meta.not_included) || []
|
|
const grnRow = notIncluded.find((x) => x && x.id === 'grn')
|
|
if (grnRow) {
|
|
j.meta.not_included = notIncluded.filter((x) => !x || x.id !== 'grn')
|
|
j.meta.grn_note =
|
|
'Global Recordings Network IS included, via their 5fish site, which resolves by ISO ' +
|
|
'639-3 and needs no API key — only the ten-second crawl delay their robots.txt asks ' +
|
|
'for, so the sweep runs a batch at a time and part of the pool is still unread. It ' +
|
|
'appears as gospel audio, never as Scripture: Words of Life is gospel messages and ' +
|
|
'Bible stories, not a translation.'
|
|
bump('not_included(grn)')
|
|
}
|
|
j.meta = { ...(j.meta || {}), resource_flags_refreshed: new Date().toISOString().slice(0, 10), has_audio_note: 'true where Faith Comes By Hearing publishes audio Scripture; "unknown" otherwise, never false. GRN is the other main audio pool and IS now searched, through their 5fish site, a batch at a time at the crawl delay they ask for — it appears as gospel audio rather than in this column, because Words of Life is gospel messages and Bible stories, not a translation.' }
|
|
writeSnapshot(resolve(REPO, 'data/tables/people-group-resources.json'), `${JSON.stringify(j, null, 1)}\n`,
|
|
(x) => (x.rows || []).length, 'people-group-resources rows')
|
|
}
|
|
|
|
// ── languages-by-country ──────────────────────────────────────────────────────────────
|
|
// The bridge view: one row per (country, language), ranked by prevalence, flagging which
|
|
// languages could carry a resource to groups that lack one. prevalence_rank,
|
|
// bridge_candidate and groups_speaking are curated over a narrower set and are untouched.
|
|
// Its resource flags had drifted the same way the other two tables had: 75 rows claiming
|
|
// the wrong thing about Scripture, 39 about whether the language carries anything at all.
|
|
// A bridge view whose resource flags are wrong recommends the wrong bridge.
|
|
{
|
|
const j = load('languages-by-country')
|
|
for (const r of j.rows || []) {
|
|
const c = r.iso639_3
|
|
if (!c) continue
|
|
const s2 = hasText(c), f2 = hasFilm(c), any2 = hasAny(c)
|
|
if (r.has_scripture !== s2) { r.has_scripture = s2; bump('lbc_has_scripture') }
|
|
if (r.has_film !== f2) { r.has_film = f2; bump('lbc_has_film') }
|
|
if ('carries_resource' in r && r.carries_resource !== any2) { r.carries_resource = any2; bump('lbc_carries_resource') }
|
|
// full_bible stays as curated: the pin index knows a scope only for eBible rows, and
|
|
// downgrading a curated "full Bible" on that basis would lose information, not fix it.
|
|
}
|
|
j.meta = { ...(j.meta || {}), resource_flags_refreshed: new Date().toISOString().slice(0, 10), resource_flags_source: 'data/tables/people-group-pins.json. prevalence_rank, bridge_candidate, groups_speaking and full_bible are curated and untouched.' }
|
|
writeSnapshot(resolve(REPO, 'data/tables/languages-by-country.json'), `${JSON.stringify(j, null, 1)}\n`,
|
|
(x) => (x.rows || []).length, 'languages-by-country rows')
|
|
}
|
|
|
|
console.log('REFRESH: fields corrected —')
|
|
for (const [k, n] of Object.entries(changed).sort((a, b) => b[1] - a[1])) console.log(` ${k.padEnd(22)} ${n}`)
|
|
if (!Object.keys(changed).length) console.log(' (nothing to change)')
|