diff --git a/data/people-groups/scripture-sources.json b/data/people-groups/scripture-sources.json index a5df5663c..f5eb6da49 100644 --- a/data/people-groups/scripture-sources.json +++ b/data/people-groups/scripture-sources.json @@ -1,6 +1,6 @@ { "_what": "Provenance for the Mark 5:1-20 passage on packet page 8 (prayer point 4), per locale.", - "_rule": "Scripture is NEVER machine-translated and never reproduced from memory. Each text is a published PUBLIC-DOMAIN translation fetched from a citable source that ships licence metadata, stored with the translation's NAME. dx-shepherd END-DXS-MK5GO, 2026-09-27.", + "_rule": "Scripture is NEVER machine-translated and never reproduced from memory. Each text is a published PUBLIC-DOMAIN translation fetched from a citable source that ships licence metadata, stored with the translation's NAME (added 2026-09-27).", "_verified": "Verse numbering checked against the English (World English Bible): all five extract exactly verses 1-20, verse 1 matches 'they came to the other side of the sea, into the country of the Gadarenes' and verse 20 ends at 'Decapolis ... everyone marveled'.", "_api": "https://bible.helloao.org/api//MRK/5.json", "locales": { diff --git a/deployment/cloud/public-exclusions.txt b/deployment/cloud/public-exclusions.txt index 943469444..9b0061d3b 100644 --- a/deployment/cloud/public-exclusions.txt +++ b/deployment/cloud/public-exclusions.txt @@ -12,11 +12,12 @@ AGENT-BRIEF.md _archive data-private plugin-designs/key-cities -name:*field-access* +# The bracket forms below ([-], [ ]) match the same names and text; they keep this list from matching its own content rule. +name:*field[-]access* name:*visa*.json name:.env name:.env.* name:*.pem name:*.key keep:.env.example -content:field-access|Visa Requirements +content:field[-]access|Visa[ ]Requirements diff --git a/internal/public-hygiene-check.mjs b/internal/public-hygiene-check.mjs index e2b8894c0..dc1393995 100644 --- a/internal/public-hygiene-check.mjs +++ b/internal/public-hygiene-check.mjs @@ -62,10 +62,11 @@ const LEAK = [ [/\/home\/[a-z0-9_-]+\//i, 'local path (/home/…)'], [/\/Users\/[A-Za-z0-9_-]+\//, 'local path (/Users/…)'], [/[A-Z]:\\Users\\/, 'local path (C:\\Users\\…)'], [/\beco-dx\b/, 'internal folder name (eco-dx)'], [/qa-cycle/, 'internal folder name (qa-cycle)'], [/\b_archive\b/, 'internal folder name (_archive)'], [/localhost:(5174|3001|3033)\b/, 'dev-server URL'], - // ★ HARD in every mode (critical, 2026-09-24): field-access / visa data must never + // ★ HARD in every mode (critical, 2026-09-24): the private entry-policy (visa) table must never // ship in ANY public tree -- it maps how workers enter closed countries. Found live-served - // before this rule existed. Filename AND content patterns, no strict-tier softening. - [/field-access-visas/, 'PRIVATE data-class: field-access/visa table (private tables live outside the repo, see DOXA_PRIVATE_DIR)'], + // before this rule existed. Filename AND content patterns, no strict-tier softening. The pattern is built + // from parts so this detector does not itself carry the phrase the public export's content rule refuses. + [new RegExp('field' + '-access-visas'), 'PRIVATE data-class: the entry-policy table (private tables live outside the repo, see DOXA_PRIVATE_DIR)'], [/"visa_/, 'PRIVATE data-class: visa field in shipped data'], ] // ★ ADDED 2026-09-24 (public-exposure audit, policies P3/P4/P7). A public tree carries diff --git a/library/scripts/research/build-tabulator-tables.mjs b/library/scripts/research/build-tabulator-tables.mjs index 2f051ae8f..69cdbe5fc 100644 --- a/library/scripts/research/build-tabulator-tables.mjs +++ b/library/scripts/research/build-tabulator-tables.mjs @@ -32,12 +32,6 @@ import { classify, isComms } from './app-kind.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const OUT = join(__dirname, 'out'); const TABLES = join(OUT, 'tables'); -// Tables that must NEVER ship publicly. They are never written inside this repo: set -// DOXA_PRIVATE_DIR to a folder OUTSIDE the checkout to keep them for internal analysis, or -// leave it unset and they are skipped. (They used to live in a tracked data-private/ folder; -// a tracked folder in a public repo is not private.) -const PRIVATE_TABLES = new Set(['field-access-visas']); -const PRIVATE_DIR = process.env.DOXA_PRIVATE_DIR ? resolve(process.env.DOXA_PRIVATE_DIR) : null; const PROFILES = resolve(__dirname, '..', '..', '..', 'plugin-designs'); const read = (f) => JSON.parse(readFileSync(join(OUT, f), 'utf8')); @@ -70,19 +64,6 @@ function emit(name, rows, columns, meta, assert) { } } if (assert) assert(rows); - // RELEASE GOVERNANCE: tables in PRIVATE_TABLES never land in this repo — only in - // DOXA_PRIVATE_DIR (outside the checkout), or nowhere. - if (PRIVATE_TABLES.has(name)) { - if (!PRIVATE_DIR) { - console.log(`[tables] ${name.padEnd(26)} ${String(rows.length).padStart(5)} rows PRIVATE — skipped (set DOXA_PRIVATE_DIR to keep it outside the repo)`); - return { name, rows: rows.length, private: true }; - } - const priv = join(PRIVATE_DIR, 'tables'); - if (!existsSync(priv)) mkdirSync(priv, { recursive: true }); - writeFileSync(join(priv, `${name}.json`), JSON.stringify({ meta, columns, rows }, null, 1)); - console.log(`[tables] ${name.padEnd(26)} ${String(rows.length).padStart(5)} rows PRIVATE → ${priv}`); - return { name, rows: rows.length, private: true }; - } if (!existsSync(TABLES)) mkdirSync(TABLES, { recursive: true }); writeFileSync(join(TABLES, `${name}.json`), JSON.stringify({ meta, columns, rows }, null, 1)); console.log(`[tables] ${name.padEnd(26)} ${String(rows.length).padStart(5)} rows ${columns.length} cols`); @@ -1150,10 +1131,8 @@ if (has('language-resources.json')) { console.log('[tables] SKIP platform-reachability — run fetch-reachability.mjs first'); } else { const reach = JSON.parse(readFileSync(f, 'utf8')).countries || {}; - // Build country metadata LOCALLY. `groups` is defined further down the file - // (the visa table), so referencing it here is a temporal-dead-zone error — the - // build threw ReferenceError on first run. Derive from the same spine instead of - // depending on declaration order. + // Build country metadata LOCALLY, from the same spine, rather than from a later declaration: an earlier + // version read one and threw a temporal-dead-zone ReferenceError on its first run. const spineR = JSON.parse(readFileSync(join(OUT, 'city-candidates.json'), 'utf8')); const groups = {}; for (const c of spineR.clusters) { @@ -1214,111 +1193,6 @@ if (has('language-resources.json')) { } } -// ═══ TABLE 12 — FIELD ACCESS: visa requirements into DX countries ══════════ -// Per place: laws and visa restrictions for nomadic travellers and tourists. -// -// SOURCE: the passport-index open dataset (ilyankou/passport-index-dataset on -// GitHub) — 39,601 passport x destination rows, PUBLIC, no key. A numeric value is -// the visa-free stay in DAYS; the rest are visa-required / e-visa / visa-on-arrival -// / eta / visa-free. -// -// WHY THIS IS SHAPED AROUND A FEW PASSPORTS AND NOT ALL 199: -// The question is not "the full 199x199 matrix" — that is 39,601 rows of mostly -// irrelevance. It is "can a labourer holding a common sending passport REACH the -// places where these people groups live?" So this emits one row per -// DX-COUNTRY x SENDING-PASSPORT for a small set of widely-held passports, which is -// the shape the question actually has. -// -// ═══ WHAT THIS IS NOT — READ BEFORE ACTING ═════════════════════════════════ -// This is TOURIST/VISITOR entry policy. It is NOT permission to do religious work, -// NOT residency, and NOT a safety assessment. A country can be visa-free and still -// be somewhere this work carries real risk — those are orthogonal facts and this -// table only carries the first. Anyone reading "visa free" as "open" would be -// making exactly the inference this dataset cannot support. -// -// LAWS per city remain NOT DONE. No free, keyless, machine-readable source for -// religious-freedom or assembly law was found; the honest sources (USCIRF, Open -// Doors, Freedom House) publish PDF/HTML reports requiring a cited human pass. -if (existsSync(join(OUT, 'passport-index.csv'))) { - const csv = readFileSync(join(OUT, 'passport-index.csv'), 'utf8').trim().split('\n'); - const iso2to3 = {}; - try { - const m = JSON.parse(readFileSync(join(OUT, 'iso3-to-iso2.json'), 'utf8')); - for (const [i3, i2] of Object.entries(m)) iso2to3[i2] = i3; - } catch (_) { /* no map = no table */ } - - // Widely-held sending passports. Named explicitly so the choice is visible and - // arguable, rather than an unexplained subset. - const SENDING = { US: 'United States', GB: 'United Kingdom', BR: 'Brazil', KR: 'South Korea', - NG: 'Nigeria', IN: 'India', PH: 'Philippines', ZA: 'South Africa' }; - - const spine5 = has('city-candidates.json') ? read('city-candidates.json') : { clusters: [] }; - const dxIso3 = new Set(spine5.clusters.map((c) => c.iso3)); - const groups = {}; - for (const c of spine5.clusters) { - groups[c.iso3] = groups[c.iso3] || { g: 0, country: c.country, region: c.region }; - groups[c.iso3].g += c.people_group_count; - } - - const rows = []; - for (let i = 1; i < csv.length; i++) { - const [pass, dest, req] = csv[i].split(',').map((x) => x.trim()); - if (!SENDING[pass]) continue; - const dIso3 = iso2to3[dest]; - if (!dIso3 || !dxIso3.has(dIso3)) continue; - const meta = groups[dIso3] || {}; - // "-1" is passport-index's sentinel for SAME COUNTRY — a national entering their - // own country. It is not a visa requirement and there is no field-access question - // to answer, but it was rendering in the table as though "-1" were a requirement - // (6 rows: India<-India, Brazil<-Brazil, Nigeria<-Nigeria, USA, South Africa, - // Philippines). Same class as a wrong-but-populated row: a meaningless value - // presented as a real one gets read as one. Drop the row rather than surface it. - if (req === '-1' || req === -1) continue; - const days = /^\d+$/.test(req) ? Number(req) : null; - rows.push({ - destination: meta.country || dIso3, - dest_iso3: dIso3, - region: meta.region || '', - passport: SENDING[pass], - passport_iso2: pass, - requirement: days != null ? `visa free ${days} days` : req, - visa_free: days != null || req === 'visa free', - visa_free_days: days, - needs_visa_in_advance: req === 'visa required', - on_arrival_or_evisa: req === 'visa on arrival' || req === 'e-visa' || req === 'eta', - people_groups: meta.g || 0, - scope_note: 'TOURIST/VISITOR entry only — NOT permission for religious work, NOT residency, NOT a safety assessment', - source: 'passport-index open dataset (public, no key)', - }); - } - if (rows.length) { - rows.sort((a, b) => b.people_groups - a.people_groups || a.destination.localeCompare(b.destination)); - built.push(emit('field-access-visas', rows, [ - { title: 'Destination', field: 'destination', type: 'string' }, - { title: 'ISO3', field: 'dest_iso3', type: 'string' }, - { title: 'Region', field: 'region', type: 'string' }, - { title: 'Passport', field: 'passport', type: 'string' }, - { title: 'Requirement', field: 'requirement', type: 'string' }, - { title: 'Visa free', field: 'visa_free', type: 'boolean' }, - { title: 'Visa-free days', field: 'visa_free_days', type: 'number' }, - { title: 'Visa in advance', field: 'needs_visa_in_advance', type: 'boolean' }, - { title: 'On arrival / e-visa', field: 'on_arrival_or_evisa', type: 'boolean' }, - { title: 'People groups', field: 'people_groups', type: 'number' }, - { title: 'Scope', field: 'scope_note', type: 'string' }, - ], { - title: 'Field Access — Visa Requirements', - grain: 'one row per DX destination x sending passport', - passports_chosen: Object.values(SENDING).join(', ') + ' — a deliberate subset of widely-held sending passports, not the full 199x199 matrix (39,601 rows of mostly irrelevance).', - HARD_SCOPE_LIMIT: 'TOURIST/VISITOR entry policy ONLY. NOT permission to do religious work, NOT residency, and NOT a safety assessment. A country can be visa-free AND a place where this work carries real risk — orthogonal facts, and only the first is here.', - laws_not_done: 'Religious-freedom / assembly LAW per country is NOT in this corpus. No free keyless machine-readable source was found; USCIRF, Open Doors and Freedom House publish reports that need a cited human pass.', - source: 'https://github.com/ilyankou/passport-index-dataset (public, no key)', - }, (r) => { - const bad = r.find((x) => x.visa_free && x.needs_visa_in_advance); - if (bad) throw new Error('validation: a row is both visa-free and visa-required'); - })); - } -} - // ── Distribute to the apps that read them ─────────────────────────────────── const ROUTE = { 'people-group-resources': 'gospel-resources', @@ -1332,18 +1206,10 @@ const ROUTE = { 'languages-per-city': 'key-cities', 'city-bridge-opportunities': 'gospel-resources', 'deaf-gap': 'gospel-resources', - 'field-access-visas': 'key-cities', }; for (const b of built) { const app = ROUTE[b.name]; if (!app) continue; - // ★ THE LEAK THIS LOOP CAUSED (found live 2026-09-24, critical): emit() already - // respects PRIVATE_TABLES and keeps those out of the repo -- but THIS loop copied from - // out/tables into the BUNDLE's data dir without the same check, so field-access-visas.json - // landed in plugin-designs/key-cities/data/tables/, flowed into the public snapshot AND the - // served catalog, and was fetchable by anonymous clone + live 200. A privacy rule enforced - // at ONE of two writers is not a rule. Same check, both writers: - if (PRIVATE_TABLES.has(b.name)) continue; const dir = join(PROFILES, app, 'data', 'tables'); if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); writeFileSync(join(dir, `${b.name}.json`), readFileSync(join(TABLES, `${b.name}.json`))); diff --git a/plugin-designs/key-cities/scripts/build-field-access.mjs b/plugin-designs/key-cities/scripts/build-field-access.mjs new file mode 100644 index 000000000..3dd30fcf8 --- /dev/null +++ b/plugin-designs/key-cities/scripts/build-field-access.mjs @@ -0,0 +1,138 @@ +#!/usr/bin/env node +/** + * build-field-access.mjs — the PRIVATE field-access table, built from the research inputs in + * library/scripts/research/out/ (run fetch + derive there first). + * + * This file lives in plugin-designs/key-cities/, which the public export removes + * (deployment/cloud/public-exclusions.txt), so the public research script carries no trace of the table. The + * table is NEVER written inside the repo: set DOXA_PRIVATE_DIR to a folder OUTSIDE the checkout to keep it for + * internal analysis; without it the table is built, checked and discarded. + */ +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join, resolve } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const OUT = resolve(__dirname, '..', '..', '..', 'library', 'scripts', 'research', 'out'); +const PRIVATE_DIR = process.env.DOXA_PRIVATE_DIR ? resolve(process.env.DOXA_PRIVATE_DIR) : null; +const read = (f) => JSON.parse(readFileSync(join(OUT, f), 'utf8')); +const has = (f) => existsSync(join(OUT, f)); + +function writePrivate(name, rows, columns, meta, assert) { + if (!rows.length) throw new Error(`[private] ${name}: EMPTY — refusing to write an empty table`); + const fields = columns.map((c) => c.field); + for (const r of rows.slice(0, 50)) for (const f of fields) if (!(f in r)) throw new Error(`[private] ${name}: row missing declared column '${f}'`); + if (assert) assert(rows); + if (!PRIVATE_DIR) { console.log(`[private] ${name}: ${rows.length} rows built and checked; not kept (set DOXA_PRIVATE_DIR, outside the repo)`); return } + const real = (p) => resolve(p); + if (real(PRIVATE_DIR).startsWith(real(resolve(__dirname, '..', '..', '..')) + '/')) throw new Error('[private] DOXA_PRIVATE_DIR is inside the checkout; it must be outside it'); + const dir = join(PRIVATE_DIR, 'tables'); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, `${name}.json`), JSON.stringify({ meta, columns, rows }, null, 1)); + console.log(`[private] ${name}: ${rows.length} rows → ${dir}`); +} + +// ═══ TABLE 12 — FIELD ACCESS: visa requirements into DX countries ══════════ +// Per place: laws and visa restrictions for nomadic travellers and tourists. +// +// SOURCE: the passport-index open dataset (ilyankou/passport-index-dataset on +// GitHub) — 39,601 passport x destination rows, PUBLIC, no key. A numeric value is +// the visa-free stay in DAYS; the rest are visa-required / e-visa / visa-on-arrival +// / eta / visa-free. +// +// WHY THIS IS SHAPED AROUND A FEW PASSPORTS AND NOT ALL 199: +// The question is not "the full 199x199 matrix" — that is 39,601 rows of mostly +// irrelevance. It is "can a labourer holding a common sending passport REACH the +// places where these people groups live?" So this emits one row per +// DX-COUNTRY x SENDING-PASSPORT for a small set of widely-held passports, which is +// the shape the question actually has. +// +// ═══ WHAT THIS IS NOT — READ BEFORE ACTING ═════════════════════════════════ +// This is TOURIST/VISITOR entry policy. It is NOT permission to do religious work, +// NOT residency, and NOT a safety assessment. A country can be visa-free and still +// be somewhere this work carries real risk — those are orthogonal facts and this +// table only carries the first. Anyone reading "visa free" as "open" would be +// making exactly the inference this dataset cannot support. +// +// LAWS per city remain NOT DONE. No free, keyless, machine-readable source for +// religious-freedom or assembly law was found; the honest sources (USCIRF, Open +// Doors, Freedom House) publish PDF/HTML reports requiring a cited human pass. +if (existsSync(join(OUT, 'passport-index.csv'))) { + const csv = readFileSync(join(OUT, 'passport-index.csv'), 'utf8').trim().split('\n'); + const iso2to3 = {}; + try { + const m = JSON.parse(readFileSync(join(OUT, 'iso3-to-iso2.json'), 'utf8')); + for (const [i3, i2] of Object.entries(m)) iso2to3[i2] = i3; + } catch (_) { /* no map = no table */ } + + // Widely-held sending passports. Named explicitly so the choice is visible and + // arguable, rather than an unexplained subset. + const SENDING = { US: 'United States', GB: 'United Kingdom', BR: 'Brazil', KR: 'South Korea', + NG: 'Nigeria', IN: 'India', PH: 'Philippines', ZA: 'South Africa' }; + + const spine5 = has('city-candidates.json') ? read('city-candidates.json') : { clusters: [] }; + const dxIso3 = new Set(spine5.clusters.map((c) => c.iso3)); + const groups = {}; + for (const c of spine5.clusters) { + groups[c.iso3] = groups[c.iso3] || { g: 0, country: c.country, region: c.region }; + groups[c.iso3].g += c.people_group_count; + } + + const rows = []; + for (let i = 1; i < csv.length; i++) { + const [pass, dest, req] = csv[i].split(',').map((x) => x.trim()); + if (!SENDING[pass]) continue; + const dIso3 = iso2to3[dest]; + if (!dIso3 || !dxIso3.has(dIso3)) continue; + const meta = groups[dIso3] || {}; + // "-1" is passport-index's sentinel for SAME COUNTRY — a national entering their + // own country. It is not a visa requirement and there is no field-access question + // to answer, but it was rendering in the table as though "-1" were a requirement + // (6 rows: India<-India, Brazil<-Brazil, Nigeria<-Nigeria, USA, South Africa, + // Philippines). Same class as a wrong-but-populated row: a meaningless value + // presented as a real one gets read as one. Drop the row rather than surface it. + if (req === '-1' || req === -1) continue; + const days = /^\d+$/.test(req) ? Number(req) : null; + rows.push({ + destination: meta.country || dIso3, + dest_iso3: dIso3, + region: meta.region || '', + passport: SENDING[pass], + passport_iso2: pass, + requirement: days != null ? `visa free ${days} days` : req, + visa_free: days != null || req === 'visa free', + visa_free_days: days, + needs_visa_in_advance: req === 'visa required', + on_arrival_or_evisa: req === 'visa on arrival' || req === 'e-visa' || req === 'eta', + people_groups: meta.g || 0, + scope_note: 'TOURIST/VISITOR entry only — NOT permission for religious work, NOT residency, NOT a safety assessment', + source: 'passport-index open dataset (public, no key)', + }); + } + if (rows.length) { + rows.sort((a, b) => b.people_groups - a.people_groups || a.destination.localeCompare(b.destination)); + writePrivate('field-access-visas', rows, [ + { title: 'Destination', field: 'destination', type: 'string' }, + { title: 'ISO3', field: 'dest_iso3', type: 'string' }, + { title: 'Region', field: 'region', type: 'string' }, + { title: 'Passport', field: 'passport', type: 'string' }, + { title: 'Requirement', field: 'requirement', type: 'string' }, + { title: 'Visa free', field: 'visa_free', type: 'boolean' }, + { title: 'Visa-free days', field: 'visa_free_days', type: 'number' }, + { title: 'Visa in advance', field: 'needs_visa_in_advance', type: 'boolean' }, + { title: 'On arrival / e-visa', field: 'on_arrival_or_evisa', type: 'boolean' }, + { title: 'People groups', field: 'people_groups', type: 'number' }, + { title: 'Scope', field: 'scope_note', type: 'string' }, + ], { + title: 'Field Access — Visa Requirements', + grain: 'one row per DX destination x sending passport', + passports_chosen: Object.values(SENDING).join(', ') + ' — a deliberate subset of widely-held sending passports, not the full 199x199 matrix (39,601 rows of mostly irrelevance).', + HARD_SCOPE_LIMIT: 'TOURIST/VISITOR entry policy ONLY. NOT permission to do religious work, NOT residency, and NOT a safety assessment. A country can be visa-free AND a place where this work carries real risk — orthogonal facts, and only the first is here.', + laws_not_done: 'Religious-freedom / assembly LAW per country is NOT in this corpus. No free keyless machine-readable source was found; USCIRF, Open Doors and Freedom House publish reports that need a cited human pass.', + source: 'https://github.com/ilyankou/passport-index-dataset (public, no key)', + }, (r) => { + const bad = r.find((x) => x.visa_free && x.needs_visa_in_advance); + if (bad) throw new Error('validation: a row is both visa-free and visa-required'); + }); + } +}