diff --git a/internal/check-pg-name-gate.mjs b/internal/check-pg-name-gate.mjs index cf7b68f3a..671183b29 100644 --- a/internal/check-pg-name-gate.mjs +++ b/internal/check-pg-name-gate.mjs @@ -5,13 +5,14 @@ // A. the table loads (data/pplr/pg-names.json, the file the tree serves as assets/pplr/pg-names.json); // B. REAL ROWS: Balantak, Balo, Abaga and Abai Sungai (translated as words) show English in every locale; // Balangao Bontoc (a transliteration) shows its own names; it and ro, which have no column, show English; -// C. TWO-SIDED on planted rows: a transliteration with its Latin names replaced by words falls back, and the -// same row with one far locale still passes (the rule is "two or more"); -// D. the whole table falls back between 30% and 45% (measured 37.6%, 8,055 of 21,419). A similarity that +// C. TWO-SIDED on planted rows, AT the threshold: EXACTLY two far Latin locales falls back, one is kept (the rule +// is "two or more"); a row with only one Latin name to check is not trusted (no evidence → English); +// D. the whole table falls back between 30% and 45% (measured 38.2%, 8,172 of 21,419). A similarity that // always says "same" (0%) or always "different" (100%) fails here even if B and C were tuned around; // E. THE ONE-READER RULE: a file in library/ or plugin-designs/ that names pg-names must take its names through -// peopleGroupName.js. pplr-data-maps is the carded exception (its coverage views read raw rows on purpose). -// A NEGATIVE CONTROL runs the scanner on a planted raw reader and must catch it. +// peopleGroupName.js, and must not index its rows itself (`.names[`) even when it imports the helper. +// pplr-data-maps is the carded exception (its coverage views read raw rows on purpose). NEGATIVE CONTROLS: a +// planted raw reader, and a planted file that imports the helper AND indexes rows, must both be caught. import { readFileSync, readdirSync, statSync } from 'node:fs' import { resolve, dirname, join, relative } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' @@ -56,22 +57,26 @@ ok(name(220001, 'it') === 'Balangao Bontoc' && name(220001, 'ro') === 'Balangao // ── C: planted rows, two-sided ── const planted = (fn) => { const r = bb.slice(); fn(r); return r } -const words = planted((r) => { r[locales.indexOf('fr')] = 'Sac à dos'; r[locales.indexOf('es')] = 'Mochila'; r[locales.indexOf('de')] = 'Rucksack' }) +const twoFar = planted((r) => { r[locales.indexOf('fr')] = 'Sac à dos'; r[locales.indexOf('de')] = 'Rucksack' }) const oneFar = planted((r) => { r[locales.indexOf('fr')] = 'Sac à dos' }) -ok(!H.isTransliterated(words, locales) && H.peopleGroupName(words, locales, 'ar') === 'Balangao Bontoc' && H.isTransliterated(oneFar, locales), - `planted rows: three words → ${H.isTransliterated(words, locales) ? 'KEPT' : 'fell back'}, one word → ${H.isTransliterated(oneFar, locales) ? 'kept' : 'FELL BACK'}`, - 'planted: a row with three Latin names replaced by words falls back (ar too); one far locale alone is kept') +const oneLatin = planted((r) => { for (const l of H.LATIN_LOCALES) if (l !== 'fr') r[locales.indexOf(l)] = '' }) +const kept = (r) => (H.isTransliterated(r, locales) ? 'kept' : 'English') +ok(kept(twoFar) === 'English' && H.peopleGroupName(twoFar, locales, 'ar') === 'Balangao Bontoc' && kept(oneFar) === 'kept' + && kept(oneLatin) === 'English' && H.peopleGroupName(oneLatin, locales, 'ar', 'Balangao Bontoc') === 'Balangao Bontoc', + `planted rows: exactly two far → ${kept(twoFar)}, one far → ${kept(oneFar)}, one Latin name only → ${kept(oneLatin)}`, + 'planted, at the threshold: exactly two far Latin locales → English (ar too); one far → kept; only one Latin name to check → English') // ── D: the whole table ── let back = 0 for (const row of Object.values(names)) if (!H.isTransliterated(row, locales)) back++ const share = back / Object.keys(names).length -ok(share > 0.30 && share < 0.45, `${back} rows (${(share * 100).toFixed(1)}%) fall back; expected 30-45% (measured 37.6%)`, +ok(share > 0.30 && share < 0.45, `${back} rows (${(share * 100).toFixed(1)}%) fall back; expected 30-45% (measured 38.2%)`, `${back} of ${Object.keys(names).length} rows (${(share * 100).toFixed(1)}%) fall back to English`) // ── E: one reader ── const EXEMPT = ['plugin-designs/pplr-data-maps/'] -const rawReaders = (files) => files.filter(({ path, src }) => /pg-names/.test(src) && !/peopleGroupName(\.js)?['"]/.test(src) && !EXEMPT.some((x) => path.startsWith(x))) +const rawReaders = (files) => files.filter(({ path, src }) => /pg-names/.test(src) && !EXEMPT.some((x) => path.startsWith(x)) + && (!/peopleGroupName(\.js)?['"]/.test(src) || /\.names\s*\[/.test(src))) const walk = (dir, out = []) => { for (const f of readdirSync(dir)) { if (f === 'node_modules' || f.startsWith('.')) continue @@ -82,8 +87,12 @@ const walk = (dir, out = []) => { return out } const sources = [...walk(P('library')), ...walk(P('plugin-designs'))].filter((f) => f.path !== 'library/utils/peopleGroupName.js') -ok(rawReaders([{ path: 'plugin-designs/x/x.vue', src: "fetch(base + '/pplr/pg-names.json').then((t) => t.names[code][3])" }]).length === 1, - 'NEGATIVE CONTROL: the one-reader scan missed a planted raw reader', 'negative control: a planted raw pg-names reader is caught') +const PLANT_RAW = "fetch(base + '/pplr/pg-names.json').then((t) => t.names[code][3])" +const PLANT_BOTH = "import { peopleGroupName } from '@map/utils/peopleGroupName.js'\n" + PLANT_RAW +const PLANT_GOOD = "import { peopleGroupName } from '@map/utils/peopleGroupName.js'\nfetch(base + '/pplr/pg-names.json').then((t) => peopleGroupName(t.names && row, t.locales, lang, name))" +ok(rawReaders([{ path: 'plugin-designs/x/a.vue', src: PLANT_RAW }, { path: 'plugin-designs/x/b.vue', src: PLANT_BOTH }, { path: 'plugin-designs/x/c.vue', src: PLANT_GOOD }]).map((f) => f.path).join() === 'plugin-designs/x/a.vue,plugin-designs/x/b.vue', + 'NEGATIVE CONTROL: the one-reader scan missed a planted raw reader (or flagged a helper-only one)', + 'negative controls: a raw reader, and one that imports the helper AND indexes rows, are both caught; a helper-only reader passes') const raw = rawReaders(sources).map((f) => f.path) ok(!raw.length, `pg-names read outside peopleGroupName.js: ${raw.join(', ')}`, `no file outside peopleGroupName.js reads pg-names rows (${sources.length} sources; pplr-data-maps carded)`) diff --git a/library/utils/peopleGroupName.js b/library/utils/peopleGroupName.js index d66fd1f8a..18493e0dc 100644 --- a/library/utils/peopleGroupName.js +++ b/library/utils/peopleGroupName.js @@ -13,7 +13,8 @@ * (bigram Dice similarity, accents folded). When TWO OR MORE of them fall below 0.5, the row is a translation, and * EVERY locale of that group shows English, because the non-Latin names (ar, hi, ru, zh) came from the same * process and cannot be checked directly. - * Measured on the whole table (2026-09-30): 8,055 of 21,419 rows (37.6%) fall back. Balantak, Balo, Abaga and Abai + * Measured on the whole table: 8,055 of 21,419 rows (37.6%) are translations, and 117 more have fewer than two + * Latin names to check, so they are not trusted either: 8,172 (38.2%) show English. Balantak, Balo, Abaga and Abai * Sungai fall back; Balangao Bontoc keeps its names. The 0.4-0.5 band (1,406 rows) is about half wrong (Bawaria → * "Bavière", Ager → "Agriculteur"), so 0.5 is the safer side: a wrongly kept translation is the worse error. * @@ -28,6 +29,7 @@ export const LATIN_LOCALES = ['fr', 'pt', 'es', 'tl', 'id', 'de'] export const MIN_SIMILARITY = 0.5 export const MAX_FAR_LOCALES = 1 // two or more far locales → the row is a translation +export const MIN_LATIN_EVIDENCE = 2 // fewer Latin names than this → no evidence either way → English // The app's locales → pg-names.json's. it and ro have no column, so they show English. const LOCALE_COLUMN = { en: 'en', es: 'es', fr: 'fr', pt: 'pt', de: 'de', ru: 'ru', ar: 'ar', hi: 'hi', zh: 'zh_CN' } @@ -47,22 +49,28 @@ export function nameSimilarity(a, b) { return (2 * shared) / (x.length - 1 + y.length - 1) } -/** True when the row's names are transliterations (keep them), false when they were translated as words. */ +/** + * True when the row's names are transliterations (keep them), false when they were translated as words. A row with + * fewer than MIN_LATIN_EVIDENCE Latin names cannot be checked (117 rows), so it is NOT trusted: English. + */ export function isTransliterated(row, locales) { if (!Array.isArray(row) || !Array.isArray(locales)) return false const en = row[locales.indexOf('en')] if (!en) return false - let far = 0 + let far = 0, seen = 0 for (const l of LATIN_LOCALES) { const v = row[locales.indexOf(l)] - if (v && nameSimilarity(v, en) < MIN_SIMILARITY) far++ + if (!v) continue + seen++ + if (nameSimilarity(v, en) < MIN_SIMILARITY) far++ } - return far <= MAX_FAR_LOCALES + return seen >= MIN_LATIN_EVIDENCE && far <= MAX_FAR_LOCALES } /** * The name to show for a group: its `lang` name when the row is a transliteration and has that column, else - * `english` (the caller's own English name; the row's English column when the caller has none). + * `english`. ★ CALLERS ALWAYS PASS `english` (the record's own name): it is what a fallback shows, and a row the + * table does not have (row undefined) has no English column to fall back to, so without it the result is ''. */ export function peopleGroupName(row, locales, lang, english) { const en = english || (Array.isArray(row) && Array.isArray(locales) ? row[locales.indexOf('en')] : '') || '' @@ -72,4 +80,4 @@ export function peopleGroupName(row, locales, lang, english) { return (typeof v === 'string' && v.trim()) ? v : en } -export default { peopleGroupName, isTransliterated, nameSimilarity, LATIN_LOCALES, MIN_SIMILARITY, MAX_FAR_LOCALES } +export default { peopleGroupName, isTransliterated, nameSimilarity, LATIN_LOCALES, MIN_SIMILARITY, MAX_FAR_LOCALES, MIN_LATIN_EVIDENCE } diff --git a/package.json b/package.json index 720e87a29..13fda8a0d 100644 --- a/package.json +++ b/package.json @@ -577,7 +577,7 @@ "check:pinned-umd": "library/utils/loadPinnedUmd.js (d3 for PPLR and the Wall) loads a pinned UMD with SRI and never touches the host page's global of the same name: a host's own d3 gains no keys, a failed load leaves it in place. Headless.", "check:popup-escaping": "the library's pin and region popups render hostile data (an , a