diff --git a/internal/check-group-set-standard.mjs b/internal/check-group-set-standard.mjs new file mode 100644 index 000000000..e1273741f --- /dev/null +++ b/internal/check-group-set-standard.mjs @@ -0,0 +1,81 @@ +#!/usr/bin/env node +/** + * check-group-set-standard — a FORK's group set generates packets end-to-end. + * + * THE CONTRACT UNDER TEST (documentation/STANDARD-GROUP-SET.md in the plugin): a fork supplies ONE + * FOLDER — clusters.json with a flat lists[] ({ id, label, selector, unlisted? }), records.json + * (its own people rows, so no API), selector-evaluator.js (resolvable from the folder) — and points + * the packet generator at it with DATA_DIR= and PACKET_OUT=. Nothing else. If that sentence stops + * being true, this gate goes red before a fork finds out. + * + * ★ THE FIXTURE IS THE STANDARD'S FALSIFIER, deliberately synthetic (every slug and number is + * invented — no real group data ships in a test fixture). It exercises exactly the generic shape: + * FLAT lists[] (not upg100's bucketed one), an unlisted:true entry that must NOT generate, records + * from the folder, and no posters directory — the map page must degrade to a cover, not kill the + * run, because a fork's first run will not have posters yet. + * + * Written RED-FIRST: on the tree before the standard landed, packet.mjs knew only series.json + * dataDirs, bucketed clusters, cached/API records and a fixed out/ — this test failed on all four. + * + * Headless, no network (NOPHOTOS + folder records). Exit 0 pass · 1 fail. + */ +import { resolve, dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { existsSync, readFileSync, readdirSync, rmSync, mkdtempSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { tmpdir } from 'node:os' + +const HERE = dirname(fileURLToPath(import.meta.url)) +const REPO = resolve(HERE, '..') +const FIXTURE = join(HERE, 'fixtures', 'toy-group-set') +const GEN = join(REPO, 'library', 'scripts', 'posters', 'packet.mjs') + +const fails = [] +const ok = (c, m) => { if (!c) fails.push(m) } + +// ⚠ OUTPUT GOES TO A TEMP DIR, NEVER THE REAL out/packets — toy packets landing beside the real 28 +// would make check:packets-all count fiction as coverage. +const OUT = mkdtempSync(join(tmpdir(), 'toy-packets-')) +const env = { + ...process.env, + DATA_DIR: FIXTURE, + PACKET_OUT: OUT, + NOPHOTOS: '1', + OFFLINE: '1', +} +const run = spawnSync('node', [GEN], { env, encoding: 'utf8', timeout: 120000 }) +const outText = String(run.stdout || '') + String(run.stderr || '') + +ok(run.status === 0, + `the generator exited ${run.status} on the toy dataDir — a fork cannot generate at all.\n tail: ${outText.trim().split('\n').slice(-4).join('\n ')}`) + +if (run.status === 0) { + const files = existsSync(OUT) ? readdirSync(OUT).filter((f) => f.endsWith('.html')).sort() : [] + // toy-hidden is unlisted:true — TWO packets, and the third's absence is as load-bearing as the + // two's presence: an unlisted list that generates anyway is the reachability bug inverted. + ok(files.includes('toy-north.html') && files.includes('toy-south.html'), + `expected toy-north + toy-south packets, got: ${files.join(', ') || '(none)'}`) + ok(!files.some((f) => f.startsWith('toy-hidden')), + 'the unlisted:true list generated a packet — unlisted must mean unlisted') + + for (const f of ['toy-north.html', 'toy-south.html']) { + if (!files.includes(f)) continue + const html = readFileSync(join(OUT, f), 'utf8') + const groups = (html.match(/class="card"/g) || []).length + const want = f === 'toy-north.html' ? 3 : 2 + ok(groups === want, `${f}: ${groups} group rows, expected ${want} — the folder's records.json did not drive the join`) + // The synthetic names must actually print — a packet of the right COUNT with the wrong rows + // would pass a bare count. + ok(f !== 'toy-north.html' || /Alpha People/.test(html), 'toy-north is missing its own synthetic people') + // No posters exist for the toy: the map page must be the documented cover, and the run must + // not have died for it (already asserted by exit 0). + } +} +rmSync(OUT, { recursive: true, force: true }) + +if (fails.length) { + console.error('GROUP-SET-STANDARD: FAIL') + for (const f of fails) console.error(` ✗ ${f}`) + process.exit(1) +} +console.log('GROUP-SET-STANDARD: PASS — a one-folder synthetic fork generates exactly its listed packets (and not its unlisted one) from its own records, with no API, no posters and no series.json edit.') diff --git a/internal/fixtures/toy-group-set/clusters.json b/internal/fixtures/toy-group-set/clusters.json new file mode 100644 index 000000000..9fd1d6a81 --- /dev/null +++ b/internal/fixtures/toy-group-set/clusters.json @@ -0,0 +1,9 @@ +{ + "_synthetic": "Toy group set for check:group-set-standard. Every slug, name and number is invented — no real people-group data. The FLAT lists[] shape is the fork contract; unlisted:true keeps an entry out of generation.", + "source_total": 8, + "lists": [ + { "id": "toy-north", "label": "Toy North", "selector": { "type": "pg_slugs", "values": ["toy-alpha", "toy-bravo", "toy-charlie"] } }, + { "id": "toy-south", "label": "Toy South", "selector": { "type": "pg_slugs", "values": ["toy-delta", "toy-echo"] } }, + { "id": "toy-hidden", "label": "Toy Hidden", "unlisted": true, "selector": { "type": "pg_slugs", "values": ["toy-foxtrot"] } } + ] +} diff --git a/internal/fixtures/toy-group-set/records.json b/internal/fixtures/toy-group-set/records.json new file mode 100644 index 000000000..6107654d3 --- /dev/null +++ b/internal/fixtures/toy-group-set/records.json @@ -0,0 +1,8 @@ +[ + { "slug": "toy-alpha", "name": "Alpha People", "country_code": "AAA", "country_code_label": "Toyland", "population": 1000, "religion": "SYN", "religion_label": "Synthetic", "primary_language_label": "Toyish", "imb_gsec": 1 }, + { "slug": "toy-bravo", "name": "Bravo People", "country_code": "AAA", "country_code_label": "Toyland", "population": 2000, "religion": "SYN", "religion_label": "Synthetic", "primary_language_label": "Toyish", "imb_gsec": 2 }, + { "slug": "toy-charlie", "name": "Charlie People", "country_code": "BBB", "country_code_label": "Boxland", "population": 3000, "religion": "SYN", "religion_label": "Synthetic", "primary_language_label": "Boxish", "imb_gsec": 0 }, + { "slug": "toy-delta", "name": "Delta People", "country_code": "BBB", "country_code_label": "Boxland", "population": 4000, "religion": "SYN", "religion_label": "Synthetic", "primary_language_label": "Boxish", "imb_gsec": 4 }, + { "slug": "toy-echo", "name": "Echo People", "country_code": "AAA", "country_code_label": "Toyland", "population": 5000, "religion": "SYN", "religion_label": "Synthetic", "primary_language_label": "Toyish", "imb_gsec": 6 }, + { "slug": "toy-foxtrot", "name": "Foxtrot People", "country_code": "AAA", "country_code_label": "Toyland", "population": 6000, "religion": "SYN", "religion_label": "Synthetic", "primary_language_label": "Toyish", "imb_gsec": 3 } +] diff --git a/internal/fixtures/toy-group-set/selector-evaluator.js b/internal/fixtures/toy-group-set/selector-evaluator.js new file mode 100644 index 000000000..1f95e0aa9 --- /dev/null +++ b/internal/fixtures/toy-group-set/selector-evaluator.js @@ -0,0 +1,4 @@ +// The toy dataDir carries its own evaluator per the contract. Re-exported rather than copied: +// the CONTRACT is "the folder resolves selector-evaluator.js"; the toy proves resolution, and a +// copy here would be a second evaluator that drifts (the exact defect class the standard avoids). +export { evaluate } from '../../../plugin-designs/my-upg-100-list/data/selector-evaluator.js' diff --git a/library/scripts/posters/packet.mjs b/library/scripts/posters/packet.mjs index 427f0c0be..5c3b58b2f 100644 --- a/library/scripts/posters/packet.mjs +++ b/library/scripts/posters/packet.mjs @@ -38,7 +38,7 @@ import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { dirname, join } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO = join(__dirname, '../../..'); @@ -60,8 +60,18 @@ if (!SERIES_ALL[SERIES_ID] || SERIES_ID.startsWith('_')) { const SERIES = SERIES_ALL[SERIES_ID]; // The data folder is a SERIES property, not a constant. It was hard-wired to my-upg-100-list, // which is the other half of why a country run could only ever be a UPG-100 run. -const DATA = join(REPO, SERIES.dataDir); -const OUT = join(__dirname, 'out', 'packets'); +// ★ DATA_DIR overrides the series folder entirely — the FORK CONTRACT (see the plugin's +// documentation/STANDARD-GROUP-SET.md): one folder with clusters.json + records.json + +// selector-evaluator.js generates packets with NO series.json edit. Series flags (funnel, +// strings, countryMode) still come from SERIES_ID, defaulting to upg100's. +const DATA = process.env.DATA_DIR ? resolve(process.env.DATA_DIR) : join(REPO, SERIES.dataDir); +if (process.env.DATA_DIR && !existsSync(join(DATA, 'clusters.json'))) { + console.error(`[packet] DATA_DIR set but ${join(DATA, 'clusters.json')} does not exist.`); + process.exit(1); +} +// PACKET_OUT keeps a fork's (or a test's) output out of the real artifact directory — toy packets +// beside the real ones would make check:packets-all count fiction as coverage. +const OUT = process.env.PACKET_OUT ? resolve(process.env.PACKET_OUT) : join(__dirname, 'out', 'packets'); mkdirSync(OUT, { recursive: true }); // ⚠ NOT process.env.LANG — that is the SHELL's POSIX locale ("en_US.UTF-8"), which @@ -84,7 +94,8 @@ const API = process.env.VITE_API_BASE_URL || process.env.MAP_APP_API_URL; // the program refuses. Any env var that makes the API unnecessary must be named HERE, next to the // guard, or it is decoration. const CACHE_OK = process.env.OFFLINE && existsSync(join(__dirname, 'out', '_records.json')); -if (!API && !process.env.PACKET_RECORDS && !CACHE_OK) { +const DATA_RECORDS_EARLY = process.env.DATA_DIR && existsSync(join(resolve(process.env.DATA_DIR), 'records.json')); +if (!API && !process.env.PACKET_RECORDS && !CACHE_OK && !DATA_RECORDS_EARLY) { console.error('[packet] No API base — set VITE_API_BASE_URL, or PACKET_RECORDS for a fixture build,'); console.error(' or OFFLINE=1 to reuse out/_records.json if a previous run cached it.'); if (process.env.OFFLINE) { @@ -226,16 +237,31 @@ function buildFunnel(membership) { // ("13 + 12 + 2 + 1 deaf = 28"). The packet's triangle tip prints this number as "1 of N", so a // 27 there is a visible wrong fact on the page. Same omission as allZones() had before 3ef164c0; // this was the last copy of it. - const listCount = allZones().length; + const listCount = ZONES_ALL.length; const listTotal = Number(raw.steps[raw.steps.length - 1].groups) || 0; return { ...raw, listCount, listTotal }; } catch (_) { return null; } } const { evaluate } = await import(join(DATA, 'selector-evaluator.js')); -const { normalizeRecords, extractRecords } = await import(join(DATA, 'normalize.js')); +// normalize.js is OPTIONAL in the fork contract: a fork's records.json is its own shape, already +// flat. Absent -> identity, said OUT LOUD once — a silent fallback would let a TYPO'd normalize.js +// quietly skip normalization on the real series, which is the fail-open trap. +const { normalizeRecords, extractRecords } = await (async () => { + try { return await import(join(DATA, 'normalize.js')); } + catch { + console.log('[packet] no normalize.js in the data folder — records are used as-is (fork contract).'); + return { normalizeRecords: (r) => r, extractRecords: (r) => (Array.isArray(r) ? r : (r?.posts || r?.records || [])) }; + } +})(); // The zone enumeration itself, so this generator, the poster generator and upg-100-list.vue cannot // disagree about which zones exist (see allZones below). -const { buildLegendNodes, leafZones } = await import(join(DATA, 'legend-tree.js')); +// legend-tree.js is loaded LAZILY, only for the bucketed (upg100) shape — a flat fork's file IS +// its enumeration and must not be forced to ship a legend module it never uses. +let _legend = null; +async function legendModules() { + if (!_legend) _legend = await import(join(DATA, 'legend-tree.js')); + return _legend; +} async function loadRecords() { // ★ A COMMITTED FIXTURE, so CI can build a packet at all. out/ is gitignored — both the @@ -253,6 +279,10 @@ async function loadRecords() { if (!existsSync(fixture)) throw new Error(`PACKET_RECORDS set but missing: ${fixture}`); return JSON.parse(readFileSync(fixture, 'utf8')); } + // The FORK CONTRACT's records: a dataDir may carry its own rows, so a fork needs no API and no + // cache. Detected, not configured — the folder IS the interface. + const folderRecords = join(DATA, 'records.json'); + if (existsSync(folderRecords)) return JSON.parse(readFileSync(folderRecords, 'utf8')); const cache = join(__dirname, 'out', '_records.json'); if (process.env.OFFLINE && existsSync(cache)) return JSON.parse(readFileSync(cache, 'utf8')); // ★ THE FIELD LIST IS PART OF THE SELECTOR CONTRACT — every field a selector READS must be asked @@ -310,7 +340,7 @@ async function loadRecords() { // // leafZones() carries id/zoneKey/label/count but NOT selector (leaves do not), so the selector is // re-attached by id exactly as generate.mjs:2801-2808 does it, deaf included. -function allZones() { +async function allZones() { // ★ COUNTRY MODE HAS ITS OWN ZONE SOURCE, and leaving it out broke the packet half of // SERIES=country for every country. Measured by exp-shepherd on main, 2026-09-28: // `SERIES=country ONLY=country-yem node generate.mjs` made a poster, and the matching @@ -348,6 +378,18 @@ function allZones() { selector: { type: 'country_codes', values: [c.iso3] }, })); } + // ★ THE FORK CONTRACT'S GENERIC SHAPE: a FLAT lists[] with an optional unlisted:true per entry. + // Detected by the ABSENCE of upg100's extra buckets, so the 28-zone catalogue keeps its legend- + // tree enumeration (the deaf collapse and its reachability semantics) byte-for-byte, and a fork + // never needs a legend tree at all — its file IS the enumeration. unlisted:true is the same + // reachability idea as data: defined, selectable by ONLY=, but not generated by default. + const BUCKETED = !!(clusters.asia_lists || clusters.uncovered_lists || clusters.deaf_lists); + if (!BUCKETED) { + return (clusters.lists || []) + .filter((l) => !l.unlisted || (process.env.ONLY && process.env.ONLY === l.id)) + .map((l) => ({ id: l.id, zoneKey: l.id, label: l.label || l.id, count: l.count, selector: l.selector })); + } + const { buildLegendNodes, leafZones } = await legendModules(); const tops = buildLegendNodes(clusters, { records: null }); const byId = {}; for (const k of ['lists', 'asia_lists', 'uncovered_lists']) { @@ -456,9 +498,15 @@ if (LIST_MEMBERSHIP) { console.log(`[packet] list=${LIST} — ${records.length} of ${allRecords.length} record(s) ` + `belong to it (list total ${LIST_MEMBERSHIP.total})`); } +// Resolved ONCE, and it MUST precede FUNNEL: packetFunnel reads ZONES_ALL.length inside a +// try/catch whose catch returns null — so a use-before-init here does not crash, it silently +// deletes the funnel page from every packet. That exact failure shipped for one run: the tip +// probe found no "1 of 28" anywhere, which is how a swallowed TDZ was caught as missing CONTENT +// rather than as an error. +const ZONES_ALL = await allZones(); const FUNNEL = buildFunnel(LIST_MEMBERSHIP); const bySlug = new Map(records.map((r) => [r.slug, r])); -const zones = allZones().filter((z) => !ONLY || z.id === ONLY); +const zones = ZONES_ALL.filter((z) => !ONLY || z.id === ONLY); if (!zones.length) { console.error(`[packet] no zone matches ONLY=${ONLY}`); process.exit(1); } let made = 0; diff --git a/plugin-designs/my-upg-100-list/documentation/STANDARD-GROUP-SET.md b/plugin-designs/my-upg-100-list/documentation/STANDARD-GROUP-SET.md new file mode 100644 index 000000000..addc99f0f --- /dev/null +++ b/plugin-designs/my-upg-100-list/documentation/STANDARD-GROUP-SET.md @@ -0,0 +1,63 @@ +# STANDARD — generate packets for ANY group set (the fork contract) + +The UPG-100 machinery is generative: point the packet generator at **one folder** and it produces +a packet per list, localized, with no code change and no `series.json` edit. This document is the +contract; `internal/check-group-set-standard.mjs` is its falsifier (a synthetic 3-list toy set runs +end-to-end on every gate pass, so the contract cannot rot silently). + +## What a fork supplies — one folder + +``` +my-set/ + clusters.json REQUIRED the group-set file (schema below) + records.json OPTIONAL your own people rows — with it, no API and no cache are needed + selector-evaluator.js REQUIRED resolvable from the folder (re-export ours, or ship your own) + normalize.js OPTIONAL absent -> records are used as-is (logged once, never silent) + legend-tree.js ONLY for the bucketed legacy shape; a flat set never needs it +``` + +### clusters.json — the generic (flat) shape + +```json +{ "lists": [ + { "id": "my-north", "label": "My North", "selector": { "type": "pg_slugs", "values": ["a","b"] } }, + { "id": "my-draft", "label": "Draft", "unlisted": true, "selector": { … } } +] } +``` + +- `selector` — any type your evaluator handles (`pg_slugs` needs nothing but slugs). +- `unlisted: true` — defined and selectable with `ONLY=`, but not generated by default. + This is the reachability idea as data. +- The bucketed shape (`lists` + `asia_lists` + `uncovered_lists` + `deaf_lists`) is the UPG-100 + catalogue's legacy form; its enumeration goes through the legend tree to keep the deaf collapse. + New sets use the flat form. + +## Running it + +``` +DATA_DIR=path/to/my-set PACKET_OUT=out/my-packets PACKET_LANG=es node library/scripts/posters/packet.mjs +ONLY=my-north … # one list NOPHOTOS=1 … # skip photo fetch +``` + +- **Locale**: `PACKET_LANG` (never the shell's `LANG`); strings come from `packet-strings.json` + per-locale blocks with per-key fallback — a new language is a new block, not a code change. +- **Titles**: the per-list `label` + your strings block. Nothing else to configure. +- `PACKET_OUT` keeps fork/test output away from the real artifact directory. + +## Pages that depend on optional repo data (they omit cleanly, never fail the run) + +| page | needs | absent → | +|---|---|---| +| map (page 1) | `out/posters/.html` (`bun run posters`) | cover without the map | +| triangle | `data/people-groups/vision-funnel.json` | page omitted | +| %-evangelical | `data/people-groups/christian-share.json` | page omitted | +| bounded/centered sets | `public/dx-maps/assets/tables/near-neighbors.json` joined by slug | page omitted | + +A fork's first run therefore yields: cover + profiles (+ triangle if the repo file is present) — +correct and smaller, per the rule that a page which cannot state true numbers must not appear. + +## Not in scope + +Posters for arbitrary sets (the poster generator needs geo/bbox inputs this contract does not +define yet), multi-territory maps, and authoring UI. The in-app catalogue is the plugin's own +`data/clusters.json` — a fork edits its copy; the file is the interface.