The dx-maps tree ships only the data tables a map uses: 20 build-only tables stay in data/tables, a gate keeps it so, and an unused zone loader goes
This commit is contained in:
Doxa Maps
parent
789d90eb2f
commit
312a7956a7
4 changed files with 87 additions and 62 deletions
72
internal/check-tables-shipped-are-used.mjs
Normal file
72
internal/check-tables-shipped-are-used.mjs
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* check-tables-shipped-are-used.mjs — the dx-maps tree ships only the data tables a map uses.
|
||||
*
|
||||
* WHY: generate-tree copies data/tables into <tree>/assets/tables as a folder, so a table that only a BUILD
|
||||
* script reads (a source index, a fetch input) shipped anyway: 20 such tables, 4.3 MB, named by nothing in
|
||||
* the tree. The research stays in data/tables; the tree carries what a map fetches.
|
||||
*
|
||||
* TWO DIRECTIONS, both on the BUILT tree:
|
||||
* (a) every shipped table is NAMED by a shipped bundle, page or manifest (its file name, or its stem, which
|
||||
* is how a config passes `${assetsBase}/tables/${file}.json`). The card shards are one family, named by
|
||||
* their `card-` pattern.
|
||||
* (b) every literal `tables/<name>.json` a shipped file fetches EXISTS in the tree, so a skip list can never
|
||||
* take away a table a map needs.
|
||||
* `--poison` plants an unnamed table into the shipped list and drops a fetched one; both must be caught.
|
||||
* `--tree <dir>` checks another built tree (default public/dx-maps). Exit 0 pass · 1 fail · 75 not measured.
|
||||
*/
|
||||
import { readdirSync, readFileSync, statSync, existsSync } from 'node:fs'
|
||||
import { join, resolve, extname } from 'node:path'
|
||||
|
||||
const argv = process.argv.slice(2)
|
||||
const TREE = resolve(argv.includes('--tree') ? argv[argv.indexOf('--tree') + 1] : 'public/dx-maps')
|
||||
const POISON = argv.includes('--poison')
|
||||
const TABLES = join(TREE, 'assets', 'tables')
|
||||
if (!existsSync(TABLES)) { console.log(`TABLES-SHIPPED-ARE-USED: NOT MEASURED — no built tree at ${TABLES}`); process.exit(75) }
|
||||
|
||||
// Everything the tree ships that could NAME a table: its code, pages and manifests (never the tables themselves).
|
||||
const TEXT = new Set(['.js', '.mjs', '.html', '.json'])
|
||||
let corpus = ''
|
||||
;(function walk(dir) {
|
||||
for (const e of readdirSync(dir)) {
|
||||
const p = join(dir, e)
|
||||
if (p === TABLES) continue
|
||||
const st = statSync(p)
|
||||
if (st.isDirectory()) walk(p)
|
||||
else if (TEXT.has(extname(e))) corpus += readFileSync(p, 'utf8') + '\n'
|
||||
}
|
||||
})(TREE)
|
||||
|
||||
const CARD = /^card-[0-9a-f]+\.json$/
|
||||
const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
const named = (file) => new RegExp(`["'\`/]${esc(file.replace(/\.json$/, ''))}(\\.json)?["'\`]`).test(corpus)
|
||||
const cardsNamed = /tables\/card-|card-\$\{|["'`]card-["'`]\s*\+/.test(corpus)
|
||||
const fetched = [...new Set([...corpus.matchAll(/tables\/([a-z0-9][a-z0-9_-]*)\.json/g)].map((m) => m[1] + '.json'))]
|
||||
|
||||
function verdict(shipped) {
|
||||
const unnamed = shipped.filter((f) => !CARD.test(f) && !named(f))
|
||||
const cards = shipped.filter((f) => CARD.test(f))
|
||||
const cardFail = cards.length && !cardsNamed
|
||||
const missing = fetched.filter((f) => !shipped.includes(f))
|
||||
return { unnamed, cards: cards.length, cardFail, missing }
|
||||
}
|
||||
|
||||
const shipped = readdirSync(TABLES).filter((f) => f.endsWith('.json'))
|
||||
if (POISON) {
|
||||
const firstFetched = fetched.find((f) => shipped.includes(f))
|
||||
const r = verdict([...shipped.filter((f) => f !== firstFetched), 'poison-unnamed-table.json'])
|
||||
const caughtA = r.unnamed.includes('poison-unnamed-table.json'), caughtB = !!firstFetched && r.missing.includes(firstFetched)
|
||||
console.log(`poison (a) a shipped table nothing names: ${caughtA ? 'caught' : 'MISSED'}`)
|
||||
console.log(`poison (b) a fetched table not shipped (${firstFetched || 'none fetched'}): ${caughtB ? 'caught' : 'MISSED'}`)
|
||||
console.log(caughtA && caughtB ? 'POISON OK: 2 caught' : 'POISON FAILED')
|
||||
process.exit(caughtA && caughtB ? 0 : 1)
|
||||
}
|
||||
|
||||
const r = verdict(shipped)
|
||||
console.log(`tree ${TREE}: ${shipped.length} table files (${r.cards} card shards${r.cards ? (cardsNamed ? ', named by their card- pattern' : ', NOT named') : ''}); ${fetched.length} fetched by a literal tables/ path`)
|
||||
for (const f of r.unnamed) console.log(`FAIL shipped but named by nothing in the tree: ${f}`)
|
||||
if (r.cardFail) console.log('FAIL card shards ship but no shipped file names the card- pattern')
|
||||
for (const f of r.missing) console.log(`FAIL fetched as tables/${f} but not shipped`)
|
||||
const n = r.unnamed.length + (r.cardFail ? 1 : 0) + r.missing.length
|
||||
console.log(n ? `TABLES-SHIPPED-ARE-USED: FAIL — ${n} problem(s)` : 'TABLES-SHIPPED-ARE-USED: PASS — every shipped table is named, every fetched one ships')
|
||||
process.exit(n ? 1 : 0)
|
||||
|
|
@ -2007,7 +2007,18 @@ function main() {
|
|||
// internal-only files dropin.exclude already names, so a new table works with no
|
||||
// build change. Kept as a directory copy because the fetch name is dynamic
|
||||
// (`${assetsBase}/tables/${file}.json`).
|
||||
const TABLES_SKIP = new Set(['open-doors.json', 'platform-reachability.json', 'index.json'])
|
||||
// BUILD-ONLY TABLES stay in data/tables (the research sources the build scripts read; their content is already
|
||||
// in the tables the maps fetch) and are not shipped: no shipped bundle, page or manifest names them.
|
||||
// check:tables-shipped-are-used fails on a shipped table nothing names, and on a fetched one that doesn't ship.
|
||||
// languages-per-city and key-cities-admin-n belong to key-cities, a gated bundle: un-skip them if it ships.
|
||||
const TABLES_BUILD_ONLY = [
|
||||
'arclight-index.json', 'arclight-share-urls.json', 'country-languages.json', 'door43-index.json',
|
||||
'doxa-detail.json', 'ebible-index.json', 'fcbh-index.json', 'godtools-index.json',
|
||||
'gospel-by-province-groups.json', 'grn-index.json', 'groupings.json', 'iso-macrolanguages.json',
|
||||
'key-cities-admin-n.json', 'languages-per-city.json', 'scriptureearth-index.json', 'sun-deaf-index.json',
|
||||
'translation-credits.json', 'translator-org-links.json', 'zone-regions.json', 'zume-index.json',
|
||||
]
|
||||
const TABLES_SKIP = new Set(['open-doors.json', 'platform-reachability.json', 'index.json', ...TABLES_BUILD_ONLY])
|
||||
const tablesSrc = join(pubAssets, 'tables')
|
||||
if (existsSync(tablesSrc)) {
|
||||
const tablesDest = join(treeAssets, 'tables')
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@
|
|||
"check:gallery-embed:live-cors": "node internal/check-research-gallery-embed.mjs --live-cors",
|
||||
"check:gallery-embed:maplibre": "node internal/check-research-gallery-embed.mjs --provider maplibre",
|
||||
"check:gated-not-in-tree": "node internal/check-gated-not-in-tree.mjs",
|
||||
"check:tables-shipped-are-used": "node internal/check-tables-shipped-are-used.mjs && node internal/check-tables-shipped-are-used.mjs --poison",
|
||||
"check:gospel-all": "node internal/check-gospel-all.mjs",
|
||||
"check:gospel-internet": "node plugin-designs/gospel-internet/scripts/check-gospel-internet-model.mjs && node plugin-designs/gospel-internet/scripts/check-gospel-internet-i18n.mjs && node plugin-designs/gospel-internet/scripts/check-language-tree.mjs && node plugin-designs/gospel-internet/scripts/check-gospel-internet-render.mjs",
|
||||
"check:gospel-tables-staged": "node internal/check-gospel-tables-staged.mjs",
|
||||
|
|
@ -358,7 +359,7 @@
|
|||
"dev": "bun x vite",
|
||||
"fix:sri": "node internal/fix-sri.mjs",
|
||||
"gallery": "node internal/generate-gallery.mjs",
|
||||
"gates:post": "node internal/run-parallel.mjs --label=post -- check:capability-derived check:drop check:parse verify check:sri check:public-hygiene check:label-joins check:facet-cascade check:facet-one-field check:heartbeat check:kpi-unit check:composite-reg check:descriptions check:no-abs-paths check:no-codename check:dep-names check:composite-srcs check:embed-targets check:nested-token check:i18n-parity check:tabs-localized check:packets-all check:packet-strings-parity check:zone-census check:zoning-declared check:share-policy check:library-index check:csv-shape check:capability-citations check:catalog-identity check:no-orphan-gates check:quoted-scripts check:script-flags check:capability-exemplars check:packet-fixture check:packet-browser check:provider-switch check:locale-reachable check:url-resolver check:manifest-covers check:one-palette check:pin-dim-slider check:legend-source check:clustering-modes check:composite-embed-knobs check:composite-locale-leaks check:linked-vocabulary check:wp-embed check:dashboard-geocoder-provider check:leaf-engine-per-provider check:auto-engine check:engine-contract check:map-props-unit check:composites-mount check:composite-interop check:no-free-identifiers check:map-fly-reduced-motion check:mst-clustering check:list-prop-unit check:profiles-mount check:cdn-pins check:no-eval check:popup-escaping check:i18n-treeshake check:pinned-umd check:one-palette:self-test check:no-cross-plugin-imports check:no-cross-plugin-imports:self-test check:map-attribution check:studio-rail check:approx-pin-note check:embedder-keys check:provider-for check:legend-logical-insets check:legend-logical-insets:self-test check:optional-t-equivalence check:text-highlight check:religion-label-not-value check:i18n-stopped-scope check:no-silent-engine check:no-silent-engine:self-test check:regions-uses-library check:shadow-css-parses check:square-map-controls check:preset-config-merge check:instance-config-keys check:frozen-labels check:plural-forms check:region-fill-repaint check:region-paint-single-source check:poster-print-fallback check:posters-exportable check:country-names check:reduced-motion check:no-dead-library check:universal-props check:readme-size check:tree-relative check:leaf-hints check:snippet-engine check:snippet-engine:self-test check:embed-base check:link-coverage check:no-public-binds check:shared-reached check:tree-weight check:no-fleet-names:self-test check:pin-hint-floor:self-test check:poster-sizes:self-test check:shell-header:self-test check:preview-flag-dev-only:self-test check:poster-type-registry:self-test check:host-floor:self-test check:phone-print-keep:self-test check:pinned-fetch-fresh:self-test",
|
||||
"gates:post": "node internal/run-parallel.mjs --label=post -- check:capability-derived check:drop check:parse verify check:sri check:public-hygiene check:tables-shipped-are-used check:label-joins check:facet-cascade check:facet-one-field check:heartbeat check:kpi-unit check:composite-reg check:descriptions check:no-abs-paths check:no-codename check:dep-names check:composite-srcs check:embed-targets check:nested-token check:i18n-parity check:tabs-localized check:packets-all check:packet-strings-parity check:zone-census check:zoning-declared check:share-policy check:library-index check:csv-shape check:capability-citations check:catalog-identity check:no-orphan-gates check:quoted-scripts check:script-flags check:capability-exemplars check:packet-fixture check:packet-browser check:provider-switch check:locale-reachable check:url-resolver check:manifest-covers check:one-palette check:pin-dim-slider check:legend-source check:clustering-modes check:composite-embed-knobs check:composite-locale-leaks check:linked-vocabulary check:wp-embed check:dashboard-geocoder-provider check:leaf-engine-per-provider check:auto-engine check:engine-contract check:map-props-unit check:composites-mount check:composite-interop check:no-free-identifiers check:map-fly-reduced-motion check:mst-clustering check:list-prop-unit check:profiles-mount check:cdn-pins check:no-eval check:popup-escaping check:i18n-treeshake check:pinned-umd check:one-palette:self-test check:no-cross-plugin-imports check:no-cross-plugin-imports:self-test check:map-attribution check:studio-rail check:approx-pin-note check:embedder-keys check:provider-for check:legend-logical-insets check:legend-logical-insets:self-test check:optional-t-equivalence check:text-highlight check:religion-label-not-value check:i18n-stopped-scope check:no-silent-engine check:no-silent-engine:self-test check:regions-uses-library check:shadow-css-parses check:square-map-controls check:preset-config-merge check:instance-config-keys check:frozen-labels check:plural-forms check:region-fill-repaint check:region-paint-single-source check:poster-print-fallback check:posters-exportable check:country-names check:reduced-motion check:no-dead-library check:universal-props check:readme-size check:tree-relative check:leaf-hints check:snippet-engine check:snippet-engine:self-test check:embed-base check:link-coverage check:no-public-binds check:shared-reached check:tree-weight check:no-fleet-names:self-test check:pin-hint-floor:self-test check:poster-sizes:self-test check:shell-header:self-test check:preview-flag-dev-only:self-test check:poster-type-registry:self-test check:host-floor:self-test check:phone-print-keep:self-test check:pinned-fetch-fresh:self-test",
|
||||
"gates:pre": "node internal/run-parallel.mjs --label=pre -- check:template-literal check:mjs-parse check:sfc-parse check:tab-radius check:synced-css check:doc-scripts check:setup-order check:package-json-unique-keys check:no-repo-pb-hooks check:no-credentials-omit check:no-credentials-omit:self-test check:shadow-reduced-motion check:poster-action-print check:targets check:targets:self-test check:browser-gates-guarded check:pin-selection check:facet-shape check:hardcoded-english:self-test check:list-selection check:selector-fields check:upg-zone-joins check:gospel-set-charts check:gospel-set-fit check:describe-group check:one-scroll-container check:packet-charts check:pins-in-state check:fork-leaders check:circular-layout check:composite-token-url check:language-tree-layout check:i18n-keys-exist check:i18n-never-translated check:packet-variants check:profile-table check:element-bootstrap check:embed-url-absolute check:frame-target check:language-tree-model check:map-menu-anchor check:selection-kinds check:tiered-points check:whiteboard-projection check:mercator-slice check:ring-vocabulary check:map-props-unit check:mobile-gating check:provide-shadowing check:legend-top-offset check:cdn-cache check:plural-forms check:select-zone-race check:script-registry check:map-control-overlap check:poster-attribution check:board-map-options check:gospel-whiteboard check:packet-map-missing-loud check:poster-layout-optin check:ported-globals-identical check:ring-bands check:packet-a1-structure check:india-states check:batch-selection-export check:zone-title-tier-a check:evangelical-share check:map-labels check:packet-offline check:help-reachable check:gated-not-in-tree check:window-globals-released check:legend-data-contract check:no-internal-role-nouns check:poster-fetch-guard check:people-group-records check:sentence-slots check:popup-href-escaped check:popup-pan-controls check:country-map-refetch check:pane-config-reforward check:viewport-meta check:viewport-meta:self-test check:label-key-coverage check:poster-browser-equiv check:poster-render-pinned check:country-sheet-one-path check:country-sheet-one-path:self-test check:side-lanes check:side-lanes:self-test check:poster-section check:poster-section:self-test check:posterverse-shelf check:host-locale-fetch check:print-frame-reclaim check:click-sheet check:coverage-consistent:chained check:coverage-consistent:poison check:dashboard-filter-a11y check:filter-multivalue check:hover-channel check:kit-options-reachable check:layout-key-namespaced check:legend-mirrors check:map-unmount-race check:no-conflict-markers check:series-isolation check:snapshot-guard check:stale-sweep-claim check:surfaces-read-the-tier check:translator-contacts check:wall-wysiwyg check:wall-print-watch check:wall-no-bg-photos check:wall-pan check:template-i18n check:list-pattern-doc check:shadow-style-labels-unique check:grid-image-loading check:retry-fetch check:panel-sheets check:poster-wall-host-seams check:poster-sections check:poster-sections:self-test check:poster-design-catalogue check:poster-design-catalogue:self-test check:upg100-zone-copy check:upg100-zone-copy:self-test check:wall-photo-weight check:no-fleet-names check:pin-hint-floor check:poster-sizes check:upg-count check:shell-header check:pg-name-gate check:preview-flag-dev-only check:poster-type-registry check:host-floor check:phone-print-keep check:pinned-fetch-fresh",
|
||||
"gen:iso3-to-alpha2": "node library/scripts/generate-iso3-to-alpha2.mjs",
|
||||
"internet:adm1": "node plugin-designs/cities-internet/scripts/build-adm1-internet.mjs",
|
||||
|
|
@ -528,6 +529,7 @@
|
|||
"check:gallery-embed": "the cap-5 Research Gallery must embed on a FOREIGN ORIGIN (a stand-in host page on localhost, the built tree on 127.0.0.1) in BOTH modes, bare <script>+<doxa-maps-spa> and <iframe>, with the layout inside the host box and every curated tab painting a map. The summary line prints the tab count PER MODE, and a mode that checked zero tabs is a FAIL. One mode only: check:gallery-embed:js or check:gallery-embed:iframe. Starts its own servers on OS-assigned ports; needs only a built tree. Exit 2 = SKIP (no built tree / no playwright).",
|
||||
"check:gallery-embed:live-cors": "the Research Gallery on a foreign page under the LIVE static layer's CORS rule (Access-Control-Allow-Origin on files inside the tree's folders, NOT on the root manifest.json, measured on the live test host): it must still boot every tab, by reading the CORS-readable manifest copy generate-tree writes beside its bundle.",
|
||||
"check:gated-not-in-tree": "a GATED profile (the build config's gatedProfiles, today key-cities) never reaches the public tree: no public/<tree>/<name>/ folder, no manifest.json entry, no page referencing <name>/<name>.js. vite.config.js builds a gated bundle into .gated/<tree>/ (ignored by git). Data tables are out of scope on purpose: key-cities.json also feeds the production Cities with Internet map. Two-sided: three planted leaks in a temporary tree, each caught. Node only.",
|
||||
"check:tables-shipped-are-used": "the dx-maps tree ships only the data tables a map uses: every table in <tree>/assets/tables is named by a shipped bundle, page or manifest (its file name or stem; the card shards as one family by their card- pattern), and every literal tables/<name>.json a shipped file fetches exists. generate-tree used to copy the whole data/tables folder, so 20 build-only tables (4.3 MB of source indexes and fetch inputs) shipped unread; they stay in data/tables. Two-sided: --poison plants an unnamed table and drops a fetched one, and both must be caught.",
|
||||
"check:gospel-all": "EVERYTHING gospel-resources owns, in one command. Its ten gates were all hand-run, which is how a dead bundle shipped: a duplicate module killed all five leaves and the one gate run that day was pointed at the map. check:no-orphan-gates was right that they were reachable — reachable is not run. Deliberately NOT in gates:pre/post: three need a browser, the shared wave holds browser gates under load, and a bundle owner should not be able to slow everyone else's build by adding to a list they do not own. Static gates run first and stop the run on failure, so a scarce browser slot is never spent confirming a table is inconsistent. Exit 75 = NOT MEASURED, never folded into a pass.",
|
||||
"check:gospel-internet": "the Gospel x Internet board, headless, against the REAL table and the REAL kit bus: the join table is what its builder makes from main (2,107 / 2,107, province placement agreeing with gospel-by-province), every facet count equals the rows the bus then keeps (the five-numbers-agree invariant), D1 OR-within-key, the country->province cascade, the 369 zero-access headline by band 220/45/45/59, URL state round-trip, CSV header-first/provenance-last with injection neutralised, no engagement column; then every catalogue key the bundle names exists in all 11 locales and is translated. Each half has a --poison arm that must fail. Plugin-local because check:i18n-keys-exist cannot see a useOptionalT-based alias.",
|
||||
"check:gospel-whiteboard": "the gospel set charts' geometry once drawn on a map: the bounded set's gap survives projection, the drawing stays inside the Mercator clamp, and three more properties, each with the arm that would catch its own defect. By the gospel owner. Headless.",
|
||||
|
|
|
|||
|
|
@ -1,60 +0,0 @@
|
|||
/**
|
||||
* zoneData.js — fetch the published UPG-100 zone tables and build the legend's tree.
|
||||
*
|
||||
* The tables belong to upg100 and are now published: L62 moved them into data/tables/, so the
|
||||
* tree copies them into the asset pool and both plugins read ONE file. That is why this fetches
|
||||
* rather than imports — a relative import into another plugin's folder is refused by
|
||||
* check:no-cross-plugin-imports, and a per-plugin copy is the drift the single-source move exists
|
||||
* to prevent.
|
||||
*
|
||||
* FETCHED ONCE and shared, like the neighbour table beside it.
|
||||
*
|
||||
* ⚠ THE TREE IS READY; THE FILTER IS NOT. Mapping a GROUP to a zone still runs through
|
||||
* admin-to-list.json, which exp measured as an older partition — 69 groups land in the wrong
|
||||
* printable zone and 436 in Unassigned. That is row 87. So this builds and counts
|
||||
* the tree, and the caller must not filter rows on it until the fold lands.
|
||||
*/
|
||||
import { resolveAssetsBase } from '@map/utils/embedBase.js'
|
||||
import { buildZoneTree } from './zoneTree.js'
|
||||
|
||||
let pending = null
|
||||
|
||||
/** { nodes, allZoneIds, definitions } from the published tables. Rejects loudly. */
|
||||
export function loadZoneTree(cfg = {}, counts = {}) {
|
||||
if (!pending) {
|
||||
pending = (async () => {
|
||||
const base = resolveAssetsBase(cfg)
|
||||
const [zr, gr] = await Promise.all([
|
||||
fetch(cfg.zoneRegionsUrl || `${base}/tables/zone-regions.json`).then(okJson),
|
||||
fetch(cfg.groupingsUrl || `${base}/tables/groupings.json`).then(okJson),
|
||||
])
|
||||
// ★ groupings.label IS A DEFINITION, NOT A NAME, and using it as one would make the
|
||||
// legend unreadable. Measured on the published file: the labels run from 7 to 206
|
||||
// characters, and the long ones are the zone's MEMBER LIST — Tibet's reads "Tibet
|
||||
// Autonomous Region + Sichuan Province + Xinjiang Uyghur Autonomous Region + ... +
|
||||
// North Korea". Only one of the fourteen is a plain name.
|
||||
// So the tree's NAME comes from the slug, which is uniform by construction, and the
|
||||
// label is carried separately as the zone's DEFINITION — which is what it actually is,
|
||||
// and useful where there is room to say what a zone contains.
|
||||
const definitions = {}
|
||||
for (const [id, g] of Object.entries(gr || {})) if (g && g.label) definitions[id] = g.label
|
||||
return { zoneRegions: zr || {}, definitions }
|
||||
})()
|
||||
}
|
||||
return pending.then(({ zoneRegions, definitions }) => ({
|
||||
zoneRegions,
|
||||
definitions,
|
||||
// No `labels` passed: buildZoneTree falls back to humanising the slug, which gives every
|
||||
// zone a name of the same shape rather than one name and thirteen member lists.
|
||||
nodes: buildZoneTree(zoneRegions, { counts }),
|
||||
// EVERY zone the data can name, not just the ones filed under a region. The legend renders
|
||||
// the difference as a named bucket; passing only the tree's ids would drop the unfiled ones
|
||||
// with no error and no empty state — just a shorter list.
|
||||
allZoneIds: Object.keys(zoneRegions),
|
||||
}))
|
||||
}
|
||||
|
||||
function okJson(res) {
|
||||
if (!res.ok) throw new Error(`${res.status} ${res.url}`)
|
||||
return res.json()
|
||||
}
|
||||
Loading…
Reference in a new issue