diff --git a/internal/check-tables-shipped-are-used.mjs b/internal/check-tables-shipped-are-used.mjs new file mode 100644 index 000000000..a4b37603c --- /dev/null +++ b/internal/check-tables-shipped-are-used.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +/** + * check-tables-shipped-are-used.mjs — the dx-maps tree ships only the data tables a map uses. + * + * WHY: generate-tree copies data/tables into /assets/tables as a folder, so a table that only a BUILD + * script reads (a source index, a fetch input) shipped anyway: 20 such tables, 4.3 MB, named by nothing in + * the tree. The research stays in data/tables; the tree carries what a map fetches. + * + * TWO DIRECTIONS, both on the BUILT tree: + * (a) every shipped table is NAMED by a shipped bundle, page or manifest (its file name, or its stem, which + * is how a config passes `${assetsBase}/tables/${file}.json`). The card shards are one family, named by + * their `card-` pattern. + * (b) every literal `tables/.json` a shipped file fetches EXISTS in the tree, so a skip list can never + * take away a table a map needs. + * `--poison` plants an unnamed table into the shipped list and drops a fetched one; both must be caught. + * `--tree ` checks another built tree (default public/dx-maps). Exit 0 pass · 1 fail · 75 not measured. + */ +import { readdirSync, readFileSync, statSync, existsSync } from 'node:fs' +import { join, resolve, extname } from 'node:path' + +const argv = process.argv.slice(2) +const TREE = resolve(argv.includes('--tree') ? argv[argv.indexOf('--tree') + 1] : 'public/dx-maps') +const POISON = argv.includes('--poison') +const TABLES = join(TREE, 'assets', 'tables') +if (!existsSync(TABLES)) { console.log(`TABLES-SHIPPED-ARE-USED: NOT MEASURED — no built tree at ${TABLES}`); process.exit(75) } + +// Everything the tree ships that could NAME a table: its code, pages and manifests (never the tables themselves). +const TEXT = new Set(['.js', '.mjs', '.html', '.json']) +let corpus = '' +;(function walk(dir) { + for (const e of readdirSync(dir)) { + const p = join(dir, e) + if (p === TABLES) continue + const st = statSync(p) + if (st.isDirectory()) walk(p) + else if (TEXT.has(extname(e))) corpus += readFileSync(p, 'utf8') + '\n' + } +})(TREE) + +const CARD = /^card-[0-9a-f]+\.json$/ +const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +const named = (file) => new RegExp(`["'\`/]${esc(file.replace(/\.json$/, ''))}(\\.json)?["'\`]`).test(corpus) +const cardsNamed = /tables\/card-|card-\$\{|["'`]card-["'`]\s*\+/.test(corpus) +const fetched = [...new Set([...corpus.matchAll(/tables\/([a-z0-9][a-z0-9_-]*)\.json/g)].map((m) => m[1] + '.json'))] + +function verdict(shipped) { + const unnamed = shipped.filter((f) => !CARD.test(f) && !named(f)) + const cards = shipped.filter((f) => CARD.test(f)) + const cardFail = cards.length && !cardsNamed + const missing = fetched.filter((f) => !shipped.includes(f)) + return { unnamed, cards: cards.length, cardFail, missing } +} + +const shipped = readdirSync(TABLES).filter((f) => f.endsWith('.json')) +if (POISON) { + const firstFetched = fetched.find((f) => shipped.includes(f)) + const r = verdict([...shipped.filter((f) => f !== firstFetched), 'poison-unnamed-table.json']) + const caughtA = r.unnamed.includes('poison-unnamed-table.json'), caughtB = !!firstFetched && r.missing.includes(firstFetched) + console.log(`poison (a) a shipped table nothing names: ${caughtA ? 'caught' : 'MISSED'}`) + console.log(`poison (b) a fetched table not shipped (${firstFetched || 'none fetched'}): ${caughtB ? 'caught' : 'MISSED'}`) + console.log(caughtA && caughtB ? 'POISON OK: 2 caught' : 'POISON FAILED') + process.exit(caughtA && caughtB ? 0 : 1) +} + +const r = verdict(shipped) +console.log(`tree ${TREE}: ${shipped.length} table files (${r.cards} card shards${r.cards ? (cardsNamed ? ', named by their card- pattern' : ', NOT named') : ''}); ${fetched.length} fetched by a literal tables/ path`) +for (const f of r.unnamed) console.log(`FAIL shipped but named by nothing in the tree: ${f}`) +if (r.cardFail) console.log('FAIL card shards ship but no shipped file names the card- pattern') +for (const f of r.missing) console.log(`FAIL fetched as tables/${f} but not shipped`) +const n = r.unnamed.length + (r.cardFail ? 1 : 0) + r.missing.length +console.log(n ? `TABLES-SHIPPED-ARE-USED: FAIL — ${n} problem(s)` : 'TABLES-SHIPPED-ARE-USED: PASS — every shipped table is named, every fetched one ships') +process.exit(n ? 1 : 0) diff --git a/internal/generate-tree.mjs b/internal/generate-tree.mjs index 73a44169c..d2886545e 100644 --- a/internal/generate-tree.mjs +++ b/internal/generate-tree.mjs @@ -2007,7 +2007,18 @@ function main() { // internal-only files dropin.exclude already names, so a new table works with no // build change. Kept as a directory copy because the fetch name is dynamic // (`${assetsBase}/tables/${file}.json`). - const TABLES_SKIP = new Set(['open-doors.json', 'platform-reachability.json', 'index.json']) + // BUILD-ONLY TABLES stay in data/tables (the research sources the build scripts read; their content is already + // in the tables the maps fetch) and are not shipped: no shipped bundle, page or manifest names them. + // check:tables-shipped-are-used fails on a shipped table nothing names, and on a fetched one that doesn't ship. + // languages-per-city and key-cities-admin-n belong to key-cities, a gated bundle: un-skip them if it ships. + const TABLES_BUILD_ONLY = [ + 'arclight-index.json', 'arclight-share-urls.json', 'country-languages.json', 'door43-index.json', + 'doxa-detail.json', 'ebible-index.json', 'fcbh-index.json', 'godtools-index.json', + 'gospel-by-province-groups.json', 'grn-index.json', 'groupings.json', 'iso-macrolanguages.json', + 'key-cities-admin-n.json', 'languages-per-city.json', 'scriptureearth-index.json', 'sun-deaf-index.json', + 'translation-credits.json', 'translator-org-links.json', 'zone-regions.json', 'zume-index.json', + ] + const TABLES_SKIP = new Set(['open-doors.json', 'platform-reachability.json', 'index.json', ...TABLES_BUILD_ONLY]) const tablesSrc = join(pubAssets, 'tables') if (existsSync(tablesSrc)) { const tablesDest = join(treeAssets, 'tables') diff --git a/package.json b/package.json index 9ddde7c9d..c1341c993 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,7 @@ "check:gallery-embed:live-cors": "node internal/check-research-gallery-embed.mjs --live-cors", "check:gallery-embed:maplibre": "node internal/check-research-gallery-embed.mjs --provider maplibre", "check:gated-not-in-tree": "node internal/check-gated-not-in-tree.mjs", + "check:tables-shipped-are-used": "node internal/check-tables-shipped-are-used.mjs && node internal/check-tables-shipped-are-used.mjs --poison", "check:gospel-all": "node internal/check-gospel-all.mjs", "check:gospel-internet": "node plugin-designs/gospel-internet/scripts/check-gospel-internet-model.mjs && node plugin-designs/gospel-internet/scripts/check-gospel-internet-i18n.mjs && node plugin-designs/gospel-internet/scripts/check-language-tree.mjs && node plugin-designs/gospel-internet/scripts/check-gospel-internet-render.mjs", "check:gospel-tables-staged": "node internal/check-gospel-tables-staged.mjs", @@ -358,7 +359,7 @@ "dev": "bun x vite", "fix:sri": "node internal/fix-sri.mjs", "gallery": "node internal/generate-gallery.mjs", - "gates:post": "node internal/run-parallel.mjs --label=post -- check:capability-derived check:drop check:parse verify check:sri check:public-hygiene check:label-joins check:facet-cascade check:facet-one-field check:heartbeat check:kpi-unit check:composite-reg check:descriptions check:no-abs-paths check:no-codename check:dep-names check:composite-srcs check:embed-targets check:nested-token check:i18n-parity check:tabs-localized check:packets-all check:packet-strings-parity check:zone-census check:zoning-declared check:share-policy check:library-index check:csv-shape check:capability-citations check:catalog-identity check:no-orphan-gates check:quoted-scripts check:script-flags check:capability-exemplars check:packet-fixture check:packet-browser check:provider-switch check:locale-reachable check:url-resolver check:manifest-covers check:one-palette check:pin-dim-slider check:legend-source check:clustering-modes check:composite-embed-knobs check:composite-locale-leaks check:linked-vocabulary check:wp-embed check:dashboard-geocoder-provider check:leaf-engine-per-provider check:auto-engine check:engine-contract check:map-props-unit check:composites-mount check:composite-interop check:no-free-identifiers check:map-fly-reduced-motion check:mst-clustering check:list-prop-unit check:profiles-mount check:cdn-pins check:no-eval check:popup-escaping check:i18n-treeshake check:pinned-umd check:one-palette:self-test check:no-cross-plugin-imports check:no-cross-plugin-imports:self-test check:map-attribution check:studio-rail check:approx-pin-note check:embedder-keys check:provider-for check:legend-logical-insets check:legend-logical-insets:self-test check:optional-t-equivalence check:text-highlight check:religion-label-not-value check:i18n-stopped-scope check:no-silent-engine check:no-silent-engine:self-test check:regions-uses-library check:shadow-css-parses check:square-map-controls check:preset-config-merge check:instance-config-keys check:frozen-labels check:plural-forms check:region-fill-repaint check:region-paint-single-source check:poster-print-fallback check:posters-exportable check:country-names check:reduced-motion check:no-dead-library check:universal-props check:readme-size check:tree-relative check:leaf-hints check:snippet-engine check:snippet-engine:self-test check:embed-base check:link-coverage check:no-public-binds check:shared-reached check:tree-weight check:no-fleet-names:self-test check:pin-hint-floor:self-test check:poster-sizes:self-test check:shell-header:self-test check:preview-flag-dev-only:self-test check:poster-type-registry:self-test check:host-floor:self-test check:phone-print-keep:self-test check:pinned-fetch-fresh:self-test", + "gates:post": "node internal/run-parallel.mjs --label=post -- check:capability-derived check:drop check:parse verify check:sri check:public-hygiene check:tables-shipped-are-used check:label-joins check:facet-cascade check:facet-one-field check:heartbeat check:kpi-unit check:composite-reg check:descriptions check:no-abs-paths check:no-codename check:dep-names check:composite-srcs check:embed-targets check:nested-token check:i18n-parity check:tabs-localized check:packets-all check:packet-strings-parity check:zone-census check:zoning-declared check:share-policy check:library-index check:csv-shape check:capability-citations check:catalog-identity check:no-orphan-gates check:quoted-scripts check:script-flags check:capability-exemplars check:packet-fixture check:packet-browser check:provider-switch check:locale-reachable check:url-resolver check:manifest-covers check:one-palette check:pin-dim-slider check:legend-source check:clustering-modes check:composite-embed-knobs check:composite-locale-leaks check:linked-vocabulary check:wp-embed check:dashboard-geocoder-provider check:leaf-engine-per-provider check:auto-engine check:engine-contract check:map-props-unit check:composites-mount check:composite-interop check:no-free-identifiers check:map-fly-reduced-motion check:mst-clustering check:list-prop-unit check:profiles-mount check:cdn-pins check:no-eval check:popup-escaping check:i18n-treeshake check:pinned-umd check:one-palette:self-test check:no-cross-plugin-imports check:no-cross-plugin-imports:self-test check:map-attribution check:studio-rail check:approx-pin-note check:embedder-keys check:provider-for check:legend-logical-insets check:legend-logical-insets:self-test check:optional-t-equivalence check:text-highlight check:religion-label-not-value check:i18n-stopped-scope check:no-silent-engine check:no-silent-engine:self-test check:regions-uses-library check:shadow-css-parses check:square-map-controls check:preset-config-merge check:instance-config-keys check:frozen-labels check:plural-forms check:region-fill-repaint check:region-paint-single-source check:poster-print-fallback check:posters-exportable check:country-names check:reduced-motion check:no-dead-library check:universal-props check:readme-size check:tree-relative check:leaf-hints check:snippet-engine check:snippet-engine:self-test check:embed-base check:link-coverage check:no-public-binds check:shared-reached check:tree-weight check:no-fleet-names:self-test check:pin-hint-floor:self-test check:poster-sizes:self-test check:shell-header:self-test check:preview-flag-dev-only:self-test check:poster-type-registry:self-test check:host-floor:self-test check:phone-print-keep:self-test check:pinned-fetch-fresh:self-test", "gates:pre": "node internal/run-parallel.mjs --label=pre -- check:template-literal check:mjs-parse check:sfc-parse check:tab-radius check:synced-css check:doc-scripts check:setup-order check:package-json-unique-keys check:no-repo-pb-hooks check:no-credentials-omit check:no-credentials-omit:self-test check:shadow-reduced-motion check:poster-action-print check:targets check:targets:self-test check:browser-gates-guarded check:pin-selection check:facet-shape check:hardcoded-english:self-test check:list-selection check:selector-fields check:upg-zone-joins check:gospel-set-charts check:gospel-set-fit check:describe-group check:one-scroll-container check:packet-charts check:pins-in-state check:fork-leaders check:circular-layout check:composite-token-url check:language-tree-layout check:i18n-keys-exist check:i18n-never-translated check:packet-variants check:profile-table check:element-bootstrap check:embed-url-absolute check:frame-target check:language-tree-model check:map-menu-anchor check:selection-kinds check:tiered-points check:whiteboard-projection check:mercator-slice check:ring-vocabulary check:map-props-unit check:mobile-gating check:provide-shadowing check:legend-top-offset check:cdn-cache check:plural-forms check:select-zone-race check:script-registry check:map-control-overlap check:poster-attribution check:board-map-options check:gospel-whiteboard check:packet-map-missing-loud check:poster-layout-optin check:ported-globals-identical check:ring-bands check:packet-a1-structure check:india-states check:batch-selection-export check:zone-title-tier-a check:evangelical-share check:map-labels check:packet-offline check:help-reachable check:gated-not-in-tree check:window-globals-released check:legend-data-contract check:no-internal-role-nouns check:poster-fetch-guard check:people-group-records check:sentence-slots check:popup-href-escaped check:popup-pan-controls check:country-map-refetch check:pane-config-reforward check:viewport-meta check:viewport-meta:self-test check:label-key-coverage check:poster-browser-equiv check:poster-render-pinned check:country-sheet-one-path check:country-sheet-one-path:self-test check:side-lanes check:side-lanes:self-test check:poster-section check:poster-section:self-test check:posterverse-shelf check:host-locale-fetch check:print-frame-reclaim check:click-sheet check:coverage-consistent:chained check:coverage-consistent:poison check:dashboard-filter-a11y check:filter-multivalue check:hover-channel check:kit-options-reachable check:layout-key-namespaced check:legend-mirrors check:map-unmount-race check:no-conflict-markers check:series-isolation check:snapshot-guard check:stale-sweep-claim check:surfaces-read-the-tier check:translator-contacts check:wall-wysiwyg check:wall-print-watch check:wall-no-bg-photos check:wall-pan check:template-i18n check:list-pattern-doc check:shadow-style-labels-unique check:grid-image-loading check:retry-fetch check:panel-sheets check:poster-wall-host-seams check:poster-sections check:poster-sections:self-test check:poster-design-catalogue check:poster-design-catalogue:self-test check:upg100-zone-copy check:upg100-zone-copy:self-test check:wall-photo-weight check:no-fleet-names check:pin-hint-floor check:poster-sizes check:upg-count check:shell-header check:pg-name-gate check:preview-flag-dev-only check:poster-type-registry check:host-floor check:phone-print-keep check:pinned-fetch-fresh", "gen:iso3-to-alpha2": "node library/scripts/generate-iso3-to-alpha2.mjs", "internet:adm1": "node plugin-designs/cities-internet/scripts/build-adm1-internet.mjs", @@ -528,6 +529,7 @@ "check:gallery-embed": "the cap-5 Research Gallery must embed on a FOREIGN ORIGIN (a stand-in host page on localhost, the built tree on 127.0.0.1) in BOTH modes, bare