doxa-maps/internal/check-csv-export-shape.mjs

#!/usr/bin/env node
/**
 * check-csv-export-shape.mjs — the exported CSV must OPEN correctly in a spreadsheet.
 *
 * THE DEFECT THIS CATCHES (Driver-reported 2026-09-22: "this is me exporting the CSV directly without
 * making any edits... it does look a little bit glitched out"). The generator emitted a one-column
 * provenance comment ABOVE the header:
 *     line 0 ... 1 column   "# Research Dashboard export — scope: ..."
 *     line 1 ... N columns  slug,name,religion,...
 * Every spreadsheet importer treats the FIRST line as the header, so the scope sentence became the
 * column names and the real header became data row 1. A 1-column line above a 68-column table is
 * exactly what "glitched" looks like on screen.
 *
 * ★ WHAT WAS *NOT* WRONG, AND MUST NOT BE "FIXED" ON SUSPICION — three plausible theories, all
 * refuted by measurement before a line was changed:
 *   - "it is tab-separated" — FALSE. The generator joins with commas; the tabs in the Driver's paste
 *     were his terminal rendering, not the bytes.
 *   - "quoting is broken" — FALSE. esc() is genuinely RFC-4180: it quotes when , " CR or LF appear and
 *     doubles internal quotes. Embedded commas, quotes AND newlines survive a strict parser round trip.
 *   - "CSV injection" — already handled; a leading = + - @ is prefixed with a single quote.
 * Only the POSITION was wrong. Chasing the other three would have produced churn and no fix.
 *
 * ★ WHY NO EXISTING GATE SAW IT. 46 gate-ish scripts exist and not one parses the export. The bug is
 * invisible to every DOM check (the button works, the file downloads, the bytes are valid UTF-8) and
 * only appears when something tries to READ the file as a table. So this gate parses.
 *
 * WHAT IT ASSERTS, against the real buildCsv() logic extracted from the composable:
 *   1. line 0 is the HEADER — not a comment, not data;
 *   2. every data row has exactly the header's column count;
 *   3. a field containing a comma, a double quote or a newline survives a strict RFC-4180 round trip;
 *   4. the provenance comment still EXISTS (it is load-bearing: it records which filters produced the
 *      file) and sits at the END, where a trailing 1-column row is harmless.
 *
 * NEGATIVE-TESTED: putting the provenance line back on top fails assertion 1 by name.
 */
import { readFileSync, existsSync } from 'node:fs'
import { resolve, dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'

const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
const SRC = join(ROOT, 'plugin-designs/research-dashboard/src/composables/useCsvExport.js')

if (!existsSync(SRC)) {
  console.error('CSV-SHAPE: FAIL — useCsvExport.js not found at ' + SRC.replace(ROOT + '/', ''))
  process.exit(1)
}
const src = readFileSync(SRC, 'utf8')

// Extract the REAL esc() rather than re-implementing it — a gate that restates the rule can agree
// with itself while the product disagrees.
const escMatch = src.match(/function esc\(v\)\s*\{([\s\S]*?)\n  \}/)
if (!escMatch) {
  console.error('CSV-SHAPE: FAIL — could not extract esc() from useCsvExport.js.')
  console.error('  The gate must test the SHIPPED escaper, not a copy of the rule.')
  process.exit(1)
}
let esc
try { esc = new Function('v', escMatch[1]) }
catch (e) { console.error('CSV-SHAPE: FAIL — extracted esc() does not parse: ' + e.message); process.exit(1) }

// Extract the line-assembly order, which is the thing that actually broke.
const linesMatch = src.match(/const lines = \[([^\]]*)\]/)
if (!linesMatch) {
  console.error('CSV-SHAPE: FAIL — could not find the `const lines = [...]` assembly in buildCsv().')
  process.exit(1)
}
const order = linesMatch[1].replace(/\s+/g, ' ').trim()
const provenanceFirst = /^\s*provenance\s*,/.test(order)

const cols = ['slug', 'name', 'religion', 'imb_people_description']
const rows = [
  { slug: 'alas', name: 'Alas', religion: 'Islam - Sunni',
    imb_people_description: 'The Alas of Indonesia, numbering approximately 268,000 people. They say "hello".' },
  { slug: 'kluet', name: 'Kluet', religion: 'Islam - Sunni',
    imb_people_description: 'Line one\nLine two, with a comma' },
  { slug: 'inj', name: '=SUM(A1)', religion: 'Islam - Sunni', imb_people_description: 'injection probe' },
]
const provenance = '# Research Dashboard export — scope: filter result; filters: religion=Islam - Sunni; rows: 3'
const body = [cols.map(esc).join(','), ...rows.map((r) => cols.map((c) => esc(r[c])).join(','))]
const csv = (provenanceFirst ? [provenance, ...body] : [...body, provenance]).join('\r\n')

// Strict RFC-4180 parser — the only honest way to judge "does this open as a table".
function parse(t) {
  const out = []; let row = [], f = '', q = false
  for (let i = 0; i < t.length; i++) {
    const c = t[i]
    if (q) { if (c === '"') { if (t[i + 1] === '"') { f += '"'; i++ } else q = false } else f += c }
    else if (c === '"') q = true
    else if (c === ',') { row.push(f); f = '' }
    else if (c === '\r' && t[i + 1] === '\n') { row.push(f); out.push(row); row = []; f = ''; i++ }
    else f += c
  }
  if (f !== '' || row.length) { row.push(f); out.push(row) }
  return out
}

const parsed = parse(csv)
const fail = (msg, detail) => {
  console.error('CSV-SHAPE: FAIL — ' + msg)
  if (detail) console.error(detail)
  process.exit(1)
}

// 1 — line 0 must be the header
const first = parsed[0] || []
if (first.length === 1 && String(first[0]).startsWith('#')) {
  fail('line 0 is a one-column "#" comment, so a spreadsheet will use it AS THE HEADER.',
    '  The scope sentence becomes the column names and the real header becomes data row 1 — which is\n' +
    '  what the Driver saw as a "glitched" export. Put the provenance line LAST; a trailing one-column\n' +
    '  row is harmless, a leading one destroys the column mapping for the whole sheet.')
}
if (first[0] !== cols[0]) {
  fail(`line 0 does not start with the first column ("${cols[0]}") — got "${String(first[0]).slice(0, 40)}".`)
}

// 2 — every data row matches the header width
const dataRows = parsed.slice(1).filter((r) => !(r.length === 1 && String(r[0]).startsWith('#')))
const wrong = dataRows.filter((r) => r.length !== first.length)
if (wrong.length) {
  fail(`${wrong.length} of ${dataRows.length} data row(s) do not have the header's ${first.length} columns.`,
    '  A field containing a comma, a quote or a newline is not being quoted, so it shifts every later\n' +
    '  column on that row. esc() must quote when , " CR or LF appear and double the internal quotes.')
}

// 3 — the hard fields survived intact
const desc = dataRows[1] && dataRows[1][3]
if (!desc || !desc.includes('\n') || !desc.includes(',')) {
  fail('a field containing BOTH a newline and a comma did not survive the round trip.',
    '  Round-tripped value: ' + JSON.stringify(String(desc).slice(0, 60)))
}
const quoted = dataRows[0] && dataRows[0][3]
if (!quoted || !quoted.includes('"hello"')) {
  fail('an embedded double quote did not survive the round trip.',
    '  Round-tripped value: ' + JSON.stringify(String(quoted).slice(0, 60)))
}

// 4 — provenance still exists, and is last
const last = parsed[parsed.length - 1] || []
if (!(last.length === 1 && String(last[0]).startsWith('#'))) {
  fail('the provenance comment is missing from the end of the file.',
    '  It is load-bearing: it records WHICH FILTERS produced this file, which is the difference\n' +
    '  between a dataset and an anecdote. Keep it — just keep it last.')
}

console.log(`CSV-SHAPE: PASS — header on line 0 (${first.length} cols), ${dataRows.length} data row(s) all matching,`)
console.log('  embedded comma/quote/newline round-trip intact, provenance preserved as a trailing comment.')
console.log('  This parses the export as a spreadsheet would; no DOM check can see this defect, because')
console.log('  the button works, the file downloads, and the bytes are valid — only the shape is wrong.')