diff --git a/internal/check-csv-export-shape.mjs b/internal/check-csv-export-shape.mjs new file mode 100644 index 000000000..7a08a367e --- /dev/null +++ b/internal/check-csv-export-shape.mjs @@ -0,0 +1,148 @@ +#!/usr/bin/env node +/** + * check-csv-export-shape.mjs — the exported CSV must OPEN correctly in a spreadsheet. + * + * THE DEFECT THIS CATCHES (Driver-reported 2026-09-22: "this is me exporting the CSV directly without + * making any edits... it does look a little bit glitched out"). The generator emitted a one-column + * provenance comment ABOVE the header: + * line 0 ... 1 column "# Research Dashboard export — scope: ..." + * line 1 ... N columns slug,name,religion,... + * Every spreadsheet importer treats the FIRST line as the header, so the scope sentence became the + * column names and the real header became data row 1. A 1-column line above a 68-column table is + * exactly what "glitched" looks like on screen. + * + * ★ WHAT WAS *NOT* WRONG, AND MUST NOT BE "FIXED" ON SUSPICION — three plausible theories, all + * refuted by measurement before a line was changed: + * - "it is tab-separated" — FALSE. The generator joins with commas; the tabs in the Driver's paste + * were his terminal rendering, not the bytes. + * - "quoting is broken" — FALSE. esc() is genuinely RFC-4180: it quotes when , " CR or LF appear and + * doubles internal quotes. Embedded commas, quotes AND newlines survive a strict parser round trip. + * - "CSV injection" — already handled; a leading = + - @ is prefixed with a single quote. + * Only the POSITION was wrong. Chasing the other three would have produced churn and no fix. + * + * ★ WHY NO EXISTING GATE SAW IT. 46 gate-ish scripts exist and not one parses the export. The bug is + * invisible to every DOM check (the button works, the file downloads, the bytes are valid UTF-8) and + * only appears when something tries to READ the file as a table. So this gate parses. + * + * WHAT IT ASSERTS, against the real buildCsv() logic extracted from the composable: + * 1. line 0 is the HEADER — not a comment, not data; + * 2. every data row has exactly the header's column count; + * 3. a field containing a comma, a double quote or a newline survives a strict RFC-4180 round trip; + * 4. the provenance comment still EXISTS (it is load-bearing: it records which filters produced the + * file) and sits at the END, where a trailing 1-column row is harmless. + * + * NEGATIVE-TESTED: putting the provenance line back on top fails assertion 1 by name. + */ +import { readFileSync, existsSync } from 'node:fs' +import { resolve, dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const SRC = join(ROOT, 'plugin-designs/research-dashboard/src/composables/useCsvExport.js') + +if (!existsSync(SRC)) { + console.error('CSV-SHAPE: FAIL — useCsvExport.js not found at ' + SRC.replace(ROOT + '/', '')) + process.exit(1) +} +const src = readFileSync(SRC, 'utf8') + +// Extract the REAL esc() rather than re-implementing it — a gate that restates the rule can agree +// with itself while the product disagrees. +const escMatch = src.match(/function esc\(v\)\s*\{([\s\S]*?)\n \}/) +if (!escMatch) { + console.error('CSV-SHAPE: FAIL — could not extract esc() from useCsvExport.js.') + console.error(' The gate must test the SHIPPED escaper, not a copy of the rule.') + process.exit(1) +} +let esc +try { esc = new Function('v', escMatch[1]) } +catch (e) { console.error('CSV-SHAPE: FAIL — extracted esc() does not parse: ' + e.message); process.exit(1) } + +// Extract the line-assembly order, which is the thing that actually broke. +const linesMatch = src.match(/const lines = \[([^\]]*)\]/) +if (!linesMatch) { + console.error('CSV-SHAPE: FAIL — could not find the `const lines = [...]` assembly in buildCsv().') + process.exit(1) +} +const order = linesMatch[1].replace(/\s+/g, ' ').trim() +const provenanceFirst = /^\s*provenance\s*,/.test(order) + +const cols = ['slug', 'name', 'religion', 'imb_people_description'] +const rows = [ + { slug: 'alas', name: 'Alas', religion: 'Islam - Sunni', + imb_people_description: 'The Alas of Indonesia, numbering approximately 268,000 people. They say "hello".' }, + { slug: 'kluet', name: 'Kluet', religion: 'Islam - Sunni', + imb_people_description: 'Line one\nLine two, with a comma' }, + { slug: 'inj', name: '=SUM(A1)', religion: 'Islam - Sunni', imb_people_description: 'injection probe' }, +] +const provenance = '# Research Dashboard export — scope: filter result; filters: religion=Islam - Sunni; rows: 3' +const body = [cols.map(esc).join(','), ...rows.map((r) => cols.map((c) => esc(r[c])).join(','))] +const csv = (provenanceFirst ? [provenance, ...body] : [...body, provenance]).join('\r\n') + +// Strict RFC-4180 parser — the only honest way to judge "does this open as a table". +function parse(t) { + const out = []; let row = [], f = '', q = false + for (let i = 0; i < t.length; i++) { + const c = t[i] + if (q) { if (c === '"') { if (t[i + 1] === '"') { f += '"'; i++ } else q = false } else f += c } + else if (c === '"') q = true + else if (c === ',') { row.push(f); f = '' } + else if (c === '\r' && t[i + 1] === '\n') { row.push(f); out.push(row); row = []; f = ''; i++ } + else f += c + } + if (f !== '' || row.length) { row.push(f); out.push(row) } + return out +} + +const parsed = parse(csv) +const fail = (msg, detail) => { + console.error('CSV-SHAPE: FAIL — ' + msg) + if (detail) console.error(detail) + process.exit(1) +} + +// 1 — line 0 must be the header +const first = parsed[0] || [] +if (first.length === 1 && String(first[0]).startsWith('#')) { + fail('line 0 is a one-column "#" comment, so a spreadsheet will use it AS THE HEADER.', + ' The scope sentence becomes the column names and the real header becomes data row 1 — which is\n' + + ' what the Driver saw as a "glitched" export. Put the provenance line LAST; a trailing one-column\n' + + ' row is harmless, a leading one destroys the column mapping for the whole sheet.') +} +if (first[0] !== cols[0]) { + fail(`line 0 does not start with the first column ("${cols[0]}") — got "${String(first[0]).slice(0, 40)}".`) +} + +// 2 — every data row matches the header width +const dataRows = parsed.slice(1).filter((r) => !(r.length === 1 && String(r[0]).startsWith('#'))) +const wrong = dataRows.filter((r) => r.length !== first.length) +if (wrong.length) { + fail(`${wrong.length} of ${dataRows.length} data row(s) do not have the header's ${first.length} columns.`, + ' A field containing a comma, a quote or a newline is not being quoted, so it shifts every later\n' + + ' column on that row. esc() must quote when , " CR or LF appear and double the internal quotes.') +} + +// 3 — the hard fields survived intact +const desc = dataRows[1] && dataRows[1][3] +if (!desc || !desc.includes('\n') || !desc.includes(',')) { + fail('a field containing BOTH a newline and a comma did not survive the round trip.', + ' Round-tripped value: ' + JSON.stringify(String(desc).slice(0, 60))) +} +const quoted = dataRows[0] && dataRows[0][3] +if (!quoted || !quoted.includes('"hello"')) { + fail('an embedded double quote did not survive the round trip.', + ' Round-tripped value: ' + JSON.stringify(String(quoted).slice(0, 60))) +} + +// 4 — provenance still exists, and is last +const last = parsed[parsed.length - 1] || [] +if (!(last.length === 1 && String(last[0]).startsWith('#'))) { + fail('the provenance comment is missing from the end of the file.', + ' It is load-bearing: it records WHICH FILTERS produced this file, which is the difference\n' + + ' between a dataset and an anecdote. Keep it — just keep it last.') +} + +console.log(`CSV-SHAPE: PASS — header on line 0 (${first.length} cols), ${dataRows.length} data row(s) all matching,`) +console.log(' embedded comma/quote/newline round-trip intact, provenance preserved as a trailing comment.') +console.log(' This parses the export as a spreadsheet would; no DOM check can see this defect, because') +console.log(' the button works, the file downloads, and the bytes are valid — only the shape is wrong.') diff --git a/package.json b/package.json index f511fb92e..92b898177 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,8 @@ "check:sfc-parse": "node internal/check-sfc-script-parses.mjs # every .vue script block must parse as the BUILD parses it — catches the LIVE-CSS backtick that check:template-literal cannot see (it reads CSS comments only). 0 false positives on 83 files, where 3 lookahead heuristics scored 98 and 136.", "check:url-resolver": "node internal/check-url-resolver.mjs # cap-5 resolve() must not prefix a rooted URL (the dev-surface 404 class)", "smoke:studio": "node internal/smoke-studio.mjs # 5s DEV-surface check: / mounts the studio, tabs>0, 0 4xx (exit 2 = dev server down)", - "check:manifest-covers": "node internal/check-manifest-covers-discovery.mjs # every DISCOVERED (non-gated) bundle must be in the built manifest" + "check:manifest-covers": "node internal/check-manifest-covers-discovery.mjs # every DISCOVERED (non-gated) bundle must be in the built manifest", + "check:csv-shape": "node internal/check-csv-export-shape.mjs # the export must OPEN as a table: header on line 0, every row matching" }, "devDependencies": { "@capacitor/android": "^7.6.9", diff --git a/plugin-designs/research-dashboard/src/composables/useCsvExport.js b/plugin-designs/research-dashboard/src/composables/useCsvExport.js index faa44bc60..4c2f95220 100644 --- a/plugin-designs/research-dashboard/src/composables/useCsvExport.js +++ b/plugin-designs/research-dashboard/src/composables/useCsvExport.js @@ -156,7 +156,29 @@ export function useCsvExport({ rowStore, rows, selectionSet, filterSet, FACET_AC .map(([k, v]) => `${k}=${labelFor ? labelFor(k, v) : v}`) .join(' | ') || 'none' const provenance = `# Research Dashboard export — scope: ${scopeLabel()}; filters: ${chips}; rows: ${picked.length}` - const lines = [provenance, cols.map(esc).join(','), ...picked.map((r) => cols.map((c) => esc(r[c])).join(','))] + // ★ THE PROVENANCE LINE GOES LAST, NOT FIRST — that one position was the whole "glitched CSV". + // Driver-reported 2026-09-22: "this is me exporting the CSV directly without making any edits... + // it does look a little bit glitched out." + // + // MEASURED with a strict RFC-4180 parser against this exact generator. EVERYTHING ELSE IS CORRECT + // and must not be "fixed" on suspicion: + // line 0 ... 1 column "# Research Dashboard export — scope: ..." + // line 1 ... N columns slug,name,religion,... + // rows ... N columns each, ZERO with a wrong column count + // embedded commas, embedded double quotes AND embedded newlines all survive the round trip, so + // esc() is genuinely RFC-4180 (quote when , " CR LF appear; double the internal quotes). The + // file is comma-separated, not tab-separated — the tabs in the Driver's paste were his terminal + // rendering, not the bytes. + // + // The defect is purely POSITIONAL: every spreadsheet importer treats the FIRST line as the header. + // So the scope sentence became the column names, the real header became data row 1, and a 1-column + // line above a 68-column table is exactly what "glitched" looks like on screen. + // + // Moving it BELOW the data keeps the provenance — which is deliberate and load-bearing, since it + // records WHICH FILTERS produced the file, the difference between a dataset and an anecdote — + // while making line 1 the header. A trailing 1-column line lands as a final row with one populated + // cell: obvious to a human, harmless to a formula, and it cannot destroy the column mapping. + const lines = [cols.map(esc).join(','), ...picked.map((r) => cols.map((c) => esc(r[c])).join(',')), provenance] return lines.join('\r\n') } // Last-export status for UI feedback (toast). The platform gem-frame iframe is sandboxed WITHOUT @@ -166,8 +188,12 @@ export function useCsvExport({ rowStore, rows, selectionSet, filterSet, FACET_AC function exportCsv() { const csv = buildCsv() const rows = pickRows().length - // header line is line 2 (line 1 is the '#' provenance comment) - const fields = (csv.split('\r\n')[1] || '').split(',').length + // The header is now line 1 — the '#' provenance comment moved to the END of the file so that a + // spreadsheet reads the real column row first (see buildCsv). This index had to move WITH it: + // reading line 2 would have counted the columns of DATA ROW 1, which is usually the same number, + // so it would have looked correct and been wrong — the quiet kind of breakage a positional change + // leaves behind. + const fields = (csv.split('\r\n')[0] || '').split(',').length try { if (typeof window !== 'undefined') window.__rdashLastCsv = csv } catch (_) { /* verify hook */ } let clipboard = false, downloaded = false // 1) clipboard — works inside the sandboxed gem-frame iframe (the guaranteed path)