Update from source 9e59157b

backend/pocketbase/README.md

@@ -0,0 +1,13 @@
+# PocketBase — the backend for plugins that need one
+
+The cloud container ([deployment/cloud](../../deployment/cloud/README.md)) runs two PocketBase
+instances: **dev** (behind the team sign-in, for building) and **prod** (its `/api/` public, for
+published plugins). Both apply the migrations in [`pb_migrations/`](./pb_migrations/) on start, so
+**the schema lives in git** and publishing never copies data from dev to prod.
+
+A plugin reads its backend address at runtime (`profile-config` `apiBase`, or
+`window.MAP_API_BASE` set by the published build's `runtime-config.js`), so moving the backend never
+needs a rebuild.
+
+v1 limits: collection rules are the only access control (lock collections by default); no per-plugin
+isolation between plugin authors yet.

deployment/cloud/Dockerfile

@@ -4,9 +4,15 @@
 # container host (Coolify "Dockerfile" build pack, Railway, or `docker build`).
 FROM quay.io/oauth2-proxy/oauth2-proxy:v7.15.3 AS oauth2proxy
 FROM oven/bun:1-debian
-RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \
+RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates curl unzip \
  && rm -rf /var/lib/apt/lists/*
 COPY --from=oauth2proxy /bin/oauth2-proxy /usr/local/bin/oauth2-proxy
+# PocketBase, pinned and checked: the backend for plugins (a dev and a prod instance, see entry.sh).
+ARG PB_VERSION=0.40.4
+ARG PB_SHA256=9042ec818570e79c3628dadcd0a756c1496d9e1173918ec409d133c02f82e5fa
+RUN curl -fsSL -o /tmp/pb.zip "https://github.com/pocketbase/pocketbase/releases/download/v${PB_VERSION}/pocketbase_${PB_VERSION}_linux_amd64.zip" \
+ && echo "${PB_SHA256}  /tmp/pb.zip" | sha256sum -c - \
+ && unzip -q /tmp/pb.zip pocketbase -d /usr/local/bin && rm /tmp/pb.zip && chmod 755 /usr/local/bin/pocketbase
 RUN printf '%s\n' \
   '#!/bin/bash' \
   'set -u; export GIT_TERMINAL_PROMPT=0' \
@@ -17,5 +23,6 @@ RUN printf '%s\n' \
   'cp /app/repo/deployment/cloud/entry.sh /tmp/entry.sh && exec bash /tmp/entry.sh' \
   > /boot.sh && chmod 755 /boot.sh
 ENV PORT=5173 BRANCH=main PULL_INTERVAL=15
+VOLUME /data
 EXPOSE 5173
 CMD ["/boot.sh"]

deployment/cloud/README.md

@@ -43,3 +43,31 @@ with the redirect URL `https://<your-host>/oauth2/callback`; PKCE (S256) is used
 | `OIDC_COOKIE_SECRET` | 32 characters; keeps people signed in across restarts (random if unset) |
 
 Secrets reach oauth2-proxy through its environment, never its command line.
+
+## Backend and production (v1, experimental)
+
+With the sign-in on, the container also runs PocketBase and can publish a production build:
+
+| Route | Who | What |
+|---|---|---|
+| `/` | signed-in | the dev server (Plugin Studio, hot reload) |
+| `/pb/` | signed-in (its `/_/` dashboard: `OIDC_ADMIN_GROUPS` only) | PocketBase **dev** |
+| `/api-prod/api/` | **public** | PocketBase **prod**'s API; its dashboard is never exposed |
+| `/cdn/dx-maps/` | **public** | the last published build: `manifest.json` at a stable URL, CORS open for embeds |
+| `POST /admin/build` · `GET /admin/status` | `OIDC_ADMIN_GROUPS` only | build and publish the pulled commit |
+
+A build runs one at a time (a second gets `409`), on a snapshot of the pulled commit, with Node's heap
+capped. It removes everything in [`public-exclusions.txt`](./public-exclusions.txt) **before**
+building, checks the output again afterwards, and refuses to publish if anything excluded is still
+there. Only then does it swap the new build in.
+
+| Variable | Meaning |
+|---|---|
+| `OIDC_ADMIN_GROUPS` | comma-separated groups allowed to build and to open PocketBase's dashboard |
+| `PB_SUPERUSER_EMAIL`, `PB_SUPERUSER_PASSWORD` | PocketBase's superuser, set on start in both instances |
+| `PUBLIC_API_BASE` | the public API URL written into the build's `runtime-config.js` (e.g. `https://<host>/api-prod`) |
+| `BUILD_MAX_OLD_SPACE_MB` | Node heap cap for a build (default `1536`) |
+
+Mount a persistent volume at **`/data`** (PocketBase data and published builds). Memory measured on
+the running container: about 0.45 GB idle, up to about 1.5 GB more during a build, so give it
+**2.5 GB**.

deployment/cloud/admin.mjs

@@ -0,0 +1,66 @@
+// admin.mjs — the container's internal router (127.0.0.1:5175), behind oauth2-proxy.
+//   /cdn/*       the last good build (/data/cdn/current), PUBLIC (oauth2-proxy skips auth here)
+//   /api-prod/*  PocketBase PROD, PUBLIC; only /api/ passes, never the /_/ dashboard
+//   /pb/*        PocketBase DEV, signed-in; its /_/ dashboard only for OIDC_ADMIN_GROUPS
+//   /admin/build POST: start a build (OIDC_ADMIN_GROUPS only); /admin/status GET
+// Groups come from oauth2-proxy's X-Forwarded-Groups, which it sets on signed-in routes only.
+import { spawn } from 'node:child_process'
+import { existsSync, readFileSync, statSync } from 'node:fs'
+import { join, normalize } from 'node:path'
+
+const env = process.env
+const ADMIN = (env.OIDC_ADMIN_GROUPS || '').split(',').map((g) => g.trim()).filter(Boolean)
+const CDN = '/data/cdn/current'
+let last = { state: 'idle' }
+
+const groupsOf = (req) => (req.headers.get('x-forwarded-groups') || '').split(',').map((g) => g.trim())
+const isAdmin = (req) => ADMIN.length > 0 && groupsOf(req).some((g) => ADMIN.includes(g))
+const text = (s, status = 200) => new Response(s + '\n', { status, headers: { 'content-type': 'text/plain; charset=utf-8' } })
+
+async function proxy(req, url, prefix, port) {
+  const target = new URL(url.pathname.slice(prefix.length - 1) + url.search, `http://127.0.0.1:${port}`)
+  const r = await fetch(target, { method: req.method, headers: req.headers, body: ['GET', 'HEAD'].includes(req.method) ? undefined : req.body, redirect: 'manual' })
+  return new Response(r.body, { status: r.status, headers: r.headers })
+}
+
+Bun.serve({
+  hostname: '127.0.0.1',
+  port: 5175,
+  async fetch(req) {
+    const url = new URL(req.url)
+    const p = url.pathname
+    if (p.startsWith('/cdn/')) {
+      const f = normalize(join(CDN, decodeURIComponent(p.slice(5))))
+      if (!f.startsWith(CDN)) return text('bad path', 400)
+      const file = existsSync(f) && statSync(f).isDirectory() ? join(f, 'index.html') : f
+      if (!existsSync(file)) return text('not found', 404)
+      return new Response(Bun.file(file), { headers: { 'access-control-allow-origin': '*', 'cache-control': 'public, max-age=86400, stale-while-revalidate=604800' } })
+    }
+    if (p.startsWith('/api-prod/')) {
+      if (!p.startsWith('/api-prod/api/')) return text('only /api/ is public', 403)
+      return proxy(req, url, '/api-prod/', 8091)
+    }
+    if (p.startsWith('/pb/')) {
+      if (p.startsWith('/pb/_/') && !isAdmin(req)) return text('the PocketBase dashboard is for admins', 403)
+      return proxy(req, url, '/pb/', 8090)
+    }
+    if (p === '/admin/status') return Response.json(last)
+    if (p === '/admin/build') {
+      if (req.method !== 'POST') return text('POST only', 405)
+      if (!isAdmin(req)) return text('builds are for team admins', 403)
+      if (last.state === 'building') return text('a build is already running', 409)
+      last = { state: 'building', started: new Date().toISOString() }
+      const child = spawn('bash', ['/app/repo/deployment/cloud/build-cdn.sh'], { stdio: ['ignore', 'pipe', 'pipe'] })
+      let log = ''
+      child.stdout.on('data', (d) => { log = (log + d).slice(-4000) })
+      child.stderr.on('data', (d) => { log = (log + d).slice(-4000) })
+      child.on('exit', (code) => {
+        last = { state: code === 0 ? 'done' : code === 75 ? 'busy' : 'failed', code, finished: new Date().toISOString(), tail: log.slice(-1500),
+          commit: existsSync('/data/cdn/COMMIT') ? readFileSync('/data/cdn/COMMIT', 'utf8').trim() : null }
+      })
+      return text('build started', 202)
+    }
+    return text('not found', 404)
+  },
+})
+console.log('[admin] listening on 127.0.0.1:5175')

deployment/cloud/apply-exclusions.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# apply-exclusions.sh <dir>            remove everything public-exclusions.txt excludes from <dir>
+# apply-exclusions.sh --check <dir>    exit 1 if anything excluded is still in <dir> (nothing removed)
+set -u
+LIST="$(dirname "$0")/public-exclusions.txt"
+MODE=apply; [ "${1:-}" = --check ] && { MODE=check; shift; }
+DIR="${1:?usage: apply-exclusions.sh [--check] <dir>}"
+[ -d "$DIR" ] || { echo "no such dir: $DIR" >&2; exit 2; }
+keep=(); names=(); paths=(); contents=()
+while IFS= read -r l; do
+  case "$l" in ''|'#'*) ;; keep:*) keep+=("${l#keep:}") ;; name:*) names+=("${l#name:}") ;;
+    content:*) contents+=("${l#content:}") ;; *) paths+=("$l") ;; esac
+done < "$LIST"
+found=0
+for p in "${paths[@]}"; do
+  # at the repo root, and anywhere below it (a built plugin sits at dx-maps/<name>)
+  while IFS= read -r hit; do
+    [ -e "$hit" ] || continue
+    if [ $MODE = apply ]; then rm -rf "$hit"; echo "  excluded: $hit"; else echo "  PRESENT: $hit"; found=1; fi
+  done < <( { [ -e "$DIR/$p" ] && echo "$DIR/$p"; find "$DIR" -path "*/$(basename "$p")" -not -path '*/node_modules/*' -prune 2>/dev/null; } | sort -u)
+done
+for n in "${names[@]}"; do
+  while IFS= read -r f; do
+    b=$(basename "$f"); skip=0; for k in "${keep[@]}"; do [ "$b" = "$k" ] && skip=1; done
+    [ $skip = 1 ] && continue
+    if [ $MODE = apply ]; then rm -rf "$f"; echo "  excluded: $f"; else echo "  PRESENT: $f"; found=1; fi
+  done < <(find "$DIR" -iname "$n" -not -path '*/node_modules/*' 2>/dev/null)
+done
+if [ $MODE = check ]; then
+  for c in "${contents[@]}"; do
+    hits=$(grep -rIlE "$c" "$DIR" --exclude-dir=node_modules 2>/dev/null | head -5)
+    [ -n "$hits" ] && { echo "  CONTENT /$c/ in:"; echo "$hits" | sed 's/^/    /'; found=1; }
+  done
+  [ $found = 1 ] && { echo "EXCLUSIONS: FAIL"; exit 1; }
+  echo "EXCLUSIONS: clean"
+fi
+exit 0

deployment/cloud/build-cdn.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# build-cdn.sh — one production build of the pulled commit into /data/cdn, one at a time.
+#   snapshot (so the pull loop can't change files mid-build) -> apply the public exclusions ->
+#   BUILD_ALLOW=1 bun run build (memory-capped) -> check the output -> atomic swap of /data/cdn/current.
+# Exit 75 = another build holds the lock.
+set -u
+exec 9>/tmp/build-cdn.lock
+flock -n 9 || { echo "another build is running"; exit 75; }
+cd /app/repo
+SHA=$(git rev-parse --short HEAD); SNAP=/tmp/build-$SHA
+rm -rf "$SNAP"; mkdir -p "$SNAP"
+git archive HEAD | tar -x -C "$SNAP" || { echo "snapshot failed"; exit 1; }
+bash /app/repo/deployment/cloud/apply-exclusions.sh "$SNAP"
+ln -s /app/repo/node_modules "$SNAP/node_modules"
+cd "$SNAP"
+echo "$(date -u +%FT%TZ) building $SHA"
+NODE_OPTIONS="--max-old-space-size=${BUILD_MAX_OLD_SPACE_MB:-1536}" BUILD_ALLOW=1 nice -n 10 bun run build \
+  || { echo "build failed"; rm -rf "$SNAP"; exit 1; }
+OUT="$SNAP/public/dx-maps"
+[ -f "$OUT/manifest.json" ] || { echo "no manifest.json in the output"; rm -rf "$SNAP"; exit 1; }
+printf 'window.MAP_API_BASE=%s;\n' "$(printf '%s' "${PUBLIC_API_BASE:-}" | python3 -c 'import json,sys;print(json.dumps(sys.stdin.read()))')" > "$OUT/runtime-config.js"
+NEW=/data/cdn/build-$SHA
+rm -rf "$NEW"; mkdir -p /data/cdn; mv "$SNAP/public" "$NEW"
+bash /app/repo/deployment/cloud/apply-exclusions.sh --check "$NEW" || { echo "REFUSED: excluded content in the build; not published"; rm -rf "$NEW" "$SNAP"; exit 1; }
+ln -sfn "$NEW" /data/cdn/current.tmp && mv -T /data/cdn/current.tmp /data/cdn/current
+echo "$SHA" > /data/cdn/COMMIT
+find /data/cdn -maxdepth 1 -name 'build-*' ! -path "$NEW" -exec rm -rf {} +
+rm -rf "$SNAP"
+echo "$(date -u +%FT%TZ) published $SHA at /cdn/dx-maps/"

deployment/cloud/entry.sh

@@ -44,6 +44,19 @@ for g in "${G[@]}"; do g="${g// /}"; [ -n "$g" ] && ALLOW+=(--allowed-group="$g"
 export OAUTH2_PROXY_CLIENT_SECRET="$OIDC_CLIENT_SECRET"
 export OAUTH2_PROXY_COOKIE_SECRET="${OIDC_COOKIE_SECRET:-$(head -c 32 /dev/urandom | base64 | tr -d '=+/' | head -c 32)}"
 
+# Backend: PocketBase DEV (/pb/, signed-in) and PROD (/api-prod/, public API only). Schema = the
+# migrations in the repo; data = the /data volume. The superuser comes from env, never printed.
+MIG=/app/repo/backend/pocketbase/pb_migrations; mkdir -p "$MIG" /data/pb-dev /data/pb-prod /data/cdn
+for inst in dev prod; do
+  if [ -n "${PB_SUPERUSER_EMAIL:-}" ] && [ -n "${PB_SUPERUSER_PASSWORD:-}" ]; then
+    pocketbase superuser upsert "$PB_SUPERUSER_EMAIL" "$PB_SUPERUSER_PASSWORD" --dir "/data/pb-$inst" >/dev/null 2>&1 \
+      && echo "[pb-$inst] superuser set" || echo "[pb-$inst] superuser upsert failed"
+  fi
+done
+pocketbase serve --http 127.0.0.1:8090 --dir /data/pb-dev --migrationsDir "$MIG" &
+pocketbase serve --http 127.0.0.1:8091 --dir /data/pb-prod --migrationsDir "$MIG" &
+bun /app/repo/deployment/cloud/admin.mjs &
+
 VITE_HOST=127.0.0.1 VITE_PORT=5174 bun x vite --config "$CONFIG" &
 oauth2-proxy \
   --provider=oidc \
@@ -57,6 +70,12 @@ oauth2-proxy \
   --email-domain='*' \
   --provider-display-name="${OIDC_DISPLAY_NAME:-your identity provider}" \
   --upstream=http://127.0.0.1:5174/ \
+  --upstream=http://127.0.0.1:5175/cdn/ \
+  --upstream=http://127.0.0.1:5175/api-prod/ \
+  --upstream=http://127.0.0.1:5175/pb/ \
+  --upstream=http://127.0.0.1:5175/admin/ \
+  --skip-auth-route='^/cdn/' \
+  --skip-auth-route='^/api-prod/api/' \
   --http-address="0.0.0.0:$PORT" \
   --reverse-proxy=true \
   --cookie-secure=true \

deployment/cloud/public-exclusions.txt

@@ -0,0 +1,23 @@
+# What must NEVER reach a public copy of this repo or a public build (the giveaway export and the
+# container's /cdn/). ONE list, read by both: deployment/cloud/apply-exclusions.sh.
+#   <path>        a file or folder, relative to the repo root, removed
+#   name:<glob>   any file whose name matches (case-insensitive), anywhere, removed
+#   keep:<name>   an exception to the name: rules
+#   content:<re>  a regex that must not appear in any built file (checked, not removed)
+.claude
+.forgejo
+.the merge queue
+.merge-queue
+AGENT-BRIEF.md
+_archive
+data-private
+library/scripts/posters/snapshots/_granular-experiment.mjs.shelved
+plugin-designs/key-cities
+name:*field-access*
+name:*visa*.json
+name:.env
+name:.env.*
+name:*.pem
+name:*.key
+keep:.env.example
+content:field-access|Visa Requirements