diff --git a/backend/pocketbase/README.md b/backend/pocketbase/README.md new file mode 100644 index 000000000..5b3992bbc --- /dev/null +++ b/backend/pocketbase/README.md @@ -0,0 +1,13 @@ +# PocketBase โ€” the backend for plugins that need one + +The cloud container ([deployment/cloud](../../deployment/cloud/README.md)) runs two PocketBase +instances: **dev** (behind the team sign-in, for building) and **prod** (its `/api/` public, for +published plugins). Both apply the migrations in [`pb_migrations/`](./pb_migrations/) on start, so +**the schema lives in git** and publishing never copies data from dev to prod. + +A plugin reads its backend address at runtime (`profile-config` `apiBase`, or +`window.MAP_API_BASE` set by the published build's `runtime-config.js`), so moving the backend never +needs a rebuild. + +v1 limits: collection rules are the only access control (lock collections by default); no per-plugin +isolation between plugin authors yet. diff --git a/backend/pocketbase/pb_migrations/.gitkeep b/backend/pocketbase/pb_migrations/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/cloud/Dockerfile b/deployment/cloud/Dockerfile index b0cf88d11..651c1a816 100644 --- a/deployment/cloud/Dockerfile +++ b/deployment/cloud/Dockerfile @@ -4,9 +4,15 @@ # container host (Coolify "Dockerfile" build pack, Railway, or `docker build`). FROM quay.io/oauth2-proxy/oauth2-proxy:v7.15.3 AS oauth2proxy FROM oven/bun:1-debian -RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \ +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates curl unzip \ && rm -rf /var/lib/apt/lists/* COPY --from=oauth2proxy /bin/oauth2-proxy /usr/local/bin/oauth2-proxy +# PocketBase, pinned and checked: the backend for plugins (a dev and a prod instance, see entry.sh). +ARG PB_VERSION=0.40.4 +ARG PB_SHA256=9042ec818570e79c3628dadcd0a756c1496d9e1173918ec409d133c02f82e5fa +RUN curl -fsSL -o /tmp/pb.zip "https://github.com/pocketbase/pocketbase/releases/download/v${PB_VERSION}/pocketbase_${PB_VERSION}_linux_amd64.zip" \ + && echo "${PB_SHA256} /tmp/pb.zip" | sha256sum -c - \ + && unzip -q /tmp/pb.zip pocketbase -d /usr/local/bin && rm /tmp/pb.zip && chmod 755 /usr/local/bin/pocketbase RUN printf '%s\n' \ '#!/bin/bash' \ 'set -u; export GIT_TERMINAL_PROMPT=0' \ @@ -17,5 +23,6 @@ RUN printf '%s\n' \ 'cp /app/repo/deployment/cloud/entry.sh /tmp/entry.sh && exec bash /tmp/entry.sh' \ > /boot.sh && chmod 755 /boot.sh ENV PORT=5173 BRANCH=main PULL_INTERVAL=15 +VOLUME /data EXPOSE 5173 CMD ["/boot.sh"] diff --git a/deployment/cloud/README.md b/deployment/cloud/README.md index c613430cc..6b6ea35ba 100644 --- a/deployment/cloud/README.md +++ b/deployment/cloud/README.md @@ -43,3 +43,31 @@ with the redirect URL `https:///oauth2/callback`; PKCE (S256) is used | `OIDC_COOKIE_SECRET` | 32 characters; keeps people signed in across restarts (random if unset) | Secrets reach oauth2-proxy through its environment, never its command line. + +## Backend and production (v1, experimental) + +With the sign-in on, the container also runs PocketBase and can publish a production build: + +| Route | Who | What | +|---|---|---| +| `/` | signed-in | the dev server (Plugin Studio, hot reload) | +| `/pb/` | signed-in (its `/_/` dashboard: `OIDC_ADMIN_GROUPS` only) | PocketBase **dev** | +| `/api-prod/api/` | **public** | PocketBase **prod**'s API; its dashboard is never exposed | +| `/cdn/dx-maps/` | **public** | the last published build: `manifest.json` at a stable URL, CORS open for embeds | +| `POST /admin/build` ยท `GET /admin/status` | `OIDC_ADMIN_GROUPS` only | build and publish the pulled commit | + +A build runs one at a time (a second gets `409`), on a snapshot of the pulled commit, with Node's heap +capped. It removes everything in [`public-exclusions.txt`](./public-exclusions.txt) **before** +building, checks the output again afterwards, and refuses to publish if anything excluded is still +there. Only then does it swap the new build in. + +| Variable | Meaning | +|---|---| +| `OIDC_ADMIN_GROUPS` | comma-separated groups allowed to build and to open PocketBase's dashboard | +| `PB_SUPERUSER_EMAIL`, `PB_SUPERUSER_PASSWORD` | PocketBase's superuser, set on start in both instances | +| `PUBLIC_API_BASE` | the public API URL written into the build's `runtime-config.js` (e.g. `https:///api-prod`) | +| `BUILD_MAX_OLD_SPACE_MB` | Node heap cap for a build (default `1536`) | + +Mount a persistent volume at **`/data`** (PocketBase data and published builds). Memory measured on +the running container: about 0.45 GB idle, up to about 1.5 GB more during a build, so give it +**2.5 GB**. diff --git a/deployment/cloud/admin.mjs b/deployment/cloud/admin.mjs new file mode 100644 index 000000000..a870d495a --- /dev/null +++ b/deployment/cloud/admin.mjs @@ -0,0 +1,66 @@ +// admin.mjs โ€” the container's internal router (127.0.0.1:5175), behind oauth2-proxy. +// /cdn/* the last good build (/data/cdn/current), PUBLIC (oauth2-proxy skips auth here) +// /api-prod/* PocketBase PROD, PUBLIC; only /api/ passes, never the /_/ dashboard +// /pb/* PocketBase DEV, signed-in; its /_/ dashboard only for OIDC_ADMIN_GROUPS +// /admin/build POST: start a build (OIDC_ADMIN_GROUPS only); /admin/status GET +// Groups come from oauth2-proxy's X-Forwarded-Groups, which it sets on signed-in routes only. +import { spawn } from 'node:child_process' +import { existsSync, readFileSync, statSync } from 'node:fs' +import { join, normalize } from 'node:path' + +const env = process.env +const ADMIN = (env.OIDC_ADMIN_GROUPS || '').split(',').map((g) => g.trim()).filter(Boolean) +const CDN = '/data/cdn/current' +let last = { state: 'idle' } + +const groupsOf = (req) => (req.headers.get('x-forwarded-groups') || '').split(',').map((g) => g.trim()) +const isAdmin = (req) => ADMIN.length > 0 && groupsOf(req).some((g) => ADMIN.includes(g)) +const text = (s, status = 200) => new Response(s + '\n', { status, headers: { 'content-type': 'text/plain; charset=utf-8' } }) + +async function proxy(req, url, prefix, port) { + const target = new URL(url.pathname.slice(prefix.length - 1) + url.search, `http://127.0.0.1:${port}`) + const r = await fetch(target, { method: req.method, headers: req.headers, body: ['GET', 'HEAD'].includes(req.method) ? undefined : req.body, redirect: 'manual' }) + return new Response(r.body, { status: r.status, headers: r.headers }) +} + +Bun.serve({ + hostname: '127.0.0.1', + port: 5175, + async fetch(req) { + const url = new URL(req.url) + const p = url.pathname + if (p.startsWith('/cdn/')) { + const f = normalize(join(CDN, decodeURIComponent(p.slice(5)))) + if (!f.startsWith(CDN)) return text('bad path', 400) + const file = existsSync(f) && statSync(f).isDirectory() ? join(f, 'index.html') : f + if (!existsSync(file)) return text('not found', 404) + return new Response(Bun.file(file), { headers: { 'access-control-allow-origin': '*', 'cache-control': 'public, max-age=86400, stale-while-revalidate=604800' } }) + } + if (p.startsWith('/api-prod/')) { + if (!p.startsWith('/api-prod/api/')) return text('only /api/ is public', 403) + return proxy(req, url, '/api-prod/', 8091) + } + if (p.startsWith('/pb/')) { + if (p.startsWith('/pb/_/') && !isAdmin(req)) return text('the PocketBase dashboard is for admins', 403) + return proxy(req, url, '/pb/', 8090) + } + if (p === '/admin/status') return Response.json(last) + if (p === '/admin/build') { + if (req.method !== 'POST') return text('POST only', 405) + if (!isAdmin(req)) return text('builds are for team admins', 403) + if (last.state === 'building') return text('a build is already running', 409) + last = { state: 'building', started: new Date().toISOString() } + const child = spawn('bash', ['/app/repo/deployment/cloud/build-cdn.sh'], { stdio: ['ignore', 'pipe', 'pipe'] }) + let log = '' + child.stdout.on('data', (d) => { log = (log + d).slice(-4000) }) + child.stderr.on('data', (d) => { log = (log + d).slice(-4000) }) + child.on('exit', (code) => { + last = { state: code === 0 ? 'done' : code === 75 ? 'busy' : 'failed', code, finished: new Date().toISOString(), tail: log.slice(-1500), + commit: existsSync('/data/cdn/COMMIT') ? readFileSync('/data/cdn/COMMIT', 'utf8').trim() : null } + }) + return text('build started', 202) + } + return text('not found', 404) + }, +}) +console.log('[admin] listening on 127.0.0.1:5175') diff --git a/deployment/cloud/apply-exclusions.sh b/deployment/cloud/apply-exclusions.sh new file mode 100755 index 000000000..960929f26 --- /dev/null +++ b/deployment/cloud/apply-exclusions.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# apply-exclusions.sh remove everything public-exclusions.txt excludes from +# apply-exclusions.sh --check exit 1 if anything excluded is still in (nothing removed) +set -u +LIST="$(dirname "$0")/public-exclusions.txt" +MODE=apply; [ "${1:-}" = --check ] && { MODE=check; shift; } +DIR="${1:?usage: apply-exclusions.sh [--check] }" +[ -d "$DIR" ] || { echo "no such dir: $DIR" >&2; exit 2; } +keep=(); names=(); paths=(); contents=() +while IFS= read -r l; do + case "$l" in ''|'#'*) ;; keep:*) keep+=("${l#keep:}") ;; name:*) names+=("${l#name:}") ;; + content:*) contents+=("${l#content:}") ;; *) paths+=("$l") ;; esac +done < "$LIST" +found=0 +for p in "${paths[@]}"; do + # at the repo root, and anywhere below it (a built plugin sits at dx-maps/) + while IFS= read -r hit; do + [ -e "$hit" ] || continue + if [ $MODE = apply ]; then rm -rf "$hit"; echo " excluded: $hit"; else echo " PRESENT: $hit"; found=1; fi + done < <( { [ -e "$DIR/$p" ] && echo "$DIR/$p"; find "$DIR" -path "*/$(basename "$p")" -not -path '*/node_modules/*' -prune 2>/dev/null; } | sort -u) +done +for n in "${names[@]}"; do + while IFS= read -r f; do + b=$(basename "$f"); skip=0; for k in "${keep[@]}"; do [ "$b" = "$k" ] && skip=1; done + [ $skip = 1 ] && continue + if [ $MODE = apply ]; then rm -rf "$f"; echo " excluded: $f"; else echo " PRESENT: $f"; found=1; fi + done < <(find "$DIR" -iname "$n" -not -path '*/node_modules/*' 2>/dev/null) +done +if [ $MODE = check ]; then + for c in "${contents[@]}"; do + hits=$(grep -rIlE "$c" "$DIR" --exclude-dir=node_modules 2>/dev/null | head -5) + [ -n "$hits" ] && { echo " CONTENT /$c/ in:"; echo "$hits" | sed 's/^/ /'; found=1; } + done + [ $found = 1 ] && { echo "EXCLUSIONS: FAIL"; exit 1; } + echo "EXCLUSIONS: clean" +fi +exit 0 diff --git a/deployment/cloud/build-cdn.sh b/deployment/cloud/build-cdn.sh new file mode 100755 index 000000000..7eb3c694f --- /dev/null +++ b/deployment/cloud/build-cdn.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# build-cdn.sh โ€” one production build of the pulled commit into /data/cdn, one at a time. +# snapshot (so the pull loop can't change files mid-build) -> apply the public exclusions -> +# BUILD_ALLOW=1 bun run build (memory-capped) -> check the output -> atomic swap of /data/cdn/current. +# Exit 75 = another build holds the lock. +set -u +exec 9>/tmp/build-cdn.lock +flock -n 9 || { echo "another build is running"; exit 75; } +cd /app/repo +SHA=$(git rev-parse --short HEAD); SNAP=/tmp/build-$SHA +rm -rf "$SNAP"; mkdir -p "$SNAP" +git archive HEAD | tar -x -C "$SNAP" || { echo "snapshot failed"; exit 1; } +bash /app/repo/deployment/cloud/apply-exclusions.sh "$SNAP" +ln -s /app/repo/node_modules "$SNAP/node_modules" +cd "$SNAP" +echo "$(date -u +%FT%TZ) building $SHA" +NODE_OPTIONS="--max-old-space-size=${BUILD_MAX_OLD_SPACE_MB:-1536}" BUILD_ALLOW=1 nice -n 10 bun run build \ + || { echo "build failed"; rm -rf "$SNAP"; exit 1; } +OUT="$SNAP/public/dx-maps" +[ -f "$OUT/manifest.json" ] || { echo "no manifest.json in the output"; rm -rf "$SNAP"; exit 1; } +printf 'window.MAP_API_BASE=%s;\n' "$(printf '%s' "${PUBLIC_API_BASE:-}" | python3 -c 'import json,sys;print(json.dumps(sys.stdin.read()))')" > "$OUT/runtime-config.js" +NEW=/data/cdn/build-$SHA +rm -rf "$NEW"; mkdir -p /data/cdn; mv "$SNAP/public" "$NEW" +bash /app/repo/deployment/cloud/apply-exclusions.sh --check "$NEW" || { echo "REFUSED: excluded content in the build; not published"; rm -rf "$NEW" "$SNAP"; exit 1; } +ln -sfn "$NEW" /data/cdn/current.tmp && mv -T /data/cdn/current.tmp /data/cdn/current +echo "$SHA" > /data/cdn/COMMIT +find /data/cdn -maxdepth 1 -name 'build-*' ! -path "$NEW" -exec rm -rf {} + +rm -rf "$SNAP" +echo "$(date -u +%FT%TZ) published $SHA at /cdn/dx-maps/" diff --git a/deployment/cloud/entry.sh b/deployment/cloud/entry.sh index 26c3193d4..ead832708 100755 --- a/deployment/cloud/entry.sh +++ b/deployment/cloud/entry.sh @@ -44,6 +44,19 @@ for g in "${G[@]}"; do g="${g// /}"; [ -n "$g" ] && ALLOW+=(--allowed-group="$g" export OAUTH2_PROXY_CLIENT_SECRET="$OIDC_CLIENT_SECRET" export OAUTH2_PROXY_COOKIE_SECRET="${OIDC_COOKIE_SECRET:-$(head -c 32 /dev/urandom | base64 | tr -d '=+/' | head -c 32)}" +# Backend: PocketBase DEV (/pb/, signed-in) and PROD (/api-prod/, public API only). Schema = the +# migrations in the repo; data = the /data volume. The superuser comes from env, never printed. +MIG=/app/repo/backend/pocketbase/pb_migrations; mkdir -p "$MIG" /data/pb-dev /data/pb-prod /data/cdn +for inst in dev prod; do + if [ -n "${PB_SUPERUSER_EMAIL:-}" ] && [ -n "${PB_SUPERUSER_PASSWORD:-}" ]; then + pocketbase superuser upsert "$PB_SUPERUSER_EMAIL" "$PB_SUPERUSER_PASSWORD" --dir "/data/pb-$inst" >/dev/null 2>&1 \ + && echo "[pb-$inst] superuser set" || echo "[pb-$inst] superuser upsert failed" + fi +done +pocketbase serve --http 127.0.0.1:8090 --dir /data/pb-dev --migrationsDir "$MIG" & +pocketbase serve --http 127.0.0.1:8091 --dir /data/pb-prod --migrationsDir "$MIG" & +bun /app/repo/deployment/cloud/admin.mjs & + VITE_HOST=127.0.0.1 VITE_PORT=5174 bun x vite --config "$CONFIG" & oauth2-proxy \ --provider=oidc \ @@ -57,6 +70,12 @@ oauth2-proxy \ --email-domain='*' \ --provider-display-name="${OIDC_DISPLAY_NAME:-your identity provider}" \ --upstream=http://127.0.0.1:5174/ \ + --upstream=http://127.0.0.1:5175/cdn/ \ + --upstream=http://127.0.0.1:5175/api-prod/ \ + --upstream=http://127.0.0.1:5175/pb/ \ + --upstream=http://127.0.0.1:5175/admin/ \ + --skip-auth-route='^/cdn/' \ + --skip-auth-route='^/api-prod/api/' \ --http-address="0.0.0.0:$PORT" \ --reverse-proxy=true \ --cookie-secure=true \ diff --git a/deployment/cloud/public-exclusions.txt b/deployment/cloud/public-exclusions.txt new file mode 100644 index 000000000..6c9d421fa --- /dev/null +++ b/deployment/cloud/public-exclusions.txt @@ -0,0 +1,23 @@ +# What must NEVER reach a public copy of this repo or a public build (the giveaway export and the +# container's /cdn/). ONE list, read by both: deployment/cloud/apply-exclusions.sh. +# a file or folder, relative to the repo root, removed +# name: any file whose name matches (case-insensitive), anywhere, removed +# keep: an exception to the name: rules +# content: a regex that must not appear in any built file (checked, not removed) +.claude +.forgejo +.main-capture +.merge-queue +AGENT-BRIEF.md +_archive +data-private +library/scripts/posters/snapshots/_granular-experiment.mjs.shelved +plugin-designs/key-cities +name:*field-access* +name:*visa*.json +name:.env +name:.env.* +name:*.pem +name:*.key +keep:.env.example +content:field-access|Visa Requirements