doxa-maps/deployment/cloud/atomic-pull.mjs

// atomic-pull.mjs: the cloud dev server shows a landing to the browsers ALL AT ONCE, never half-written.
// Used by vite.cloud.config.mjs only: local `bun run dev` and every build are unchanged.
//
// WHY. entry.sh follows BRANCH with `git reset --hard` straight into the tree Vite serves and watches. A landing
// rewrites hundreds of files one after another, and Vite answered the first of them while git was still writing
// the rest: an open page was reloaded onto a mix of old and new modules, and its tab clicks then fetched nothing
// until it was reloaded by hand (the cloud server's log, measured). Vite also keeps an error raised while no
// browser is connected and shows it to the next browser that connects, so a module read half-written could put
// an error overlay on a page opened long after the pull had finished.
//
// HOW. A GATE opens the moment git takes its index lock (<git dir>/index.lock, watched with fs.watch): `git reset
// --hard` holds it from before its first file write to after its last (measured: 15-155 ms, every write inside,
// public/ included, which Vite does not watch). The gate stays open while the lock exists, then until the tree
// has been quiet for QUIET ms (ATOMIC_QUIET_MS, default 1200): each file event, and each raw notification from
// the kernel that comes before one, moves the closing later (Vite's own events for the files git wrote arrive
// after the lock is gone). A lock that came and went with no file event at all (`git status` refreshing the index)
// closes the gate at once. A file event with no lock (anything else writing the tree) opens the gate too. Never
// later than CEILING ms after it opened (ATOMIC_CEILING_MS, default 10000): a browser is never held longer.
// While the gate is open:
//   - what Vite sends the browsers about the tree (update, full-reload, prune, error) is held, not sent;
//   - HTTP requests wait (pages, modules, public/ files, the reconnect ping); the backend routes (/api/, /cdn/,
//     /api-prod/, /pb/, /admin/) never do, they don't read the tree;
//   - a dev-server restart (a config file changed) waits too, so it loads the new config once, complete.
// When it closes: the modules of the files touched (the file events, and `git diff` between the commit before
// and after when git moved HEAD) are invalidated, so none read mid-write is served from the cache and every
// other module still answers 304; the waiting requests go on; if anything was held the browsers get exactly ONE
// full-reload. With a restart waiting, the requests are answered 503 + Retry-After: 1 instead (this server is
// about to close; a page asked for gets a page that reloads itself in a second) and the restart runs, which
// reloads the browsers itself. The held messages are dropped, a stale error with them. An error raised while the
// gate is closed waits 250 ms first: a gate opening meanwhile means it came from a file git was still writing.
// One log line per gate, one at start ("armed"). A writer that never stops: at the ceiling the gate lets
// everything through and stays out of the way (plain Vite) until the tree has been quiet once.
//
// WHERE IT HOOKS (read from Vite's source, 5.4 as pinned in bun.lock, and 7.3):
//   - ws.send: in Vite 5 server.hot broadcasts through it (channel.send), and in Vite 6/7 server.ws IS the client
//     environment's hot channel (environments.client.hot) and server.hot. Every message to a browser passes here,
//     and an error that never reaches it is never buffered for the next browser either.
//   - the request gate is registered from a configureServer hook with order 'pre', before every other plugin's
//     middleware; Vite adds its own (transform, static files, the ping) after all of those.
//   - server.restart: what Vite calls when a config file or one of its imports changes.
//   - server.watcher (chokidar): add/change/unlink/addDir/unlinkDir open and extend the gate; raw only extends it.
//     What the server writes for itself under the tree is left out of the watcher by vite.cloud.config.mjs
//     (server.watch.ignored), so it never holds a page.
//
// Off with ATOMIC_QUIET_MS=0, and with PULL_INTERVAL=0 (a checkout someone edits by hand keeps plain hot reload).
import { execFile } from 'node:child_process'
import { existsSync, readFileSync, statSync, watch } from 'node:fs'
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'

const HELD = new Set(['update', 'full-reload', 'prune', 'error'])
const BACKEND = /^\/(?:api|cdn|api-prod|pb|admin)(?:\/|$)/
const EVENTS = ['add', 'change', 'unlink', 'addDir', 'unlinkDir']
const RELOAD = { type: 'full-reload', path: '*' }
const LOCK_POLL_MS = 50
const RESTARTING_PAGE = '<!doctype html><meta charset="utf-8"><meta http-equiv="refresh" content="1">' +
  '<title>Restarting</title><p>The dev server is restarting on a new version. This page reloads itself in a second.</p>\n'
const num = (v, d) => (v === undefined || v === null || v === '' || !Number.isFinite(Number(v)) ? d : Number(v))
const slash = (p) => String(p).split(sep).join('/')

export function atomicPull(opts = {}) {
  const env = opts.env || process.env
  const quiet = num(opts.quietMs ?? env.ATOMIC_QUIET_MS, 1200)
  const ceiling = Math.max(num(opts.ceilingMs ?? env.ATOMIC_CEILING_MS, 10000), quiet)
  const grace = num(opts.errorGraceMs, 250)
  const off = !(quiet > 0) || env.PULL_INTERVAL === '0'
  return {
    name: 'cloud-atomic-pull',
    enforce: 'pre',
    apply: 'serve',
    configureServer: {
      order: 'pre',
      handler(server) {
        if (!off) install(server, { quiet, ceiling, grace })
      },
    },
  }
}

// The checkout's git dir (a worktree's .git is a file naming it) and the dir its refs live in.
function findGit(root) {
  for (let dir = root; ; dir = dirname(dir)) {
    const dotGit = join(dir, '.git')
    try {
      if (statSync(dotGit).isDirectory()) return { top: dir, gitDir: dotGit, commonDir: dotGit }
      const m = /^gitdir:\s*(.+)$/m.exec(readFileSync(dotGit, 'utf8'))
      if (m) {
        const gitDir = resolve(dir, m[1].trim())
        let commonDir = gitDir
        try { commonDir = resolve(gitDir, readFileSync(join(gitDir, 'commondir'), 'utf8').trim()) } catch { /* not a worktree */ }
        return { top: dir, gitDir, commonDir }
      }
    } catch { /* no .git here */ }
    if (dirname(dir) === dir) return null
  }
}

function install(server, { quiet, ceiling, grace }) {
  const root = resolve(server.config.root)
  const logger = server.config.logger
  // Nothing to hold without a browser channel and a watcher (middleware mode, hmr or watch turned off).
  if (typeof server.ws?.send !== 'function' || !server.watcher) return
  const under = (file) => {
    const r = relative(root, resolve(root, String(file)))
    return r !== '..' && !r.startsWith('..' + sep) && !isAbsolute(r)
  }

  let openedAt = 0        // when the gate opened; 0 = closed
  let gateId = 0
  let opener = ''         // what opened it: 'git' (the index lock) or 'file' (a file event)
  let deadline = 0        // when it closes if nothing else changes
  let timer = null
  let confirm = null      // the setImmediate that double-checks a deadline that has passed
  let ceilingTimer = null
  let passing = false     // after a ceiling: plain Vite until the tree is quiet
  let passDeadline = 0
  let passTimer = null
  let held = 0
  let waiting = []
  let restart = null      // a restart asked for while the gate was open: { force, settle: [] }
  let restarting = false  // that restart is running: this server is about to close
  let closed = false      // this server instance has closed (a restart replaced it, or the process exits)
  let treeEvents = false  // a file event (or raw notification) under the root since the gate opened
  let touched = new Set() // the files those named: their modules are invalidated when the gate closes
  let lockHeld = false    // git's index lock appeared during this gate and has not been seen gone
  let headBefore = ''     // HEAD when git took the lock: `git diff` against HEAD at closing names what it wrote
  let closing = false     // that `git diff` is running
  let landed = null       // what it found: { from, to, files }, for the log line
  const graceErrors = new Set()

  // Messages to the browsers.
  const ws = server.ws
  const send = ws.send
  ws.send = function atomicPullSend(...args) {
    const type = typeof args[0] === 'string' ? 'custom' : args[0]?.type
    if (closed || !HELD.has(type)) return send.apply(this, args)
    if (openedAt) { held++; return }
    if (type === 'error' && grace > 0 && !passing) {
      const t = setTimeout(() => { graceErrors.delete(t); if (!closed) send.apply(ws, args) }, grace)
      graceErrors.add(t)
      return
    }
    return send.apply(this, args)
  }

  // Requests. While a restart runs, this server is about to close: a request it took in would be cut off (504)
  // or answered with the old config, so it is told to come back in a second; the restarted server answers then.
  const unavailable = (req, res) => {
    if (res.headersSent || res.writableEnded || res.destroyed) return
    const page = /\btext\/html\b/.test(String(req.headers?.accept || ''))
    res.writeHead(503, { 'Retry-After': '1', 'Cache-Control': 'no-store', 'Content-Type': `${page ? 'text/html' : 'text/plain'}; charset=utf-8` })
    res.end(page ? RESTARTING_PAGE : 'the dev server is restarting on a new version: retry in a second\n')
  }
  server.middlewares.use(function atomicPullGate(req, res, next) {
    if (closed || BACKEND.test((req.url || '').split('?')[0])) return next()
    if (restarting) return unavailable(req, res)
    if (!openedAt) return next()
    waiting.push({ req, res, next })
  })

  // Restarts.
  const realRestart = server.restart
  server.restart = function atomicPullRestart(forceOptimize) {
    if (!openedAt || closed) return realRestart.call(server, forceOptimize)
    restart ||= { force: false, settle: [] }
    restart.force ||= !!forceOptimize
    return new Promise((ok, fail) => restart.settle.push((e) => (e ? fail(e) : ok())))
  }

  // The gate's clock.
  const schedule = () => {
    clearTimeout(timer); clearImmediate(confirm); confirm = null
    timer = setTimeout(tick, lockHeld ? LOCK_POLL_MS : Math.max(0, deadline - Date.now()))
  }
  const tick = () => {
    timer = null
    // git is still writing: look at its lock again shortly (a missed "gone" notification costs LOCK_POLL_MS).
    if (lockHeld) {
      if (existsSync(lockPath)) { timer = setTimeout(tick, LOCK_POLL_MS); return }
      lockGone(Date.now())
    }
    const left = deadline - Date.now()
    if (left > 0) { timer = setTimeout(tick, left); return }
    // A busy process can run this timer before the file events that queued up meanwhile: look again right after
    // the event loop's next I/O poll has read them.
    confirm = setImmediate(() => {
      confirm = null
      if (lockHeld) { timer = setTimeout(tick, LOCK_POLL_MS); return }
      const again = deadline - Date.now()
      if (again > 0) timer = setTimeout(tick, again)
      else release(false)
    })
  }
  const passTick = () => {
    const left = passDeadline - Date.now()
    if (left > 0) { passTimer = setTimeout(passTick, left); return }
    passTimer = null
    passing = false
  }

  function open(now, by) {
    openedAt = now
    gateId++
    opener = by
    deadline = by === 'git' ? now : now + quiet
    treeEvents = false
    touched = new Set()
    headBefore = ''
    landed = null
    lockHeld = false
    timer = setTimeout(tick, by === 'git' ? LOCK_POLL_MS : quiet)
    ceilingTimer = setTimeout(() => release(true), ceiling)
    // An error sent just before the gate opened came from a file git had begun to write.
    for (const t of graceErrors) { clearTimeout(t); held++ }
    graceErrors.clear()
  }

  const onEvent = (file) => {
    if (closed || restarting || !under(file)) return
    const now = Date.now()
    if (passing) { passDeadline = now + quiet; return }
    if (!openedAt) open(now, 'file')
    treeEvents = true
    touched.add(slash(resolve(root, String(file))))
    deadline = Math.max(deadline, now + quiet)
  }
  // chokidar's raw events come straight from the kernel's notification, before the file's stat that its add and
  // change events wait for (on a loaded machine that stat lagged the writes by over a second): they keep an open
  // gate open while git is still writing. They never open one: a read that updates a file's access time is one.
  const onRaw = (_event, name, details) => {
    const watched = details?.watchedPath
    if (closed || restarting || !(openedAt || passing) || !under(watched ?? root)) return
    const until = Date.now() + quiet
    if (passing) { passDeadline = Math.max(passDeadline, until); return }
    deadline = Math.max(deadline, until)
    treeEvents = true
    if (watched) {
      touched.add(slash(watched))
      if (name) touched.add(slash(join(watched, String(name))))
    }
  }
  // First in line, ahead of Vite's own handlers, so the gate is open before anything they send.
  const watcher = server.watcher
  for (const ev of EVENTS) {
    if (typeof watcher.prependListener === 'function') watcher.prependListener(ev, onEvent)
    else watcher.on(ev, onEvent)
  }
  watcher.on('raw', onRaw)

  // git's index lock. The kernel reports it on the same inotify queue as the tree, in order: its creation before
  // git's first write, its removal after the last. A lock already gone when its notification is read still opens
  // the gate (git was writing a moment ago); the notifications of what it wrote are read in the same batch.
  const git = findGit(root)
  const lockPath = git ? join(git.gitDir, 'index.lock') : ''
  const readHead = () => {
    try {
      let v = readFileSync(join(git.gitDir, 'HEAD'), 'utf8').trim()
      for (let i = 0; i < 4 && v.startsWith('ref: '); i++) {
        const ref = v.slice(5).trim()
        try { v = readFileSync(join(git.commonDir, ref), 'utf8').trim() } catch {
          const line = readFileSync(join(git.commonDir, 'packed-refs'), 'utf8').split('\n').find((l) => l.endsWith(' ' + ref))
          v = line ? line.split(' ')[0] : ''
        }
      }
      return /^[0-9a-f]{40}([0-9a-f]{24})?$/.test(v) ? v : ''
    } catch { return '' }
  }
  function lockGone(now) {
    lockHeld = false
    // Nothing under the tree moved while git held the lock: it wrote nothing Vite watches (`git status`, or only
    // public/), and every write it made is done. Otherwise wait for Vite's own (later) events of those writes.
    deadline = Math.max(deadline, now + (treeEvents ? quiet : 0))
  }
  const onGit = (_type, name) => {
    if (closed || restarting || (name && String(name) !== 'index.lock')) return
    const now = Date.now()
    if (passing) { passDeadline = Math.max(passDeadline, now + quiet); return }
    if (!openedAt) open(now, 'git')
    if (!headBefore) headBefore = readHead()
    if (existsSync(lockPath)) { lockHeld = true; return }
    lockGone(now)
    if (!closing) schedule()   // else the running `git diff` looks at lockHeld and the deadline when it ends
  }
  let lockWatch = null
  let lockPoll = null
  let lockMode = 'off (no .git above the root)'
  const pollLock = () => {
    let was = existsSync(lockPath)
    lockPoll = setInterval(() => {
      const is = existsSync(lockPath)
      if (is !== was) { was = is; onGit('poll', 'index.lock') }
    }, LOCK_POLL_MS)
    lockPoll.unref?.()
  }
  if (git) {
    try {
      lockWatch = watch(git.gitDir, { persistent: false }, onGit)
      lockWatch.on('error', (e) => {
        lockWatch?.close(); lockWatch = null
        logger.warn(`[atomic-pull] git-lock watch failed (${e?.code || e?.message}): polling it every ${LOCK_POLL_MS} ms`)
        if (!closed && !lockPoll) pollLock()
      })
      lockMode = 'on'
    } catch (e) {
      pollLock()
      lockMode = `polled every ${LOCK_POLL_MS} ms (fs.watch: ${e?.code || e?.message})`
    }
  }

  // `git diff` between the commit before the lock and HEAD now: every file git wrote, even one whose file event
  // was lost. Bounded; on any failure the file events alone decide.
  const changedFiles = (a, b) => new Promise((ok) => {
    execFile('git', ['diff', '--name-only', '-z', '--no-renames', a, b],
      { cwd: git.top, timeout: 2000, maxBuffer: 64 << 20, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } },
      (e, out) => ok(e ? [] : String(out).split('\0').filter(Boolean).map((p) => slash(resolve(git.top, p)))))
  })
  const graphs = () => (server.environments
    ? Object.values(server.environments).map((e) => e.moduleGraph)
    : [server.moduleGraph]).filter((g) => typeof g?.getModulesByFile === 'function')
  // Only the touched files' modules (and, as Vite does on a change, their importers): an unchanged module keeps
  // its transform, so it still answers 304 without being compiled again.
  const invalidate = (files) => {
    let n = 0
    for (const g of graphs()) {
      const seen = new Set()
      for (const f of files) for (const mod of g.getModulesByFile(f) || []) { g.invalidateModule(mod, seen); n++ }
    }
    return n
  }

  function release(byCeiling) {
    if (!openedAt) return
    if (closing && !byCeiling) return
    if (!closing && !byCeiling && headBefore) {
      const head = readHead()
      if (head && head !== headBefore) {
        clearTimeout(timer); clearImmediate(confirm); timer = confirm = null
        closing = true
        const id = gateId
        changedFiles(headBefore, head).then((files) => {
          if (id !== gateId || !openedAt) { try { invalidate(files) } catch { /* the gate already closed */ } return }
          closing = false
          landed = { from: landed?.from || headBefore, to: head, files: (landed?.files || 0) + files.length }
          headBefore = head
          for (const f of files) touched.add(f)
          if (lockHeld || deadline > Date.now()) schedule()   // git or the tree moved again meanwhile
          else release(false)
        })
        return
      }
    }
    finish(byCeiling)
  }

  function finish(byCeiling) {
    const took = Date.now() - openedAt
    clearTimeout(timer); clearImmediate(confirm); clearTimeout(ceilingTimer)
    timer = confirm = ceilingTimer = null
    openedAt = 0
    closing = false
    lockHeld = false
    const messages = held
    const requests = waiting
    const r = restart
    const files = touched
    const land = landed ? `; git ${landed.from.slice(0, 9)}..${landed.to.slice(0, 9)}, ${landed.files} files` : ''
    held = 0; waiting = []; restart = null; touched = new Set(); landed = null
    if (byCeiling) {
      passing = true
      passDeadline = Date.now() + quiet
      passTimer = setTimeout(passTick, quiet)
    }
    let mods = 0
    try { mods = invalidate(files) } catch (e) { logger.warn(`[atomic-pull] could not invalidate the module graph: ${e?.message || e}`) }
    if (r) restarting = true
    for (const w of requests) {
      if (w.res.writableEnded || w.res.destroyed || w.req.socket?.destroyed) continue
      if (r) unavailable(w.req, w.res)
      else w.next()
    }
    if (messages && !r) send.call(ws, RELOAD)
    // A lock that came and went with nothing held and nothing written (`git status`) is not worth a line.
    if (messages || requests.length || r || byCeiling || files.size) {
      const outcome = r ? `server restart (${requests.length} requests answered 503, retry in 1 s)` : messages ? '1 full-reload' : 'no reload'
      const why = byCeiling ? `; ceiling ${ceiling} ms reached, passing through until the tree is quiet` : ''
      const by = opener === 'git' ? "git's index lock" : 'a file change'
      logger.info(`[atomic-pull] held ${messages} messages, ${requests.length} requests for ${took} ms (opened by ${by}${land}; ${mods} modules invalidated); ${outcome}${why}`, { timestamp: true })
    }
    if (r) runRestart(r, messages)
  }

  async function runRestart(r, messages) {
    let err = null
    try { await realRestart.call(server, r.force) } catch (e) { err = e }
    restarting = false
    for (const settle of r.settle) settle(err)
    // A failed restart leaves this server serving (Vite logs "server restart failed"): reload the pages from here.
    if (messages && !closed && server.ws === ws) send.call(ws, RELOAD)
  }

  // One line when this server is up, so a rollout can be confirmed in the log.
  const armed = () => {
    if (!closed) logger.info(`[atomic-pull] armed: quiet ${quiet} ms, cap ${ceiling} ms, git-lock watch ${lockMode}`, { timestamp: true })
  }
  const hs = server.httpServer
  if (hs && !hs.listening) hs.once('listening', armed)
  else armed()

  hs?.once('close', () => {
    closed = true
    clearTimeout(timer); clearImmediate(confirm); clearTimeout(ceilingTimer); clearTimeout(passTimer)
    for (const t of graceErrors) clearTimeout(t)
    graceErrors.clear()
    waiting = []   // their sockets closed with the server
    lockWatch?.close(); clearInterval(lockPoll)
    for (const ev of EVENTS) watcher.off?.(ev, onEvent)
    watcher.off?.('raw', onRaw)
  })
}