doxa-maps/plugin-designs/gospel-resources/scripts/fetch-arclight-share-urls.mjs

#!/usr/bin/env node
/**
 * fetch-arclight-share-urls.mjs — get the REAL per-language JESUS Film deep link.
 *
 * THE DEFECT THIS FIXES. We shipped https://www.jesusfilm.org/watch/?languageId=<id> for
 * 2,087 languages. MEASURED IN A BROWSER: the current site ignores that query parameter —
 * a real id and a bogus id render the same generic library page. The film was real and
 * the link did not carry anyone to it.
 *
 * Arclight's media-components endpoint returns a shareUrl per language, and that IS a
 * deep link: /watch/jesus/german-standard. It discriminates, which the old URL never did —
 * a real slug returns ~900KB carrying the media component id `1_jf-0-0` 369 times, a
 * bogus slug returns a 33KB page carrying it 0 times.
 *
 * WHY FETCHED PER LANGUAGE AND NOT DERIVED FROM THE NAME. The slug looks derivable —
 * "Pashto, Southern" -> pashto-southern — and 8 of 8 sampled names transformed correctly.
 * Eight is not 2,087, and a wrong slug is not a broken link but a link to SOMEONE ELSE'S
 * language, which is the failure this repo guards hardest against. The API is
 * authoritative, so it is asked. Rate-limited, resumable, and cached so it is asked once.
 */
import { readFileSync, writeFileSync, existsSync } from 'node:fs'
import { writeSnapshot } from './_snapshot-guard.mjs'
import { resolve, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'

const HERE = dirname(fileURLToPath(import.meta.url))
const REPO = resolve(HERE, '../../..')
const ARC = resolve(REPO, 'data/tables/arclight-index.json')
const OUT = resolve(REPO, 'data/tables/arclight-share-urls.json')
const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
const JF = '1_jf-0-0'

/**
 * `jos` IS SKIPPED, AND THE REASON IS UPSTREAM. ISO 639-3 jos is Jordanian Sign Language.
 * Arclight's roster files iso3 "jos" under languageId 139080, whose own name field reads
 * "Lebanese Sign Language" — a different language (lbs). Meanwhile the JESUS Film DOES exist
 * at languageId 7138, named Jordanian Sign Language, which is the id the pin index carries.
 *
 * So the ISO join — the thing this whole map is built on — disagrees with the publisher's own
 * label for this one code, and the fallback below resolved 139080 to Magdalena: the wrong film
 * AND the wrong language, shown to three people groups. Left exactly as it was rather than
 * guessed at. Resolving it means deciding whether the code or the name is authoritative here,
 * and that is a data question for someone who can check it against Ethnologue, not something
 * to settle inside a fetcher.
 */
const SKIP = new Set(['jos'])
const DELAY = Number(process.env.DELAY_MS || 250)

const arc = JSON.parse(readFileSync(ARC, 'utf8')).codes
const prior = existsSync(OUT) ? JSON.parse(readFileSync(OUT, 'utf8')) : { codes: {} }
const out = { ...prior.codes }
const entries = Object.entries(arc)
let fetched = 0, cached = 0, missing = 0

for (let i = 0; i < entries.length; i++) {
  const [code, v] = entries[i]
  if (SKIP.has(code)) { cached++; continue }
  if (out[code] !== undefined) { cached++; continue }
  try {
    const r = await fetch(`https://api.arclight.org/v2/media-components/${JF}/languages/${v.id}`, { headers: { 'User-Agent': UA } })
    if (r.ok) {
      const d = await r.json()
      // https, always: the API returns http:// and a plain-http link in a field context
      // is a downgrade nobody asked for.
      const u = String(d.shareUrl || '').replace(/^http:\/\//, 'https://')
      if (u) { out[code] = u; fetched++ } else missing++
    } else if (r.status !== 404) {
      /* ONLY A 404 MEANS "THE JESUS COMPONENT HAS NO ENTRY FOR THIS LANGUAGE". A 429, a 500
       * or a blip means we do not know yet, and the fallback below must not run on it.
       * Measured: the first version treated ANY non-ok as "no JESUS Film" and wrote Magdalena
       * over Jordanian Sign Language, which HAS the JESUS Film at arc.gt/g8wmw — a transient
       * failure silently downgrading a language to a different film. Left undefined so the
       * next run retries it. */
      missing++
    } else {
      /* ═══ THE JESUS FILM COMPONENT DOES NOT CARRY EVERY LANGUAGE THE ROSTER COUNTS ═══
       *
       * counts.featureFilm >= 1 says a FEATURE exists in this language. It does not say the
       * feature is JESUS. For five codes this endpoint 404s, and the build was falling back
       * to the retired ?languageId= URL — unverifiable by fetch, and labelled "JESUS Film"
       * on a language the JESUS component has no entry for. Measured 2026-09-25:
       *   ars  Arabic, Najdi     -> The Savior                    (8 groups, film was their ONLY resource)
       *   gjk  Kachi Koli        -> Magdalena — Director's Cut
       *   mzn  Mazanderani       -> Magdalena — Director's Cut
       *   lia  Limba, West-Central -> NO component at all
       *   jos  Jordanian Sign Language -> the JESUS Film after all, at arc.gt/g8wmw
       *
       * So ask what feature this language actually has. A real JESUS Film Project feature in
       * someone's own language is worth shipping — dropping ars would have pushed 8 groups
       * into the gap while a gospel film exists in their language — but it must be shipped
       * UNDER ITS OWN NAME. This bundle already refuses to call Open Bible Stories Scripture;
       * calling The Savior the JESUS Film is the same lie.
       *
       * Same publisher, so the doctrinal screen is unchanged: these are JESUS Film Project
       * features, the catalogue the brief names explicitly. Nothing outside it is taken.
       *
       * `null` is recorded when no feature exists, and the pin build DROPS the film rather
       * than falling back to a URL nobody can check. */
      let alt = null
      try {
        const c = await fetch(`https://api.arclight.org/v2/media-components?languageIds=${v.id}&limit=25`, { headers: { 'User-Agent': UA } })
        if (c.ok) {
          const cj = await c.json()
          const feature = (cj._embedded?.mediaComponents || []).find((m) => m.subType === 'featureFilm')
          if (feature) {
            const fr = await fetch(`https://api.arclight.org/v2/media-components/${feature.mediaComponentId}/languages/${v.id}`, { headers: { 'User-Agent': UA } })
            if (fr.ok) {
              const fd = await fr.json()
              const fu = String(fd.shareUrl || '').replace(/^http:\/\//, 'https://')
              if (fu) alt = { u: fu, t: feature.title || feature.mediaComponentId }
            }
          }
        }
      } catch { /* fall through to null */ }
      out[code] = alt
      if (alt) fetched++; else missing++
    }
  } catch { missing++ }
  // The CHECKPOINT write is deliberately unguarded: this script resumes from the existing
  // file as a cache, so `out` starts full and only grows. It cannot shrink, which is the
  // thing the guard exists to catch. The FINAL write below is guarded like every other
  // snapshot.
  // CHECKPOINT-WRITE-OK
  if (fetched % 100 === 0 && fetched) {
    writeFileSync(OUT, JSON.stringify({ _note: 'Per-language JESUS Film deep links (shareUrl) from Arclight media-components. Partial write; rerun to resume.', generated: new Date().toISOString().slice(0, 10), codes: out }))
    console.log(`  ... ${i + 1}/${entries.length}  fetched=${fetched} cached=${cached} missing=${missing}`)
  }
  await new Promise((r) => setTimeout(r, DELAY))
}
writeSnapshot(OUT, JSON.stringify({
  _note: 'Per-language JESUS Film Project deep links (shareUrl) from Arclight media-components. A STRING is the JESUS Film (component 1_jf-0-0). An OBJECT {u,t} is a different JESUS Film Project feature in that language — the JESUS component has no entry for it — and t is the real title, which the surfaces must show instead of "JESUS Film". null means no feature exists and the film is dropped rather than linked to the retired ?languageId= URL, which nothing can verify.',
  generated: new Date().toISOString().slice(0, 10),
  stats: { languages: entries.length, withShareUrl: Object.keys(out).length, missing },
  codes: out,
}), (j) => Object.keys(j.codes || {}).length, 'Arclight shareUrl codes')
console.log(`DONE  languages ${entries.length}  shareUrl ${Object.keys(out).length}  missing ${missing}`)