bible/assets/sw.js

// The app's renderer, and its backend. Hand-written: this file is the source,
// served as-is apart from the three lines server.py stamps (see below).
//
// It reads as a small server, because that is what it is. Requests come in
// through a router (js/router.js) and are answered by handlers; the handlers
// render the nunjucks templates in templates/ (js/render.js) and read and
// write the bookmark in IndexedDB. htmx on the page talks to it exactly as it
// would to an origin.
//
//   GET  /              the book list
//   GET  /:page         one book -- /GEN, /EXO
//   GET  /api/bookmark  the navbar slot
//   POST /api/bookmark  set the bookmark, and return the slot
//   GET  /api/reference the navbar's chapter and verse picker
//
// Anything with a dot in its last segment is a file, not a route, and is
// served from the cache or the network without going near the router. Pages
// have no extension: what is on disk is bible/GEN.html, what the reader sees
// is /GEN, and server.py resolves one to the other. bible/index.html is the
// root itself -- there is no /index, the book list lives at /.
//
// The files on disk are fragments: a title element and one root element. The
// shell around them is templates/base.njk, applied per request, so changing the
// chrome means editing one template rather than regenerating every book. A
// handler returns the bare fragment to htmx and the composed document to a
// navigation -- one route, both callers, which is the whole reason the request
// object carries `navigate`.
//
// The composing is in js/render.js, which boot.html can load too. Normally it
// does not: boot.html registers this worker and reloads into it. It composes in
// the page when no worker is possible at all -- no secure context, or a browser
// without them -- and then it does so from these same templates rather than
// leaving a spinner up forever.
//
// Every same-origin GET is network-first: ask the network, fall back to the
// cache when it fails. Offline, fetch rejects immediately, so the fallback is
// effectively instant -- the only cost is one round trip while online, and you
// never see a stale page. Every response that arrives is written back to the
// cache, so a book that has been read once stays readable offline. Templates
// are the one exception, and readTemplate() says why.
//
// The whole corpus is kept offline, but not by making anyone wait for it and
// not by telling them about it. Install caches the shell alone -- a handful of
// small files -- and the app is usable the moment that is done. The books are
// then fetched a few at a time in the background while the reader reads, with
// nothing on screen to say so. Close the tab half way through and the next
// visit picks up where it left off: the cache is the record of what has been
// fetched, so there is no separate bookkeeping to lose.

// Imported scripts are stored with the registration, so these survive offline
// without being in PRECACHE. importScripts rather than the fetch-and-eval
// module loader the router came from: it is the one the platform supports in a
// worker, and it costs no waterfall.
importScripts('./js/nunjucks.min.js', './js/router.js', './js/render.js');

const NETWORK_TIMEOUT_MS = 3000;

// The background download gets a longer leash than a page the reader is
// waiting on: nothing is blocked on it, and giving up early on a slow link
// only means fetching the same book again later.
const DOWNLOAD_TIMEOUT_MS = 30000;

// Few enough to leave the connection room for the page being read, more than
// one so a slow response does not stall the queue. The old code asked for
// every file at once, which is what made a weak connection collapse.
const DOWNLOAD_AT_ONCE = 4;

// All three are stamped in by server.py from the files actually on disk, so an
// edit reaches installed devices with no build step: the worker's bytes
// change, the browser sees a new worker, and stale caches go on activate. The
// values below are the standalone fallback.
//
// The two versions are deliberately separate, over the two directories the
// server puts together. VERSION covers the hand-written chrome in assets/,
// templates included, so editing the shell drops and refetches a few hundred
// kilobytes. The other covers the generated pages in bible/, which change only
// when compile_bible.py runs -- so a change of chrome no longer costs every
// device the whole corpus again.
const VERSION = 'dev';
const LIBRARY_VERSION = 'dev';
const LIBRARY = [];

const SHELL_CACHE = 'bible-shell-' + VERSION;
const LIBRARY_CACHE = 'bible-library-' + LIBRARY_VERSION;

// The same two values, handed to the shell so a device can be asked what it is
// running. A phone that has kept an old worker is the hardest thing here to
// see from the outside, and these are what say so. See templates/version.njk.
const VERSIONS = { shell: VERSION, library: LIBRARY_VERSION };

// Resolved once, and the set is the test for which cache a response belongs
// in. Membership, not a path pattern: the split is whichever directory the
// file came from, which is a fact only the server has.
const LIBRARY_URLS = new Set(LIBRARY.map((path) => new URL(path, location.href).href));

// The shell and what it needs to paint. "./" -- the book list -- is in here as
// well as in the library: it is the one page the app falls back to, so it
// should not have to wait for the download to reach it.
//
// boot.html is here for the one case it can still be reached with a worker
// installed: a reload that bypasses this file. The three scripts in js/ that
// this file imports are not, because importScripts already keeps them.
const PRECACHE = [
  './boot.html',
  './',
  './manifest.webmanifest',
  './css/app.css',
  './css/bootstrap.min.css',
  './js/app.js',
  './js/htmx.min.js',
  './js/bootstrap.bundle.min.js'
// The templates come from render.js, which is the file that knows what one is
// -- and which loads exactly this list into nunjucks before it renders
// anything. Two lists would be one list and a way for the renderer to be
// asked for a template that was never cached.
].concat(TEMPLATE_URLS);

function cacheFor(req) {
  return caches.open(LIBRARY_URLS.has(req.url) ? LIBRARY_CACHE : SHELL_CACHE);
}

self.addEventListener('install', (event) => {
  event.waitUntil((async () => {
    const cache = await caches.open(SHELL_CACHE);
    // Added one at a time rather than addAll(): addAll is atomic, so a single
    // failed asset would leave the app with no offline copy at all.
    let failed = 0;
    await Promise.all(PRECACHE.map((url) =>
      cache.add(new Request(url, { cache: 'reload' })).catch(() => { failed++; })));
    if (failed) console.warn('precache: ' + failed + ' of ' + PRECACHE.length + ' failed');
    await self.skipWaiting();
  })());
});

self.addEventListener('activate', (event) => {
  event.waitUntil((async () => {
    const keys = await caches.keys();
    await Promise.all(keys
      .filter((k) => k.startsWith('bible-') && k !== SHELL_CACHE && k !== LIBRARY_CACHE)
      .map((k) => caches.delete(k)));
    await self.clients.claim();
  })());
});

// A dot in the last segment means a file that exists on disk: a script, a
// stylesheet, an icon, a template. Page URLs never have one, so this is the
// whole boundary between what the router owns and what is simply served.
function isFile(url) {
  return url.pathname.split('/').pop().indexOf('.') !== -1;
}

self.addEventListener('fetch', (event) => {
  const req = event.request;
  const url = new URL(req.url);
  if (url.origin !== location.origin) return;

  // The background download rides along on whatever the page is already
  // asking for. It gets no event of its own: activate cannot be used, because
  // waitUntil there holds back activation and every fetch with it, and a
  // worker is only kept alive while an event of its own is pending. So each
  // request the reader makes is what carries the download forward.
  keepFilling(event);

  if (!isFile(url)) {
    const routed = router.handle(event, url);
    // A route that declines -- by returning nothing -- leaves the URL to be
    // served as a file, the same as one that never matched.
    if (routed) {
      return event.respondWith(routed.then((res) => res || answer(req, event)));
    }
  }

  if (req.method !== 'GET') return;
  event.respondWith(answer(req, event));
});

// ---------------------------------------------------------------- fetching

// Aborted rather than raced, so a hung request is actually torn down instead
// of leaking while the cache answers.
async function fromNetwork(req, timeout) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeout || NETWORK_TIMEOUT_MS);
  try {
    return await fetch(req, { signal: controller.signal });
  } finally {
    clearTimeout(timer);
  }
}

function cached(req) {
  return caches.match(req, { ignoreSearch: true });
}

// A string argument becomes a fresh Request, which matters beyond
// convenience: it is not a navigation, so the origin server can tell an
// internal fetch from a browser one and hand back the fragment rather than
// the boot page.
async function answer(input, event) {
  const req = typeof input === 'string' ? new Request(input) : input;

  try {
    const res = await fromNetwork(req);
    if (res && res.ok && res.type === 'basic') {
      const copy = res.clone();
      // Not awaited: the response should not wait on the cache write.
      const write = cacheFor(req).then((c) => c.put(req, copy));
      if (event) event.waitUntil(write);
      else write.catch(() => {});
    }
    return res;
  } catch (err) {
    const hit = await cached(req);
    if (hit) return hit;
    throw err;
  }
}

// Pages are always asked for with revalidation forced.
//
// server.py marks them no-cache, which asks the browser for the same thing --
// but this worker's own fetch goes through the browser's HTTP cache too, so a
// page stored under an earlier and wronger Cache-Control keeps being served
// from it until that entry expires, however many times the library version
// changes underneath. Saying it on the request is what gets past one that
// already is. Costs a conditional GET, which the response header was asking
// for anyway.
function pageRequest(url) {
  return new Request(url, { cache: 'no-cache' });
}

// Used for verses.json as well, which is generated beside the pages and
// versions with them, and would go stale the same way for the same reason.

async function readText(input, event) {
  const res = await answer(input, event);
  if (!res || !res.ok) throw new Error('cannot load ' + (input.url || input));
  return res.text();
}

// --------------------------------------------------------------- templates

// Templates are read cache-first, alone among everything here. They are
// precached on install, and editing one changes VERSION, which installs a new
// worker with a new cache -- so a hit cannot be stale. Going to the network
// for them would put six round trips in front of the first page every time the
// browser has reaped this worker, which it does constantly.
async function readTemplate(path) {
  const hit = await cached(path);
  return hit ? hit.text() : readText(path);
}

const view = createRenderer(readTemplate);

// ------------------------------------------------------------------ routes

// Patterns are relative to the worker's scope, so this is "/" and "/GEN" even
// if the app is ever served from a subdirectory.
const router = new Router(new URL('./', location.href).pathname);

// ---------------------------------------------------------------- reference

// bible/verses.json: every book's chapters, and every chapter's verse labels.
// compile_bible.py writes it from the same tree it renders the books from, so
// every option built here names an anchor that is in the page it lands on.
//
// A chapter whose verses run 1..n is stored as the number n; only the eight in
// the corpus with a gap or a range -- Sirach 26 ends "19-27", 28, 29 -- spell
// their labels out. 6 KB for 38,000 verses rather than 180.
const VERSES_URL = './verses.json';

let verses = null;

// Held for the life of this worker, and only for that. Regenerating the corpus
// changes LIBRARY_VERSION, which is a new worker with a new cache, so a copy
// held here cannot outlive the pages it describes.
//
// It lives in the library cache, so a device that has never been online long
// enough to fetch it has no picker rather than no page -- see referenceOf.
function readVerses() {
  if (!verses) {
    verses = readText(pageRequest(VERSES_URL))
      .then(JSON.parse)
      .catch((err) => { verses = null; throw err; });
  }
  return verses;
}

// "ch3" or "v3_16" -- the id of the element an option lands on, which is also
// where the chapter it belongs to is written down. Options are built here, so
// this is only ever reading back what this file sent; a URL is still a URL,
// and anything else is null.
function parseAnchor(id) {
  const chapter = /^ch(\d+)$/.exec(id || '');
  if (chapter) return { chapter: +chapter[1], verse: null };
  const verse = /^v(\d+)_[0-9-]+$/.exec(id || '');
  if (verse) return { chapter: +verse[1], verse: id };
  return null;
}

function verseLabels(chapters, number) {
  const entry = chapters[number - 1];
  if (typeof entry !== 'number') return entry || [];
  const out = [];
  for (let i = 1; i <= entry; i++) out.push(String(i));
  return out;
}

// The picker's model for one book, or null if that is not a book with verses
// in it -- the book list, a notice, and the two front-matter books that have
// no verse anchors to offer.
//
// anchor says which chapter to show the verses of, and which verse to mark
// selected. Anything this cannot read -- nothing at all, an empty select, a
// footnote id, a chapter past the end of the book -- is chapter 1, and chapter
// 1 is filled in as if it had been picked. There is no state where a chapter
// is only sort of chosen: a select showing an option is a select that chose
// it, and the verses beside it have to be that chapter's.
//
// Every book that gets a picker has verses in its first chapter -- the two
// that do not, the front matter and the glossary, are turned away above -- so
// falling back to 1 always fills the verse list rather than emptying it.
//
// The verse is left unselected when the anchor names only a chapter. That is
// not the same as unfilled: the options are chapter 1's, and the select shows
// the first of them. It says where you are going, not where you are.
function referenceFor(map, code, anchor) {
  const chapters = map && Object.prototype.hasOwnProperty.call(map, code)
    ? map[code] : null;
  if (!chapters || !chapters.length) return null;

  // The front matter and the glossary are books with chapters and no verses.
  // A picker over them would be a chapter select beside an empty one, so they
  // get none, the same as the book list.
  if (!chapters.some((entry) => (typeof entry === 'number' ? entry : entry.length))) {
    return null;
  }

  const at = parseAnchor(anchor) || { chapter: 1, verse: null };
  const number = at.chapter >= 1 && at.chapter <= chapters.length ? at.chapter : 1;
  // Only when the anchor got as far as naming this chapter. A fallback to 1
  // has not chosen a verse, it has just run out of anchor.
  const verse = at.chapter === number ? at.verse : null;

  return {
    book: code,
    chapter: 'ch' + number,
    chapters: chapters.map((_, i) => ({ id: 'ch' + (i + 1), label: String(i + 1) })),
    verse: verse,
    verses: verseLabels(chapters, number)
      .map((label) => ({ id: 'v' + number + '_' + label, label: label }))
  };
}

// The book a page URL is showing: "GEN" for /GEN, "" for the book list.
function bookOf(url) {
  return url ? router.relative(url).slice(1) : '';
}

// Never throws. The map is fetched over the network first and answered from
// the library cache when that fails, so the one case this swallows is a device
// that has gone offline before the background download reached verses.json:
// the pages it has read still read, and the picker is hidden until a later
// navigation finds the map. The same degradation the bookmark makes.
async function referenceOf(code, anchor) {
  if (!code) return null;
  try {
    return referenceFor(await readVerses(), code, anchor);
  } catch (err) {
    return null;
  }
}

const OFFLINE_NOTICE = ['Not available offline',
  'This page has not been opened on this device, so there is no copy stored ' +
  'here. Reconnect and try again, or open one you have read before.'];
const MISSING_NOTICE = ['Not found', 'There is no page at this address.'];

// The fragment a page URL names, or one explaining why there is none. A page
// asks the origin for its own address -- /GEN fetches ./GEN -- and server.py
// is what knows the file behind it is bible/GEN.html.
async function fragmentFor(path, event) {
  let res;
  try {
    res = await answer(pageRequest(path), event);
  } catch (err) {
    return view.notice(...OFFLINE_NOTICE);
  }
  return res.ok ? res.text() : view.notice(...MISSING_NOTICE);
}

// One handler for every caller. htmx swapping a link into a shell already on
// screen wants the bare fragment; a navigation has no shell yet, and a history
// restore is rebuilding one, so both get the fragment composed into a document.
//
// code is the book the path names, or "" for the book list -- the one thing
// the picker needs and the path already knows.
async function servePage(path, code, call) {
  const fragment = await fragmentFor(path, call.event);
  const reference = await referenceOf(code, null);
  if (!call.navigate && !call.restore) {
    // The picker is in the navbar and every boosted link targets #content, so
    // a swap never reaches it. It rides along out of band instead, the way a
    // bookmark write carries the modal's status line: one response, both
    // halves of the page. Sent even when it is null -- that is the hidden
    // state, and arriving at the book list has to put the picker away.
    return fragment + '\n'
      + await view.render('reference.njk', { reference: reference, oob: true });
  }
  try {
    return await view.compose(fragment, { mark: await getBookmark(),
                                          reference: reference,
                                          version: VERSIONS });
  } catch (err) {
    // A template that will not render is a bug, not a condition to design for,
    // but it should not take the text down with it: the bare fragment has no
    // chrome and no styling, and it is still the page that was asked for.
    console.error('compose failed: ' + err.message);
    return fragment;
  }
}

// "/" before "/:page", because both are two segments and the first match wins.
router.get('/', (call) => servePage('./', '', call));
router.get('/:page', (call) =>
  servePage('./' + call.params.page, call.params.page, call));

router.get('/api/bookmark', () => bookmarkFragment(''));

// The picker, and the two moves it makes.
//
//   no parameters   the whole picker, standing where the anchor in the address
//                   bar puts it. Asked for by the picker itself: on load,
//                   after a history restore -- which swaps the history element
//                   straight in and processes no out-of-band elements -- and
//                   when a worker first appears under a page boot.html
//                   composed.
//   at=ch3          the verses of that chapter, and land on the chapter.
//   at=v3_16        the same verses, that one selected, and land on the verse.
//
// One parameter for both selects: at is the id of the element to land on, and
// ch3 and v3_16 are told apart by their shape. Which select it came from is
// not a second fact.
//
// Neither the book nor the anchor is in the URL, deliberately. Both come from
// HX-Current-URL, which htmx sets from location.href on every request it
// makes: the whole address, fragment included.
//
// For the book, because the picker sits in the navbar and outlives the page
// under it -- an address baked into its attributes is the one thing here that
// can go stale. For the anchor, because this is the only way a fragment
// reaches this file at all. A browser opening /GEN#v3_16 asks for /GEN and
// keeps the rest to itself, so the shell is composed without it and the picker
// is rendered at 1:1 like any other. The page then asks this route for itself,
// and that request is the one carrying the anchor.
//
// Which also puts the picker where it belongs after a back or forward, and
// under a page boot.html composed once a worker turns up, for nothing extra:
// those requests carry the address bar too.
//
// Two response headers carry what a change means beyond the options:
//
//   settle:0     swap the options now. The default waits 20ms, and this
//                answer is one the reader is waiting on.
//   HX-Replace-Url  put the anchor in the address bar without stacking a
//                history entry per option spun past.
//
// The scroll used to be a third, show:#id:top, and is app.js's again. Not
// because the header was wrong but because the moment was: on iOS the change
// event arrives while the native picker is still drawn over the page, and
// dismissing it restores the scroll offset it opened at -- so a scroll made
// inside the swap is undone before it is seen. The page has to be the one
// doing it, because only the page is still there afterwards to try again.
// See landOnReference() in js/app.js. The address bar above is how it knows
// where to go: htmx updates history before it swaps.
router.get('/api/reference', async (call) => {
  const params = call.url.searchParams;
  const page = call.currentUrl();
  // Set when a select was moved -- by whether the parameter is there, not by
  // whether it says anything. A select posts an empty value when it has no
  // options, and reading that as "no select moved" answers it with a whole
  // picker: htmx swaps a <div> into a <select>, which renders as nothing, and
  // the select is then empty for good -- every change after that posts the
  // same empty value and gets the same answer.
  //
  // An empty value is still an answerable question. It names no chapter, so it
  // gets chapter 1, which is what everything here does with an anchor it
  // cannot read.
  const moved = params.has('at') ? params.get('at') : null;

  // The address bar is the fallback, and only that: once a select has been
  // used it is the one saying where to go, and HX-Replace-Url has not caught
  // up yet.
  const reference = await referenceOf(
    bookOf(page), moved === null ? (page && page.hash.slice(1)) : moved);

  // settled, because this picker is already standing where the anchor put it.
  // The one that came with the page is not, and carries a load trigger to ask
  // -- which this answer must not carry as well, or it would ask forever.
  if (moved === null) {
    return view.render('reference.njk', { reference: reference, settled: true });
  }
  // The options were rendered from this map, so failing to parse them back
  // means the map moved under an open page: leave the picker alone rather
  // than swapping in options for a book that is no longer on screen.
  if (!reference) return new Response('', { status: 204 });

  const target = reference.verse || reference.chapter;
  return new Response(await view.render('verses.njk', { reference: reference }), {
    headers: {
      'Content-Type': 'text/html; charset=utf-8',
      'Cache-Control': 'no-store',
      'HX-Reswap': 'innerHTML settle:0',
      'HX-Replace-Url': page.pathname + page.search + '#' + target
    }
  });
});

router.post('/api/bookmark', async (call) => {
  try {
    const form = await call.form();
    const ref = String(form.get('ref') || '').trim();
    const href = String(form.get('href') || '').trim();
    if (!ref || !href) return new Response('', { status: 400 });
    await withStore('readwrite',
      (s) => s.put({ ref: ref, href: href, savedAt: Date.now() }, BOOKMARK_KEY));
    return bookmarkFragment('Bookmark set.');
  } catch (err) {
    return bookmarkFragment('Could not save: storage unavailable.');
  }
});

// ----------------------------------------------------------------- library

// The background download of the whole corpus.
//
// There is no progress to keep: the cache is the record. Each run asks it what
// is already there, which is what makes this survive the worker being shut
// down mid-run, resume on the next visit, and stay honest when a page arrives
// some other way -- reading a book puts it in the same cache.
let filling = null;      // the run in flight, so two do not start at once
let complete = false;    // a run got through the whole list; stop looking

function keepFilling(event) {
  if (complete) return;
  // navigator.onLine is only ever trustworthy when it says false, which is the
  // direction that matters: it stops a doomed run being started, and retried,
  // on every request while the device is plainly offline.
  if (!filling && self.navigator.onLine !== false) {
    filling = fillLibrary().catch(() => {}).finally(() => { filling = null; });
  }
  if (filling) event.waitUntil(filling);
}

async function fillLibrary() {
  const cache = await caches.open(LIBRARY_CACHE);
  const have = new Set((await cache.keys()).map((req) => req.url));
  const missing = [];
  LIBRARY_URLS.forEach((url) => { if (!have.has(url)) missing.push(url); });
  if (!missing.length) { complete = true; return; }

  let stopped = false;
  const runners = [];
  for (let i = 0; i < DOWNLOAD_AT_ONCE && i < missing.length; i++) runners.push(run());
  await Promise.all(runners);
  // Only after a run that was never cut short: anything else leaves pages
  // outstanding, and the next request should look again.
  if (!stopped) complete = true;

  async function run() {
    while (missing.length && !stopped) {
      const url = missing.pop();
      try {
        const res = await fromNetwork(pageRequest(url), DOWNLOAD_TIMEOUT_MS);
        // A response that arrived but is not usable -- a page that has since
        // been removed, say -- is skipped rather than retried. The connection
        // is plainly fine, and stopping for it would strand everything after.
        if (res.ok && res.type === 'basic') await cache.put(url, res);
      } catch (err) {
        // Stop the whole run rather than grinding through a hundred certain
        // failures. The next request starts a fresh one, so reconnecting is
        // all it takes to carry on.
        stopped = true;
      }
    }
  }
}

// ---------------------------------------------------------------- bookmark

const DB_NAME = 'bible';
const DB_STORE = 'kv';
const BOOKMARK_KEY = 'bookmark';

// There is no way to clear a bookmark, so there is no empty state: a reader
// who has never set one is treated as being at the beginning. Everything
// downstream can assume a mark -- bookmark.njk's empty branch is for the boot
// page, which has no worker to ask, not for a reader without one.
const DEFAULT_BOOKMARK = { ref: 'Genesis 1:1', href: 'GEN#v1_1' };

function openDb() {
  return new Promise((resolve, reject) => {
    const req = indexedDB.open(DB_NAME, 1);
    req.onupgradeneeded = () => req.result.createObjectStore(DB_STORE);
    req.onsuccess = () => resolve(req.result);
    req.onerror = () => reject(req.error);
  });
}

// Resolved on the transaction rather than the request, so the value is only
// reported once the write it belongs to has actually committed.
async function withStore(mode, fn) {
  const db = await openDb();
  try {
    return await new Promise((resolve, reject) => {
      const tx = db.transaction(DB_STORE, mode);
      const req = fn(tx.objectStore(DB_STORE));
      tx.oncomplete = () => resolve(req ? req.result : undefined);
      tx.onabort = tx.onerror = () => reject(tx.error);
    });
  } finally {
    db.close();
  }
}

// Storage being unavailable degrades to the default rather than a broken page,
// which is the same fallback as never having set one.
function getBookmark() {
  return withStore('readonly', (s) => s.get(BOOKMARK_KEY))
    .catch(() => null)
    .then((mark) => mark || DEFAULT_BOOKMARK);
}

async function bookmarkFragment(status) {
  let html = await view.render('bookmark.njk', { mark: await getBookmark() });
  if (status) {
    // Out of band, so one response updates both the navbar slot and the
    // modal's status line without the modal having to ask separately. Same
    // template the shell uses, so the two cannot drift apart.
    html += '\n' + await view.render('status.njk', { text: status, oob: true });
  }
  return html;
}