WIP

render_books.py

@@ -0,0 +1,587 @@
+#!/usr/bin/env python3
+"""Render bible.json into one semantic HTML page per book.
+
+Reads the JSON produced by extract_bible.py and writes books/GEN.html and so
+on, plus an index. The markup carries structure and no presentation: every
+USFM distinction survives as a class or data attribute, so the whole corpus
+can be restyled from bible.css without regenerating anything.
+
+Grouping worth knowing about: consecutive poetry lines are wrapped in a
+<div class="poetry"> that a blank-line block closes, so stanzas are real
+elements rather than spacing; consecutive list items become one <ul>.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import html
+import json
+import os
+import shutil
+import struct
+import sys
+import zlib
+
+# Inline node type -> (tag, class). ``cite`` and ``dfn`` carry real meaning
+# here: a book name cited in a note, and a glossary term at its definition.
+INLINE_TAGS = {
+    "wj": ("span", "wj"),          # words of Jesus
+    "keyword": ("dfn", "k"),       # glossary headword
+    "bookref": ("cite", "bk"),     # book title cited in a note
+    "fq": ("span", "fq"),          # the words a footnote is about
+    "fqa": ("span", "fqa"),        # an alternative rendering
+    "fl": ("span", "fl"),          # footnote label
+    "fine": ("span", "fine"),
+    "xt": ("span", "xt"),          # cross-reference target
+}
+
+# Block type -> tag. Headings sit at h3 so the outline never skips a level:
+# h1 book, h2 chapter, h3 section.
+BLOCK_TAGS = {
+    "paragraph": "p",
+    "line": "p",
+    "heading": "h3",
+    "title": "p",
+    "speaker": "p",
+    "item": "li",
+}
+
+
+APP_NAME = "World English Bible"
+APP_SHORT = "WEB Bible"
+THEME = "#1a202c"
+
+# iOS only honours the apple-prefixed meta tags; the unprefixed ones are the
+# standard. Both are emitted so the app installs correctly on either.
+HEAD = """<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
+<title>%(title)s</title>
+<link rel="stylesheet" href="bible.css">
+<script src="app.js" defer></script>
+<link rel="manifest" href="manifest.webmanifest">
+<meta name="theme-color" content="%(theme)s">
+<link rel="apple-touch-icon" href="icons/apple-touch-icon.png">
+<meta name="mobile-web-app-capable" content="yes">
+<meta name="apple-mobile-web-app-capable" content="yes">
+<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
+<meta name="apple-mobile-web-app-title" content="%(short)s">
+</head>
+<body>
+"""
+
+# Registered on load so it never competes with first paint.
+SW_SCRIPT = """<script>
+if ('serviceWorker' in navigator) {
+  addEventListener('load', function () {
+    navigator.serviceWorker.register('sw.js').catch(function (e) {
+      console.warn('service worker registration failed:', e);
+    });
+  });
+}
+</script>
+"""
+
+
+def head_html(title):
+    return HEAD % {"title": esc(title), "theme": THEME, "short": esc(APP_SHORT)}
+
+
+def esc(text):
+    return html.escape(text, quote=False)
+
+
+def verse_id(chapter, label):
+    """Page-unique verse anchor, e.g. v23_1 (or v11_15-16 for a range)."""
+    return "v%d_%s" % (chapter, label)
+
+
+def ref_href(node, book, chapter):
+    """Link an intra-corpus reference; whole books are single pages."""
+    target_book = node.get("book") or book
+    target_ch = node.get("chapter", chapter)
+    verse = node.get("verse")
+    base = "" if target_book == book else "%s.html" % target_book
+    if verse:
+        return "%s#%s" % (base, verse_id(target_ch, verse))
+    return "%s#ch%d" % (base, target_ch)
+
+
+def render_inline(nodes, book, chapter, out):
+    for node in nodes:
+        kind = node["t"]
+        if kind == "text":
+            out.append(esc(node["v"]))
+        elif kind == "verse":
+            # Trailing space keeps the number off the text before any CSS
+            # lands; style .v with padding and it stays correct either way.
+            out.append('<span class="v" id="%s">%s</span> ' % (
+                verse_id(chapter, node["label"]), esc(node["label"])))
+        elif kind == "noteref":
+            n = node["id"].replace("FN", "")
+            out.append('<a class="noteref" id="fnref%s_%s" href="#fn%s_%s">'
+                       "<sup>%s</sup></a>" % (chapter, n, chapter, n,
+                                              esc(node.get("marker") or "*")))
+        elif kind == "br":
+            out.append("<br>")
+        elif kind == "ref":
+            out.append('<a class="ref" href="%s">' % ref_href(node, book, chapter))
+            render_inline(node.get("c", []), book, chapter, out)
+            out.append("</a>")
+        elif kind in INLINE_TAGS:
+            tag, cls = INLINE_TAGS[kind]
+            out.append('<%s class="%s">' % (tag, cls))
+            render_inline(node.get("c", []), book, chapter, out)
+            out.append("</%s>" % tag)
+        else:  # unknown node type: keep the words, drop the wrapper
+            render_inline(node.get("c", []), book, chapter, out)
+
+
+def render_block(block, book, chapter, out):
+    btype = block["type"]
+    tag = BLOCK_TAGS.get(btype, "p")
+    classes = [block["style"]]
+    attrs = ""
+    if "indent" in block:
+        attrs += ' data-indent="%d"' % block["indent"]
+    if btype == "line":
+        classes.append("line")
+    out.append("      <%s class=\"%s\"%s>" % (tag, " ".join(classes), attrs))
+    render_inline(block.get("content", []), book, chapter, out)
+    out.append("</%s>\n" % tag)
+
+
+def render_notes(notes, chapter, book, out):
+    if not notes:
+        return
+    out.append('      <section class="footnotes">\n')
+    out.append('        <h4 class="footnotes-title">Notes</h4>\n')
+    out.append("        <ol>\n")
+    for note in notes:
+        n = (note["id"] or "").replace("FN", "")
+        out.append('          <li class="note" id="fn%s_%s" data-kind="%s">'
+                   % (chapter, n, note["kind"]))
+        out.append('<span class="marker">%s</span> ' % esc(note.get("marker") or "*"))
+        if note.get("ref"):
+            out.append('<a class="backref" href="#fnref%s_%s">%s</a> '
+                       % (chapter, n, esc(note["ref"])))
+        out.append('<span class="ft">')
+        render_inline(note["content"], book, chapter, out)
+        out.append("</span></li>\n")
+    out.append("        </ol>\n      </section>\n")
+
+
+def render_chapter(chapter, book, out):
+    num = chapter["number"]
+    out.append('    <section class="chapter" id="ch%d" data-chapter="%d">\n'
+               % (num, num))
+    out.append('      <h2 class="chapter-number">%d</h2>\n' % num)
+
+    group = None  # "poetry" | "list" | None
+
+    def close_group():
+        nonlocal group
+        if group == "poetry":
+            out.append("      </div>\n")
+        elif group == "list":
+            out.append("      </ul>\n")
+        group = None
+
+    for block in chapter["blocks"]:
+        btype = block["type"]
+        if btype == "break":
+            close_group()          # a blank line ends the stanza
+            continue
+        want = {"line": "poetry", "item": "list"}.get(btype)
+        if want != group:
+            close_group()
+            if want == "poetry":
+                out.append('      <div class="poetry">\n')
+            elif want == "list":
+                out.append('      <ul class="list">\n')
+            group = want
+        render_block(block, book, num, out)
+    close_group()
+
+    render_notes(chapter["notes"], num, book, out)
+    out.append("    </section>\n")
+
+
+def render_book(book, translation, out_dir):
+    code, name = book["code"], book["name"]
+    out = []
+    title = "%s — %s" % (name, translation["name"])
+    out.append(head_html(title))
+    out.append('<article class="book" id="%s" data-book="%s" data-name="%s" '
+               'data-section="%s">\n' % (code, code, esc(name), book["section"]))
+
+    # Header: the full multi-line title if the source had one.
+    out.append('  <header class="book-header">\n')
+    out.append("    <h1>")
+    if book["titles"]:
+        for part in book["titles"]:
+            out.append('<span class="%s">%s</span>' % (part["style"], esc(part["text"])))
+    else:
+        out.append(esc(name))
+    out.append("</h1>\n")
+    out.append('    <nav class="chapter-nav" aria-label="Chapters">\n      <ul>\n')
+    for chapter in book["chapters"]:
+        out.append('        <li><a href="#ch%d">%d</a></li>\n'
+                   % (chapter["number"], chapter["number"]))
+    out.append("      </ul>\n    </nav>\n")
+    out.append('    <p class="home"><a href="index.html">All books</a></p>\n')
+    out.append("  </header>\n")
+
+    out.append('  <div class="chapters">\n')
+    for chapter in book["chapters"]:
+        render_chapter(chapter, code, out)
+    out.append("  </div>\n")
+
+    out.append('  <footer class="colophon"><p>%s. %s.</p></footer>\n'
+               % (esc(translation["name"]), esc(translation["license"])))
+    out.append("</article>\n" + SW_SCRIPT + "</body>\n</html>\n")
+
+    path = os.path.join(out_dir, "%s.html" % code)
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write("".join(out))
+    return path
+
+
+SECTION_LABELS = [
+    ("ot", "Old Testament"),
+    ("nt", "New Testament"),
+    ("deuterocanon", "Deuterocanon"),
+    ("other", "Other"),
+]
+
+
+def render_index(books, translation, out_dir):
+    out = [head_html(translation["name"]),
+           '<main class="index">\n  <h1>%s</h1>\n' % esc(translation["name"]),
+           '  <p class="bookmark" id="bookmark" hidden></p>\n']
+    for key, label in SECTION_LABELS:
+        group = [b for b in books if b["section"] == key]
+        if not group:
+            continue
+        out.append('  <section class="book-group" data-section="%s">\n' % key)
+        out.append("    <h2>%s</h2>\n    <ul>\n" % esc(label))
+        for book in group:
+            out.append('      <li><a href="%s.html">%s</a> '
+                       '<span class="count">%d</span></li>\n'
+                       % (book["code"], esc(book["name"]), len(book["chapters"])))
+        out.append("    </ul>\n  </section>\n")
+    out.append('  <footer class="colophon"><p>%s. %s.</p></footer>\n'
+               % (esc(translation["name"]), esc(translation["license"])))
+    out.append("</main>\n" + SW_SCRIPT + "</body>\n</html>\n")
+    path = os.path.join(out_dir, "index.html")
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write("".join(out))
+    return path
+
+
+# Emitted classes, listed so the stylesheet has a checklist to work from.
+CSS_STUB = """/* Stylesheet for the rendered books. Everything below is a class the
+   renderer emits; none of it is styled yet.
+
+   Structure   book, book-header, chapter-nav, home, chapters, chapter,
+               chapter-number, poetry, list, footnotes, colophon, index,
+               book-group, count
+   Paragraphs  p m nb pi mi pc ip          (.line marks a poetry line;
+   Poetry      q q2 q3 qs                   data-indent="1|2|3" gives depth)
+   Headings    s ms is                     (rendered as <h3>)
+   Other       d (psalm/acrostic title), sp (speaker), li ili (list items),
+               mt mt2 mt3 (book title parts)
+   Inline      v (verse number), wj (words of Jesus), k (<dfn>), bk (<cite>),
+               ref, noteref
+   Notes       note, marker, backref, ft, fq, fqa, fl, fine, xt
+               (li.note carries data-kind="footnote|xref")
+   Bookmark    bookmark, bookmark-label, bookmark-link (index page),
+               verse-dialog, verse-dialog-ref, verse-dialog-note,
+               verse-dialog-actions, verse-dialog-status, verse-dialog-url
+*/
+
+/* Affordance, not design: verse numbers open the bookmark dialog, so they
+   need to look clickable. Everything else is left to you. */
+.v { cursor: pointer; }
+"""
+
+
+# --------------------------------------------------------------------------
+# Progressive web app assets
+# --------------------------------------------------------------------------
+# The site has to install to an iPhone home screen and open with no network,
+# so everything it needs is precached by a service worker at install time.
+
+def write_png(path, size, rows):
+    """Minimal RGB PNG writer, so the toolchain stays dependency-free."""
+    raw = b"".join(b"\x00" + bytes(row) for row in rows)
+    def chunk(kind, data):
+        return (struct.pack(">I", len(data)) + kind + data
+                + struct.pack(">I", zlib.crc32(kind + data) & 0xffffffff))
+    png = b"\x89PNG\r\n\x1a\n"
+    png += chunk(b"IHDR", struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0))
+    png += chunk(b"IDAT", zlib.compress(raw, 9))
+    png += chunk(b"IEND", b"")
+    with open(path, "wb") as fh:
+        fh.write(png)
+
+
+def _in_rrect(x, y, x0, y0, x1, y1, r):
+    """Point-in-rounded-rectangle: clamp to the inner rect, measure back."""
+    cx = min(max(x, x0 + r), x1 - r)
+    cy = min(max(y, y0 + r), y1 - r)
+    return (x - cx) ** 2 + (y - cy) ** 2 <= r * r
+
+
+BG = (26, 32, 44)        # slate, matches THEME
+PAGE = (247, 250, 252)
+SPINE = (99, 179, 237)
+INK = (160, 174, 192)
+
+
+def icon_rows(size):
+    """An open page with a coloured spine. Content sits inside the maskable
+    safe zone, so one drawing serves both `any` and `maskable` purposes."""
+    s = float(size)
+    px0, py0, px1, py1 = 0.24 * s, 0.20 * s, 0.76 * s, 0.80 * s
+    radius = 0.045 * s
+    spine_x1 = px0 + 0.075 * s
+    lines = [(0.30, 0.375), (0.435, 0.51), (0.57, 0.645), (0.705, 0.74)]
+    ss = 2                    # 2x supersampling to soften the curves
+    rows = []
+    for py in range(size):
+        row = bytearray()
+        for pxi in range(size):
+            r = g = b = 0
+            for oy in range(ss):
+                for ox in range(ss):
+                    x = pxi + (ox + 0.5) / ss
+                    y = py + (oy + 0.5) / ss
+                    col = BG
+                    if _in_rrect(x, y, px0, py0, px1, py1, radius):
+                        col = PAGE
+                        if x <= spine_x1:
+                            col = SPINE
+                        else:
+                            for top, bot in lines:
+                                if top * s <= y <= bot * s and x <= 0.68 * s:
+                                    col = INK
+                                    break
+                    r += col[0]; g += col[1]; b += col[2]
+            n = ss * ss
+            row += bytes((r // n, g // n, b // n))
+        rows.append(row)
+    return rows
+
+
+def copy_assets(out_dir, src="assets"):
+    """Hand-written static files (app.js) live in assets/ and are copied in,
+    so they stay editable as source rather than as Python strings."""
+    if not os.path.isdir(src):
+        return
+    for name in sorted(os.listdir(src)):
+        path = os.path.join(src, name)
+        if os.path.isfile(path):
+            shutil.copyfile(path, os.path.join(out_dir, name))
+
+
+def write_icons(out_dir):
+    icon_dir = os.path.join(out_dir, "icons")
+    os.makedirs(icon_dir, exist_ok=True)
+    for name, size in (("icon-192.png", 192), ("icon-512.png", 512),
+                       ("apple-touch-icon.png", 180)):
+        write_png(os.path.join(icon_dir, name), size, icon_rows(size))
+
+
+def write_manifest(out_dir, translation):
+    manifest = {
+        "name": APP_NAME,
+        "short_name": APP_SHORT,
+        "description": "%s. %s." % (translation["name"], translation["license"]),
+        "start_url": "./index.html",
+        "scope": "./",
+        "display": "standalone",
+        "orientation": "portrait",
+        "background_color": THEME,
+        "theme_color": THEME,
+        "icons": [
+            {"src": "icons/icon-192.png", "sizes": "192x192",
+             "type": "image/png", "purpose": "any maskable"},
+            {"src": "icons/icon-512.png", "sizes": "512x512",
+             "type": "image/png", "purpose": "any maskable"},
+        ],
+    }
+    path = os.path.join(out_dir, "manifest.webmanifest")
+    with open(path, "w", encoding="utf-8") as fh:
+        json.dump(manifest, fh, indent=2)
+
+
+SW_TEMPLATE = """// Generated by render_books.py -- do not edit by hand.
+//
+// STRATEGY decides what a same-origin GET does:
+//
+//   network-first  ask the network, fall back to the cache when it fails.
+//                  Offline, fetch rejects immediately, so the fallback is
+//                  effectively instant -- the only cost is one round trip
+//                  while online, and you never see a stale page.
+//   cache-first    answer from the cache the moment there is a hit. Fastest
+//                  possible load, but a rebuild needs two reloads to appear.
+//
+// Either way every response that arrives is written back to the cache, so the
+// offline copy stays current, and a new build changes VERSION, dropping the
+// caches belonging to older builds.
+const STRATEGY = '%(strategy)s';
+const NETWORK_TIMEOUT_MS = 3000;
+const VERSION = '%(version)s';
+const CACHE = 'bible-' + VERSION;
+const ASSETS = %(assets)s;
+
+self.addEventListener('install', (event) => {
+  event.waitUntil((async () => {
+    const cache = await caches.open(CACHE);
+    // Added one at a time rather than addAll(): addAll is atomic, so a single
+    // failed asset would leave the app with no offline copy at all.
+    let failed = 0;
+    await Promise.all(ASSETS.map((url) =>
+      cache.add(new Request(url, { cache: 'reload' })).catch(() => { failed++; })));
+    if (failed) console.warn('precache: ' + failed + ' of ' + ASSETS.length + ' failed');
+    await self.skipWaiting();
+  })());
+});
+
+self.addEventListener('activate', (event) => {
+  event.waitUntil((async () => {
+    const keys = await caches.keys();
+    await Promise.all(keys
+      .filter((k) => k.startsWith('bible-') && k !== CACHE)
+      .map((k) => caches.delete(k)));
+    await self.clients.claim();
+  })());
+});
+
+self.addEventListener('fetch', (event) => {
+  const req = event.request;
+  if (req.method !== 'GET') return;
+  if (new URL(req.url).origin !== location.origin) return;
+  event.respondWith(respond(event));
+});
+
+// Aborted rather than raced, so a hung request is actually torn down instead
+// of leaking while the cache answers.
+async function fromNetwork(req) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), NETWORK_TIMEOUT_MS);
+  try {
+    return await fetch(req, { signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function cached(req) {
+  return caches.match(req, { ignoreSearch: true });
+}
+
+async function respond(event) {
+  const req = event.request;
+
+  if (STRATEGY === 'cache-first') {
+    const hit = await cached(req);
+    if (hit) return hit;
+  }
+
+  try {
+    const res = await fromNetwork(req);
+    if (res && res.ok && res.type === 'basic') {
+      const copy = res.clone();
+      // Not awaited: the response should not wait on the cache write.
+      event.waitUntil(caches.open(CACHE).then((c) => c.put(req, copy)));
+    }
+    return res;
+  } catch (err) {
+    const hit = await cached(req);
+    if (hit) return hit;
+    // Offline and never cached. Navigations get the index rather than the
+    // browser's error page; anything else genuinely fails.
+    if (req.mode === 'navigate') {
+      const index = await cached('./index.html');
+      if (index) return index;
+    }
+    throw err;
+  }
+}
+"""
+
+
+def write_service_worker(out_dir, strategy="network-first"):
+    """Precache every file actually produced, versioned by their contents."""
+    digest = hashlib.sha256()
+    assets = []
+    for root, _dirs, names in os.walk(out_dir):
+        for name in sorted(names):
+            path = os.path.join(root, name)
+            rel = os.path.relpath(path, out_dir).replace(os.sep, "/")
+            if rel == "sw.js":
+                continue           # the browser revalidates the worker itself
+            assets.append("./" + rel)
+            with open(path, "rb") as fh:
+                digest.update(rel.encode() + fh.read())
+    assets.sort()
+    assets.insert(0, "./")         # a bare-root launch is its own cache entry
+    body = SW_TEMPLATE % {
+        "version": digest.hexdigest()[:12],
+        "assets": json.dumps(assets, indent=2),
+        "strategy": strategy,
+    }
+    with open(os.path.join(out_dir, "sw.js"), "w", encoding="utf-8") as fh:
+        fh.write(body)
+    return assets
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("-i", "--input", default="bible.json",
+                    help="JSON from extract_bible.py (default: bible.json)")
+    ap.add_argument("-o", "--out", default="books",
+                    help="output directory (default: books)")
+    ap.add_argument("--cache-first", action="store_true",
+                    help="service worker answers from cache before trying the "
+                         "network: fastest loads, but a rebuild takes two "
+                         "reloads to show up (default: network-first)")
+    args = ap.parse_args(argv)
+
+    with open(args.input, encoding="utf-8") as fh:
+        data = json.load(fh)
+    os.makedirs(args.out, exist_ok=True)
+
+    total = 0
+    for book in data["books"]:
+        path = render_book(book, data["translation"], args.out)
+        total += os.path.getsize(path)
+    render_index(data["books"], data["translation"], args.out)
+
+    css = os.path.join(args.out, "bible.css")
+    if not os.path.exists(css):
+        with open(css, "w", encoding="utf-8") as fh:
+            fh.write(CSS_STUB)
+
+    copy_assets(args.out)
+    write_icons(args.out)
+    write_manifest(args.out, data["translation"])
+    strategy = "cache-first" if args.cache_first else "network-first"
+    assets = write_service_worker(args.out, strategy)
+
+    print("books   %d" % len(data["books"]), file=sys.stderr)
+    print("precache %d assets (%s)" % (len(assets), strategy), file=sys.stderr)
+    print("html    %.1f MB total, %.0f KB average"
+          % (total / 1e6, total / len(data["books"]) / 1e3), file=sys.stderr)
+    print("wrote   %s/" % args.out, file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())