Clean

utils/compile_bible.py

@@ -0,0 +1,814 @@
+#!/usr/bin/env python3
+"""Compile the eBible.org HTML export into the fragments the app serves.
+
+One step, source to output. It reads a directory of eBible .htm files -- a flat
+set of per-chapter pages, ``JHN03.htm``, plus a site index -- and writes
+books/GEN.html and so on, plus books/index.html.
+
+There used to be two scripts and a 15 MB bible.json between them. The JSON was
+worth having when the corpus was going to be queried; nothing queries it, so
+it was a second representation to keep in step for no one's benefit. The parse
+tree now lives for as long as one process run.
+
+The two halves are still clearly separated below, because they are genuinely
+different jobs and the seam is where a second output format would attach:
+
+  READING     eBible's generated markup is a flat sequence of
+              ``<div class='...'>`` blocks whose class is the USFM paragraph
+              marker (``p``, ``q2``, ``s``, ...), with verse numbers, footnote
+              markers and character styles as inline spans. This half turns
+              that into a tree of blocks and inline nodes. Navigation and the
+              copyright footer are dropped; footnotes are kept.
+
+  WRITING     That tree becomes one semantic fragment per book: a <title> and
+              one root element carrying data-template, not a document. The
+              shell around it is assembled from the nunjucks templates in
+              assets/templates/ -- by the service worker, or by boot.html when
+              there can never be one -- so chrome changes never require running
+              this. The markup carries structure and no presentation: every
+              USFM distinction survives as a class or data attribute, so the
+              whole corpus can be restyled from the stylesheet without being
+              regenerated.
+
+This is the only build step in the project, and books/ is all it owns.
+Everything hand-written lives in assets/ and is served straight from there.
+
+Grouping worth knowing about: consecutive poetry lines are wrapped in a
+<div class="poetry"> that a blank-line block closes, so stanzas are real
+elements rather than spacing; consecutive list items become one <ul>.
+
+    python3 utils/compile_bible.py source -o books
+"""
+
+from __future__ import annotations
+
+import argparse
+import html
+import os
+import re
+import sys
+from html.parser import HTMLParser
+
+
+# ==========================================================================
+# READING -- eBible .htm into a tree of blocks and inline nodes
+# ==========================================================================
+
+# Chapter files are a three-character book code plus a zero-padded chapter
+# number (three digits for Psalms).
+CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$")
+
+# Links between chapters: "JOB05.htm#V13", "JHN03.htm", or a bare "#V7".
+HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$")
+
+# Block-level div classes -> semantic block type.
+BLOCK_TYPES = {
+    "p": "paragraph",    # normal prose paragraph
+    "m": "paragraph",    # continuation, no first-line indent
+    "nb": "paragraph",   # no break with previous
+    "pi": "paragraph",   # indented
+    "mi": "paragraph",   # indented, no first-line indent
+    "pc": "paragraph",   # centered
+    "ip": "paragraph",   # introduction paragraph
+    "q": "line",         # poetry line
+    "q2": "line",
+    "q3": "line",
+    "qs": "line",        # Selah
+    "b": "break",        # blank line between stanzas
+    "d": "title",        # descriptive title (psalm superscription, acrostic letter)
+    "s": "heading",      # section heading
+    "ms": "heading",     # major section heading ("BOOK 3")
+    "is": "heading",     # introduction heading
+    "sp": "speaker",     # speaker label (Song of Solomon)
+    "li": "item",        # list item
+    "ili": "item",       # introduction list item (glossary entries)
+    "mt": "booktitle",
+    "mt2": "booktitle",
+    "mt3": "booktitle",
+}
+
+# Nesting depth for poetry lines and indented paragraphs.
+INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1}
+
+# Heading / title rank.
+LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2}
+
+# Character-style spans -> inline node type. ``None`` means unwrap the span and
+# keep its children inline (``ft`` is just "footnote body text").
+INLINE = {
+    "wj": "wj",        # words of Jesus
+    "k": "keyword",    # glossary keyword
+    "bk": "bookref",   # book name cited in a note
+    "fq": "fq",        # the quoted words the footnote is about
+    "fqa": "fqa",      # an alternative rendering
+    "fl": "fl",        # footnote label
+    "fine": "fine",
+    "xt": "xt",        # cross-reference target
+    "ft": None,
+}
+
+# Divs whose entire subtree is navigation or licensing, not content.
+DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"}
+
+
+def parse_href(href):
+    """Turn an intra-corpus link into a structured reference, or None."""
+    m = HREF_RE.match(href or "")
+    if not m or not any(m.groups()):
+        return None
+    book, chapter, verse = m.groups()
+    ref = {}
+    if book:
+        ref["book"] = book
+        ref["chapter"] = int(chapter)
+    if verse:
+        ref["verse"] = int(verse)
+    return ref or None
+
+
+# --------------------------------------------------------------------------
+# Whitespace
+# --------------------------------------------------------------------------
+
+def normalize(nodes):
+    """Collapse whitespace across the whole inline tree, in document order.
+
+    Runs of whitespace become a single space and spaces are not duplicated
+    across element boundaries, so "word</span> <span>next" keeps exactly one.
+    """
+    state = {"prev_space": True}  # True at the start suppresses a leading space
+
+    def walk(lst):
+        out = []
+        for node in lst:
+            if node["t"] == "text":
+                text = re.sub(r"[\s ]+", " ", node["v"])
+                if state["prev_space"]:
+                    text = text.lstrip(" ")
+                if text:
+                    state["prev_space"] = text.endswith(" ")
+                    out.append({"t": "text", "v": text})
+            elif "c" in node:
+                node["c"] = walk(node["c"])
+                if node["c"]:
+                    out.append(node)
+            else:
+                state["prev_space"] = False
+                out.append(node)
+        return out
+
+    result = walk(nodes)
+    rstrip_last(result)
+    return result
+
+
+def rstrip_last(nodes):
+    """Drop the trailing space on a tree's final text node.
+
+    Only a genuinely trailing node is stripped. A space before a block's
+    closing verse or note marker separates it from the preceding word and
+    has to survive, so the walk stops at the last node rather than looking
+    past markers for something to strip.
+    """
+    if not nodes:
+        return False
+    node = nodes[-1]
+    if node["t"] == "text":
+        node["v"] = node["v"].rstrip()
+        if not node["v"]:
+            nodes.pop()
+            return rstrip_last(nodes)
+        return True
+    if "c" in node:
+        return rstrip_last(node["c"])
+    return False
+
+
+def plain_text(nodes, skip=("noteref",)):
+    """Flatten an inline tree to readable text, dropping note markers."""
+    parts = []
+    for node in nodes:
+        if node["t"] in skip:
+            continue
+        if node["t"] == "text":
+            parts.append(node["v"])
+        elif node["t"] == "br":
+            parts.append(" ")
+        elif "c" in node:
+            parts.append(plain_text(node["c"], skip))
+    return "".join(parts)
+
+
+# --------------------------------------------------------------------------
+# Chapter parser
+# --------------------------------------------------------------------------
+
+class ChapterParser(HTMLParser):
+    """Parse one chapter file into blocks + notes.
+
+    Content is appended to ``self.sink``, the inline-node list currently being
+    filled. A stack of frames records what to restore (and what to finish) as
+    each element closes; ``sink is None`` discards text, which is how
+    navigation and the copyright footer are dropped.
+    """
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.blocks = []
+        self.notes = []
+        self.titles = []          # mt/mt2/mt3, hoisted to the book
+        self.chapter_label = None
+        self.is_toc = False       # a chapter-0 navigation page
+
+        self.in_main = False
+        self.stack = []
+        self.sink = None
+
+    # -- frame helpers ----------------------------------------------------
+
+    def _push(self, tag, sink, on_close=None, **extra):
+        frame = {"tag": tag, "prev": self.sink, "close": on_close}
+        frame.update(extra)
+        self.stack.append(frame)
+        self.sink = sink
+        return frame
+
+    def _emit(self, node):
+        if self.sink is not None:
+            self.sink.append(node)
+
+    # -- HTMLParser hooks -------------------------------------------------
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        classes = (attrs.get("class") or "").split()
+        cls = classes[0] if classes else ""
+
+        if tag in ("br", "hr", "img", "meta", "link", "input"):
+            if tag == "br":
+                self._emit({"t": "br"})
+            return
+
+        if not self.in_main:
+            # Everything before <div class="main"> is <head> and top nav.
+            if tag == "div" and "main" in classes:
+                self.in_main = True
+                self._push(tag, None)
+            return
+
+        if cls in DROP_CLASSES:
+            if cls in ("toc", "toc1", "toc2"):
+                self.is_toc = True
+            self._push(tag, None)
+            return
+
+        if tag == "div" and cls == "chapterlabel":
+            buf = []
+            self._push(tag, buf, self._close_chapter_label, nodes=buf)
+            return
+
+        if tag == "div" and cls in BLOCK_TYPES:
+            block = {"type": BLOCK_TYPES[cls], "style": cls, "content": []}
+            if cls in INDENT:
+                block["indent"] = INDENT[cls]
+            if cls in LEVEL:
+                block["level"] = LEVEL[cls]
+            self._push(tag, block["content"], self._close_block, block=block)
+            return
+
+        if tag == "p" and cls in ("f", "x"):
+            note = {
+                "id": attrs.get("id"),
+                "kind": "footnote" if cls == "f" else "xref",
+                "marker": None,
+                "ref": None,
+                "verse": None,
+                "content": [],
+            }
+            self._push(tag, note["content"], self._close_note, note=note)
+            return
+
+        if tag == "span" and "verse" in classes:
+            buf = []
+            self._push(tag, buf, self._close_verse, nodes=buf)
+            return
+
+        if tag == "span" and cls == "notemark":
+            # The note's own marker inside the footnote list.
+            buf = []
+            self._push(tag, buf, self._close_note_marker, nodes=buf)
+            return
+
+        if tag == "a" and cls == "notemark":
+            # An in-text marker linking down to the note; its nested
+            # <span class="popup"> duplicates the note body and is dropped.
+            node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")}
+            self._emit(node)
+            buf = []
+            self._push(tag, buf, self._close_noteref, node=node, nodes=buf)
+            return
+
+        if tag == "a" and cls == "notebackref":
+            buf = []
+            self._push(tag, buf, self._close_backref, nodes=buf)
+            return
+
+        if tag == "span" and cls in INLINE:
+            kind = INLINE[cls]
+            if kind is None:
+                self._push(tag, self.sink)  # unwrap
+            else:
+                node = {"t": kind, "c": []}
+                self._emit(node)
+                self._push(tag, node["c"])
+            return
+
+        if tag == "a":
+            ref = parse_href(attrs.get("href"))
+            if ref is not None and self.sink is not None:
+                node = {"t": "ref", "c": []}
+                node.update(ref)
+                self._emit(node)
+                self._push(tag, node["c"])
+            else:
+                self._push(tag, self.sink)
+            return
+
+        # Anything else (nested spans, stray markup) is transparent.
+        self._push(tag, self.sink)
+
+    def handle_endtag(self, tag):
+        if not self.in_main or not self.stack:
+            return
+        for i in range(len(self.stack) - 1, -1, -1):
+            if self.stack[i]["tag"] == tag:
+                break
+        else:
+            return  # unmatched close tag; ignore
+        while len(self.stack) > i:
+            frame = self.stack.pop()
+            self.sink = frame["prev"]
+            if frame["close"]:
+                frame["close"](frame)
+        if tag == "div" and not self.stack:
+            self.in_main = False  # <div class="main"> closed
+
+    def handle_data(self, data):
+        if self.sink is not None and data:
+            self.sink.append({"t": "text", "v": data})
+
+    # -- frame close handlers ---------------------------------------------
+
+    def _close_chapter_label(self, frame):
+        self.chapter_label = plain_text(frame["nodes"]).strip()
+
+    def _close_block(self, frame):
+        block = frame["block"]
+        block["content"] = normalize(block["content"])
+        if block["type"] == "booktitle":
+            if block["content"]:
+                self.titles.append({
+                    "style": block["style"],
+                    "level": block.get("level"),
+                    "text": plain_text(block["content"]),
+                })
+            return
+        if block["type"] == "break":
+            block.pop("content")
+            self.blocks.append(block)
+            return
+        if block["content"]:
+            self.blocks.append(block)
+
+    def _close_note(self, frame):
+        note = frame["note"]
+        note["content"] = normalize(note["content"])
+        note["text"] = plain_text(note["content"])
+        self.notes.append(note)
+
+    def _close_verse(self, frame):
+        label = plain_text(frame["nodes"]).replace(" ", " ").strip()
+        node = {"t": "verse", "label": label}
+        m = re.match(r"^(\d+)(?:[-–](\d+))?$", label)
+        if m:
+            node["n"] = int(m.group(1))
+            if m.group(2):
+                node["end"] = int(m.group(2))
+        self._emit(node)
+
+    def _close_note_marker(self, frame):
+        marker = plain_text(frame["nodes"]).strip()
+        for f in reversed(self.stack):
+            if "note" in f:
+                f["note"]["marker"] = marker
+                return
+
+    def _close_noteref(self, frame):
+        frame["node"]["marker"] = plain_text(frame["nodes"]).strip()
+
+    def _close_backref(self, frame):
+        text = plain_text(frame["nodes"]).strip()
+        for f in reversed(self.stack):
+            if "note" in f:
+                f["note"]["ref"] = text
+                m = re.search(r":(\d+)", text)
+                if m:
+                    f["note"]["verse"] = int(m.group(1))
+                return
+
+
+# --------------------------------------------------------------------------
+# Corpus
+# --------------------------------------------------------------------------
+
+class IndexParser(HTMLParser):
+    """Read book order, display names and canon section from index.htm."""
+
+    # Link class on the site index marks which part of the canon a book is in.
+    SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"}
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.books = []       # [{code, name, section, order}]
+        self.title = None
+        self._link = None
+        self._in_title = False
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if tag == "title":
+            self._in_title = True
+        elif tag == "a":
+            m = CHAPTER_RE.match(os.path.basename(attrs.get("href") or ""))
+            section = self.SECTIONS.get((attrs.get("class") or "").strip())
+            if m and section:
+                self._link = {
+                    "code": m.group(1),
+                    "section": section,
+                    "order": len(self.books) + 1,
+                    "name": "",
+                }
+
+    def handle_endtag(self, tag):
+        if tag == "title":
+            self._in_title = False
+        elif tag == "a" and self._link:
+            self._link["name"] = self._link["name"].strip()
+            self.books.append(self._link)
+            self._link = None
+
+    def handle_data(self, data):
+        if self._in_title and self.title is None:
+            self.title = data.strip()
+        elif self._link is not None:
+            self._link["name"] += data
+
+
+def parse_chapter(path):
+    parser = ChapterParser()
+    with open(path, encoding="utf-8-sig") as fh:
+        parser.feed(fh.read())
+    parser.close()
+    return parser
+
+
+def extract(source_dir):
+    index_path = os.path.join(source_dir, "index.htm")
+    index = IndexParser()
+    with open(index_path, encoding="utf-8-sig") as fh:
+        index.feed(fh.read())
+
+    meta = {b["code"]: b for b in index.books}
+
+    # Group chapter files by book code.
+    by_book = {}
+    for name in sorted(os.listdir(source_dir)):
+        m = CHAPTER_RE.match(name)
+        if m:
+            by_book.setdefault(m.group(1), []).append((int(m.group(2)), name))
+
+    books = []
+    skipped = []
+    for code in sorted(by_book, key=lambda c: (meta.get(c, {}).get("order", 999), c)):
+        info = meta.get(code, {"code": code, "name": code, "section": "other",
+                               "order": 999})
+        book = {
+            "code": code,
+            "name": info["name"],
+            "section": info["section"],
+            "titles": [],
+            "chapters": [],
+        }
+        for number, name in sorted(by_book[code]):
+            parsed = parse_chapter(os.path.join(source_dir, name))
+            if parsed.is_toc or not parsed.blocks:
+                # Chapter-0 files are per-book tables of contents, not text.
+                skipped.append(name)
+                continue
+            if parsed.titles and not book["titles"]:
+                book["titles"] = parsed.titles
+            label = parsed.chapter_label
+            chapter = {
+                "number": int(label) if label and label.isdigit() else number,
+                "blocks": parsed.blocks,
+                "notes": parsed.notes,
+            }
+            book["chapters"].append(chapter)
+        if book["chapters"]:
+            books.append(book)
+
+    return {
+        "translation": {
+            "id": "engwebu",
+            "name": index.title or "World English Bible Updated",
+            "license": "Public Domain",
+            "source": "eBible.org",
+        },
+        "books": books,
+    }, skipped
+
+
+# ==========================================================================
+# WRITING -- that tree into one HTML fragment per book
+# ==========================================================================
+
+# Inline node type -> (tag, class). ``cite`` and ``dfn`` carry real meaning
+# here: a book name cited in a note, and a glossary term at its definition.
+INLINE_TAGS = {
+    "wj": ("span", "wj"),          # words of Jesus
+    "keyword": ("dfn", "k"),       # glossary headword
+    "bookref": ("cite", "bk"),     # book title cited in a note
+    "fq": ("span", "fq"),          # the words a footnote is about
+    "fqa": ("span", "fqa"),        # an alternative rendering
+    "fl": ("span", "fl"),          # footnote label
+    "fine": ("span", "fine"),
+    "xt": ("span", "xt"),          # cross-reference target
+}
+
+# Block type -> tag. Headings sit at h3 so the outline never skips a level:
+# h1 book, h2 chapter, h3 section.
+BLOCK_TAGS = {
+    "paragraph": "p",
+    "line": "p",
+    "heading": "h3",
+    "title": "p",
+    "speaker": "p",
+    "item": "li",
+}
+
+
+
+def esc(text):
+    return html.escape(text, quote=False)
+
+
+def verse_id(chapter, label):
+    """Page-unique verse anchor, e.g. v23_1 (or v11_15-16 for a range)."""
+    return "v%d_%s" % (chapter, label)
+
+
+def ref_href(node, book, chapter):
+    """Link an intra-corpus reference; whole books are single pages."""
+    target_book = node.get("book") or book
+    target_ch = node.get("chapter", chapter)
+    verse = node.get("verse")
+    base = "" if target_book == book else target_book
+    if verse:
+        return "%s#%s" % (base, verse_id(target_ch, verse))
+    return "%s#ch%d" % (base, target_ch)
+
+
+def render_inline(nodes, book, chapter, out):
+    for node in nodes:
+        kind = node["t"]
+        if kind == "text":
+            out.append(esc(node["v"]))
+        elif kind == "verse":
+            # Trailing space keeps the number off the text before any CSS
+            # lands; style .v with padding and it stays correct either way.
+            out.append('<span class="v" id="%s">%s</span> ' % (
+                verse_id(chapter, node["label"]), esc(node["label"])))
+        elif kind == "noteref":
+            n = node["id"].replace("FN", "")
+            out.append('<a class="noteref" id="fnref%s_%s" href="#fn%s_%s">'
+                       "<sup>%s</sup></a>" % (chapter, n, chapter, n,
+                                              esc(caller(node.get("marker")))))
+        elif kind == "br":
+            out.append("<br>")
+        elif kind == "ref":
+            out.append('<a class="ref" href="%s">' % ref_href(node, book, chapter))
+            render_inline(node.get("c", []), book, chapter, out)
+            out.append("</a>")
+        elif kind in INLINE_TAGS:
+            tag, cls = INLINE_TAGS[kind]
+            out.append('<%s class="%s">' % (tag, cls))
+            render_inline(node.get("c", []), book, chapter, out)
+            out.append("</%s>" % tag)
+        else:  # unknown node type: keep the words, drop the wrapper
+            render_inline(node.get("c", []), book, chapter, out)
+
+
+def render_block(block, book, chapter, out):
+    btype = block["type"]
+    tag = BLOCK_TAGS.get(btype, "p")
+    classes = [block["style"]]
+    attrs = ""
+    if "indent" in block:
+        attrs += ' data-indent="%d"' % block["indent"]
+    if btype == "line":
+        classes.append("line")
+    out.append("      <%s class=\"%s\"%s>" % (tag, " ".join(classes), attrs))
+    render_inline(block.get("content", []), book, chapter, out)
+    out.append("</%s>\n" % tag)
+
+
+# Footnote callers arrive from the source as * † ‡ § ✡. The star is the only
+# one of them a browser may decide is an emoji: no serif font carries a glyph
+# for U+2721, so the fallback chain reaches the colour emoji font and drops a
+# purple tile into the middle of a verse. U+FE0E is the standardised way to ask
+# for the text glyph of the same character -- the symbol, not the emoji. There
+# is no separate text codepoint to switch to; this is the only lever.
+TEXT_PRESENTATION = "\ufe0e"
+EMOJI_CAPABLE = "\u2721"
+
+
+def caller(marker):
+    """A footnote marker, asked to render as text rather than as an emoji."""
+    return "".join(c + TEXT_PRESENTATION if c in EMOJI_CAPABLE else c
+                   for c in (marker or "*"))
+
+
+def render_notes(notes, chapter, book, out):
+    if not notes:
+        return
+    out.append('      <section class="footnotes">\n')
+    out.append('        <h4 class="footnotes-title">Notes</h4>\n')
+    out.append("        <ol>\n")
+    for note in notes:
+        n = (note["id"] or "").replace("FN", "")
+        out.append('          <li class="note" id="fn%s_%s" data-kind="%s">'
+                   % (chapter, n, note["kind"]))
+        out.append('<span class="marker">%s</span> ' % esc(caller(note.get("marker"))))
+        if note.get("ref"):
+            out.append('<a class="backref" href="#fnref%s_%s">%s</a> '
+                       % (chapter, n, esc(note["ref"])))
+        out.append('<span class="ft">')
+        render_inline(note["content"], book, chapter, out)
+        out.append("</span></li>\n")
+    out.append("        </ol>\n      </section>\n")
+
+
+def render_chapter(chapter, book, out):
+    num = chapter["number"]
+    out.append('    <section class="chapter" id="ch%d" data-chapter="%d">\n'
+               % (num, num))
+    out.append('      <h2 class="chapter-number">%d</h2>\n' % num)
+
+    group = None  # "poetry" | "list" | None
+
+    def close_group():
+        nonlocal group
+        if group == "poetry":
+            out.append("      </div>\n")
+        elif group == "list":
+            out.append("      </ul>\n")
+        group = None
+
+    for block in chapter["blocks"]:
+        btype = block["type"]
+        if btype == "break":
+            close_group()          # a blank line ends the stanza
+            continue
+        want = {"line": "poetry", "item": "list"}.get(btype)
+        if want != group:
+            close_group()
+            if want == "poetry":
+                out.append('      <div class="poetry">\n')
+            elif want == "list":
+                out.append('      <ul class="list">\n')
+            group = want
+        render_block(block, book, num, out)
+    close_group()
+
+    render_notes(chapter["notes"], num, book, out)
+    out.append("    </section>\n")
+
+
+def render_book(book, translation, out_dir):
+    code, name = book["code"], book["name"]
+    out = []
+    title = "%s — %s" % (name, translation["name"])
+    # A fragment: the title the shell will use, then one root element whose
+    # data-template names the template that wraps it.
+    out.append("<title>%s</title>\n" % esc(title))
+    out.append('<article class="book" data-template="book" id="%s" data-book="%s" '
+               'data-name="%s" data-section="%s">\n'
+               % (code, code, esc(name), book["section"]))
+
+    # Header: the full multi-line title if the source had one.
+    out.append('  <header class="book-header">\n')
+    out.append("    <h1>")
+    if book["titles"]:
+        # Separated by a space, not run together. The spans record what USFM
+        # called them -- mt2 and mt3 were lines above and below the main title
+        # -- but they are set as one line here, and without the space between
+        # them Genesis reads "The First Book of Moses,Commonly CalledGenesis".
+        # The markup still says what each part is, so stacking them again is a
+        # stylesheet away.
+        out.append(" ".join('<span class="%s">%s</span>'
+                            % (part["style"], esc(part["text"]))
+                            for part in book["titles"]))
+    else:
+        out.append(esc(name))
+    out.append("</h1>\n")
+    # The title and nothing else. Both of the things that used to sit under it
+    # are in the navbar now, where they are in reach from anywhere in the book
+    # rather than only from the top of it: the chapter list became the
+    # reference picker, which reads its chapters and verses straight off this
+    # markup, and "All books" is what the navbar brand already does. In Psalms
+    # the chapter list alone was a hundred and fifty links, carried in the
+    # fragment and in every device's offline copy.
+    out.append("  </header>\n")
+
+    out.append('  <div class="chapters">\n')
+    for chapter in book["chapters"]:
+        render_chapter(chapter, code, out)
+    out.append("  </div>\n")
+
+    out.append('  <footer class="colophon"><p>%s. %s.</p></footer>\n'
+               % (esc(translation["name"]), esc(translation["license"])))
+    out.append("</article>\n")
+
+    path = os.path.join(out_dir, "%s.html" % code)
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write("".join(out))
+    return path
+
+
+SECTION_LABELS = [
+    ("ot", "Old Testament"),
+    ("nt", "New Testament"),
+    ("deuterocanon", "Deuterocanon"),
+    ("other", "Other"),
+]
+
+
+def render_index(books, translation, out_dir):
+    out = ["<title>%s</title>\n" % esc(translation["name"]),
+           '<div class="index" data-template="index">\n']
+    for key, label in SECTION_LABELS:
+        group = [b for b in books if b["section"] == key]
+        if not group:
+            continue
+        out.append('  <section class="book-group" data-section="%s">\n' % key)
+        out.append("    <h2>%s</h2>\n    <ul>\n" % esc(label))
+        for book in group:
+            out.append('      <li><a href="%s">%s</a> '
+                       '<span class="count">%d</span></li>\n'
+                       % (book["code"], esc(book["name"]), len(book["chapters"])))
+        out.append("    </ul>\n  </section>\n")
+    out.append('  <footer class="colophon"><p>%s. %s.</p></footer>\n'
+               % (esc(translation["name"]), esc(translation["license"])))
+    out.append("</div>\n")
+    path = os.path.join(out_dir, "index.html")
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write("".join(out))
+    return path
+
+
+# ==========================================================================
+# Entry point
+# ==========================================================================
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("source", nargs="?", default="source",
+                    help="directory of eBible .htm files (default: source)")
+    ap.add_argument("-o", "--out", default="books",
+                    help="output directory (default: books)")
+    args = ap.parse_args(argv)
+
+    data, skipped = extract(args.source)
+    os.makedirs(args.out, exist_ok=True)
+
+    total = 0
+    for book in data["books"]:
+        total += os.path.getsize(render_book(book, data["translation"], args.out))
+    render_index(data["books"], data["translation"], args.out)
+
+    chapters = sum(len(b["chapters"]) for b in data["books"])
+    notes = sum(len(c["notes"]) for b in data["books"] for c in b["chapters"])
+    print("read     %s/" % args.source.rstrip("/"), file=sys.stderr)
+    print("books    %d" % len(data["books"]), file=sys.stderr)
+    print("chapters %d" % chapters, file=sys.stderr)
+    print("notes    %d" % notes, file=sys.stderr)
+    print("skipped  %d (contents pages)" % len(skipped), file=sys.stderr)
+    print("html     %.1f MB total, %.0f KB average"
+          % (total / 1e6, total / len(data["books"]) / 1e3), file=sys.stderr)
+    print("wrote    %s/" % args.out.rstrip("/"), file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())