mark/bible
1
0 Fork 0
Code Releases Activity

Remove comments

Browse Source
This commit is contained in:
mark committed 2026-09-09 12:09:44 -04:00
1 parent 872175153a
commit 057b4be7d7
18 files changed
+66 -1350
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/compile_bible.py
+42 -265
View File
@@ -1,44 +1,4 @@
#!/usr/bin/env python3
"""Compile the eBible.org HTML export into the fragments the app serves.
One step, source to output. It reads a directory of eBible .htm files -- a flat
set of per-chapter pages, ``JHN03.htm``, plus a site index -- and writes
bible/GEN.html and so on, plus bible/index.html.
There used to be two scripts and a 15 MB bible.json between them. The JSON was
worth having when the corpus was going to be queried; nothing queries it, so
it was a second representation to keep in step for no one's benefit. The parse
tree now lives for as long as one process run.
The two halves are still clearly separated below, because they are genuinely
different jobs and the seam is where a second output format would attach:
READING eBible's generated markup is a flat sequence of
``<div class='...'>`` blocks whose class is the USFM paragraph
marker (``p``, ``q2``, ``s``, ...), with verse numbers, footnote
markers and character styles as inline spans. This half turns
that into a tree of blocks and inline nodes. Navigation and the
copyright footer are dropped; footnotes are kept.
WRITING That tree becomes one semantic fragment per book: a <title> and
one root element carrying data-template, not a document. The
shell around it is assembled from the nunjucks templates in
assets/templates/ by the service worker, so chrome changes never
require running this. The markup carries structure and no presentation: every
USFM distinction survives as a class or data attribute, so the
whole corpus can be restyled from the stylesheet without being
regenerated.
This is the only build step in the project, and bible/ is all it owns.
Its output is not committed: bible/ is in .gitignore, so a deploy runs this.
Everything hand-written lives in assets/ and is served straight from there.
Grouping worth knowing about: consecutive poetry lines are wrapped in a
<div class="poetry"> that a blank-line block closes, so stanzas are real
elements rather than spacing; consecutive list items become one <ul>.
python3 utils/compile_bible.py source -o bible
"""
"""Compile the eBible.org HTML export into the fragments the app serves."""
from __future__ import annotations
@@ -50,70 +10,54 @@ import re
import sys
from html.parser import HTMLParser
# ==========================================================================
# READING -- eBible .htm into a tree of blocks and inline nodes
# ==========================================================================
# Chapter files are a three-character book code plus a zero-padded chapter
# number (three digits for Psalms).
CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$")
# Links between chapters: "JOB05.htm#V13", "JHN03.htm", or a bare "#V7".
HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$")
# Block-level div classes -> semantic block type.
BLOCK_TYPES = {
"p": "paragraph", # normal prose paragraph
"m": "paragraph", # continuation, no first-line indent
"nb": "paragraph", # no break with previous
"pi": "paragraph", # indented
"mi": "paragraph", # indented, no first-line indent
"pc": "paragraph", # centered
"ip": "paragraph", # introduction paragraph
"q": "line", # poetry line
"p": "paragraph",
"m": "paragraph",
"nb": "paragraph",
"pi": "paragraph",
"mi": "paragraph",
"pc": "paragraph",
"ip": "paragraph",
"q": "line",
"q2": "line",
"q3": "line",
"qs": "line", # Selah
"b": "break", # blank line between stanzas
"d": "title", # descriptive title (psalm superscription, acrostic letter)
"s": "heading", # section heading
"ms": "heading", # major section heading ("BOOK 3")
"is": "heading", # introduction heading
"sp": "speaker", # speaker label (Song of Solomon)
"li": "item", # list item
"ili": "item", # introduction list item (glossary entries)
"qs": "line",
"b": "break",
"d": "title",
"s": "heading",
"ms": "heading",
"is": "heading",
"sp": "speaker",
"li": "item",
"ili": "item",
"mt": "booktitle",
"mt2": "booktitle",
"mt3": "booktitle",
}
# Nesting depth for poetry lines and indented paragraphs.
INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1}
# Heading / title rank.
LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2}
# Character-style spans -> inline node type. ``None`` means unwrap the span and
# keep its children inline (``ft`` is just "footnote body text").
INLINE = {
"wj": "wj", # words of Jesus
"k": "keyword", # glossary keyword
"bk": "bookref", # book name cited in a note
"fq": "fq", # the quoted words the footnote is about
"fqa": "fqa", # an alternative rendering
"fl": "fl", # footnote label
"wj": "wj",
"k": "keyword",
"bk": "bookref",
"fq": "fq",
"fqa": "fqa",
"fl": "fl",
"fine": "fine",
"xt": "xt", # cross-reference target
"xt": "xt",
"ft": None,
}
# Divs whose entire subtree is navigation or licensing, not content.
DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"}
def parse_href(href):
"""Turn an intra-corpus link into a structured reference, or None."""
m = HREF_RE.match(href or "")
if not m or not any(m.groups()):
return None
@@ -126,18 +70,8 @@ def parse_href(href):
ref["verse"] = int(verse)
return ref or None
# --------------------------------------------------------------------------
# Whitespace
# --------------------------------------------------------------------------
def normalize(nodes):
"""Collapse whitespace across the whole inline tree, in document order.
Runs of whitespace become a single space and spaces are not duplicated
across element boundaries, so "word</span> <span>next" keeps exactly one.
"""
state = {"prev_space": True} # True at the start suppresses a leading space
state = {"prev_space": True}
def walk(lst):
out = []
@@ -162,15 +96,7 @@ def normalize(nodes):
rstrip_last(result)
return result
def rstrip_last(nodes):
"""Drop the trailing space on a tree's final text node.
Only a genuinely trailing node is stripped. A space before a block's
closing verse or note marker separates it from the preceding word and
has to survive, so the walk stops at the last node rather than looking
past markers for something to strip.
"""
if not nodes:
return False
node = nodes[-1]
@@ -184,9 +110,7 @@ def rstrip_last(nodes):
return rstrip_last(node["c"])
return False
def plain_text(nodes, skip=("noteref",)):
"""Flatten an inline tree to readable text, dropping note markers."""
parts = []
for node in nodes:
if node["t"] in skip:
@@ -199,34 +123,20 @@ def plain_text(nodes, skip=("noteref",)):
parts.append(plain_text(node["c"], skip))
return "".join(parts)
# --------------------------------------------------------------------------
# Chapter parser
# --------------------------------------------------------------------------
class ChapterParser(HTMLParser):
"""Parse one chapter file into blocks + notes.
Content is appended to ``self.sink``, the inline-node list currently being
filled. A stack of frames records what to restore (and what to finish) as
each element closes; ``sink is None`` discards text, which is how
navigation and the copyright footer are dropped.
"""
def __init__(self):
super().__init__(convert_charrefs=True)
self.blocks = []
self.notes = []
self.titles = [] # mt/mt2/mt3, hoisted to the book
self.titles = []
self.chapter_label = None
self.is_toc = False # a chapter-0 navigation page
self.is_toc = False
self.in_main = False
self.stack = []
self.sink = None
# -- frame helpers ----------------------------------------------------
def _push(self, tag, sink, on_close=None, **extra):
frame = {"tag": tag, "prev": self.sink, "close": on_close}
frame.update(extra)
@@ -238,8 +148,6 @@ class ChapterParser(HTMLParser):
if self.sink is not None:
self.sink.append(node)
# -- HTMLParser hooks -------------------------------------------------
def handle_starttag(self, tag, attrs):
attrs = dict(attrs)
classes = (attrs.get("class") or "").split()
@@ -251,7 +159,6 @@ class ChapterParser(HTMLParser):
return
if not self.in_main:
# Everything before <div class="main"> is <head> and top nav.
if tag == "div" and "main" in classes:
self.in_main = True
self._push(tag, None)
@@ -295,14 +202,11 @@ class ChapterParser(HTMLParser):
return
if tag == "span" and cls == "notemark":
# The note's own marker inside the footnote list.
buf = []
self._push(tag, buf, self._close_note_marker, nodes=buf)
return
if tag == "a" and cls == "notemark":
# An in-text marker linking down to the note; its nested
# <span class="popup"> duplicates the note body and is dropped.
node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")}
self._emit(node)
buf = []
@@ -317,7 +221,7 @@ class ChapterParser(HTMLParser):
if tag == "span" and cls in INLINE:
kind = INLINE[cls]
if kind is None:
self._push(tag, self.sink) # unwrap
self._push(tag, self.sink)
else:
node = {"t": kind, "c": []}
self._emit(node)
@@ -335,7 +239,6 @@ class ChapterParser(HTMLParser):
self._push(tag, self.sink)
return
# Anything else (nested spans, stray markup) is transparent.
self._push(tag, self.sink)
def handle_endtag(self, tag):
@@ -345,21 +248,19 @@ class ChapterParser(HTMLParser):
if self.stack[i]["tag"] == tag:
break
else:
return # unmatched close tag; ignore
return
while len(self.stack) > i:
frame = self.stack.pop()
self.sink = frame["prev"]
if frame["close"]:
frame["close"](frame)
if tag == "div" and not self.stack:
self.in_main = False # <div class="main"> closed
self.in_main = False
def handle_data(self, data):
if self.sink is not None and data:
self.sink.append({"t": "text", "v": data})
# -- frame close handlers ---------------------------------------------
def _close_chapter_label(self, frame):
self.chapter_label = plain_text(frame["nodes"]).strip()
@@ -417,20 +318,13 @@ class ChapterParser(HTMLParser):
f["note"]["verse"] = int(m.group(1))
return
# --------------------------------------------------------------------------
# Corpus
# --------------------------------------------------------------------------
class IndexParser(HTMLParser):
"""Read book order, display names and canon section from index.htm."""
# Link class on the site index marks which part of the canon a book is in.
SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"}
def __init__(self):
super().__init__(convert_charrefs=True)
self.books = [] # [{code, name, section, order}]
self.books = []
self.title = None
self._link = None
self._in_title = False
@@ -464,7 +358,6 @@ class IndexParser(HTMLParser):
elif self._link is not None:
self._link["name"] += data
def parse_chapter(path):
parser = ChapterParser()
with open(path, encoding="utf-8-sig") as fh:
@@ -472,7 +365,6 @@ def parse_chapter(path):
parser.close()
return parser
def extract(source_dir):
index_path = os.path.join(source_dir, "index.htm")
index = IndexParser()
@@ -481,7 +373,6 @@ def extract(source_dir):
meta = {b["code"]: b for b in index.books}
# Group chapter files by book code.
by_book = {}
for name in sorted(os.listdir(source_dir)):
m = CHAPTER_RE.match(name)
@@ -503,7 +394,6 @@ def extract(source_dir):
for number, name in sorted(by_book[code]):
parsed = parse_chapter(os.path.join(source_dir, name))
if parsed.is_toc or not parsed.blocks:
# Chapter-0 files are per-book tables of contents, not text.
skipped.append(name)
continue
if parsed.titles and not book["titles"]:
@@ -528,26 +418,17 @@ def extract(source_dir):
"books": books,
}, skipped
# ==========================================================================
# WRITING -- that tree into one HTML fragment per book
# ==========================================================================
# Inline node type -> (tag, class). ``cite`` and ``dfn`` carry real meaning
# here: a book name cited in a note, and a glossary term at its definition.
INLINE_TAGS = {
"wj": ("span", "wj"), # words of Jesus
"keyword": ("dfn", "k"), # glossary headword
"bookref": ("cite", "bk"), # book title cited in a note
"fq": ("span", "fq"), # the words a footnote is about
"fqa": ("span", "fqa"), # an alternative rendering
"fl": ("span", "fl"), # footnote label
"wj": ("span", "wj"),
"keyword": ("dfn", "k"),
"bookref": ("cite", "bk"),
"fq": ("span", "fq"),
"fqa": ("span", "fqa"),
"fl": ("span", "fl"),
"fine": ("span", "fine"),
"xt": ("span", "xt"), # cross-reference target
"xt": ("span", "xt"),
}
# Block type -> tag. Headings sit at h3 so the outline never skips a level:
# h1 book, h2 chapter, h3 section.
BLOCK_TAGS = {
"paragraph": "p",
"line": "p",
@@ -557,19 +438,13 @@ BLOCK_TAGS = {
"item": "li",
}
def esc(text):
return html.escape(text, quote=False)
def verse_id(chapter, label):
"""Page-unique verse anchor, e.g. v23_1 (or v11_15-16 for a range)."""
return "v%d_%s" % (chapter, label)
def ref_href(node, book, chapter):
"""Link an intra-corpus reference; whole books are single pages."""
target_book = node.get("book") or book
target_ch = node.get("chapter", chapter)
verse = node.get("verse")
@@ -578,26 +453,12 @@ def ref_href(node, book, chapter):
return "%s#%s" % (base, verse_id(target_ch, verse))
return "%s#ch%d" % (base, target_ch)
def render_inline(nodes, book, chapter, out):
for node in nodes:
kind = node["t"]
if kind == "text":
out.append(esc(node["v"]))
elif kind == "verse":
# Trailing space keeps the number off the text before any CSS
# lands; style .v with padding and it stays correct either way.
#
# A control, said here rather than by a pass over the page once it
# has landed. role is what announces it and what earns Reboot's
# [role=button] pointer cursor, tabindex is what puts it in the tab
# order, and the label is what a screen reader says in place of a
# bare number. All three are true of every verse in the corpus and
# none of them depends on anything only the browser knows, so
# there is nothing here for a script to work out: app.js used to
# set them on load, after every htmx swap and after every history
# restore, walking a whole book each time. The click itself never
# needed that pass -- it is delegated at #content in base.njk.
label = esc(node["label"])
out.append('<span class="v" id="%s" role="button" tabindex="0" '
'aria-label="Verse %s, options">%s</span> '
@@ -618,10 +479,9 @@ def render_inline(nodes, book, chapter, out):
out.append('<%s class="%s">' % (tag, cls))
render_inline(node.get("c", []), book, chapter, out)
out.append("</%s>" % tag)
else: # unknown node type: keep the words, drop the wrapper
else:
render_inline(node.get("c", []), book, chapter, out)
def render_block(block, book, chapter, out):
btype = block["type"]
tag = BLOCK_TAGS.get(btype, "p")
@@ -635,23 +495,13 @@ def render_block(block, book, chapter, out):
render_inline(block.get("content", []), book, chapter, out)
out.append("</%s>\n" % tag)
# Footnote callers arrive from the source as * † ‡ § ✡. The star is the only
# one of them a browser may decide is an emoji: no serif font carries a glyph
# for U+2721, so the fallback chain reaches the colour emoji font and drops a
# purple tile into the middle of a verse. U+FE0E is the standardised way to ask
# for the text glyph of the same character -- the symbol, not the emoji. There
# is no separate text codepoint to switch to; this is the only lever.
TEXT_PRESENTATION = "\ufe0e"
EMOJI_CAPABLE = "\u2721"
def caller(marker):
"""A footnote marker, asked to render as text rather than as an emoji."""
return "".join(c + TEXT_PRESENTATION if c in EMOJI_CAPABLE else c
for c in (marker or "*"))
def render_notes(notes, chapter, book, out):
if not notes:
return
@@ -671,14 +521,13 @@ def render_notes(notes, chapter, book, out):
out.append("</span></li>\n")
out.append(" </ol>\n </section>\n")
def render_chapter(chapter, book, out):
num = chapter["number"]
out.append(' <section class="chapter" id="ch%d" data-chapter="%d">\n'
% (num, num))
out.append(' <h2 class="chapter-number">%d</h2>\n' % num)
group = None # "poetry" | "list" | None
group = None
def close_group():
nonlocal group
@@ -691,7 +540,7 @@ def render_chapter(chapter, book, out):
for block in chapter["blocks"]:
btype = block["type"]
if btype == "break":
close_group() # a blank line ends the stanza
close_group()
continue
want = {"line": "poetry", "item": "list"}.get(btype)
if want != group:
@@ -707,41 +556,24 @@ def render_chapter(chapter, book, out):
render_notes(chapter["notes"], num, book, out)
out.append(" </section>\n")
def render_book(book, translation, out_dir):
code, name = book["code"], book["name"]
out = []
title = "%s — %s" % (name, translation["name"])
# A fragment: the title the shell will use, then one root element whose
# data-template names the template that wraps it.
out.append("<title>%s</title>\n" % esc(title))
out.append('<article class="book" data-template="book" id="%s" data-book="%s" '
'data-name="%s" data-section="%s">\n'
% (code, code, esc(name), book["section"]))
# Header: the full multi-line title if the source had one.
out.append(' <header class="book-header">\n')
out.append(" <h1>")
if book["titles"]:
# Separated by a space, not run together. The spans record what USFM
# called them -- mt2 and mt3 were lines above and below the main title
# -- but they are set as one line here, and without the space between
# them Genesis reads "The First Book of Moses,Commonly CalledGenesis".
# The markup still says what each part is, so stacking them again is a
# stylesheet away.
out.append(" ".join('<span class="%s">%s</span>'
% (part["style"], esc(part["text"]))
for part in book["titles"]))
else:
out.append(esc(name))
out.append("</h1>\n")
# The title and nothing else. Both of the things that used to sit under it
# are in the navbar now, where they are in reach from anywhere in the book
# rather than only from the top of it: the chapter list became the
# reference picker, which reads its chapters and verses straight off this
# markup, and "All books" is what the navbar brand already does. In Psalms
# the chapter list alone was a hundred and fifty links, carried in the
# fragment and in every device's offline copy.
out.append(" </header>\n")
out.append(' <div class="chapters">\n')
@@ -758,7 +590,6 @@ def render_book(book, translation, out_dir):
fh.write("".join(out))
return path
SECTION_LABELS = [
("ot", "Old Testament"),
("nt", "New Testament"),
@@ -766,7 +597,6 @@ SECTION_LABELS = [
("other", "Other"),
]
def render_index(books, translation, out_dir):
out = ["<title>%s</title>\n" % esc(translation["name"]),
'<div class="index" data-template="index">\n']
@@ -789,48 +619,9 @@ def render_index(books, translation, out_dir):
fh.write("".join(out))
return path
# ==========================================================================
# WRITING -- the reference map
# ==========================================================================
# bible/verses.json: every book's chapters, and every chapter's verse labels.
#
# The navbar's reference picker is built from this, by the service worker. It
# used to be built in the browser by reading the book already on the page --
# the chapters were the sections and a chapter's verses were the anchors inside
# it -- which was free but only worked where there was a document to read. A
# worker has no DOM, and the picker now lives in a shell the worker composes
# before any document exists, so what the picker needs has to be a fact on
# disk rather than something recovered from markup.
#
# Written here, from the same tree the fragments are rendered from, so the two
# cannot disagree: every option this file produces names an anchor render_book
# wrote, because both come from the same verse nodes.
#
# The encoding is compact on purpose. A chapter whose verses run 1..n is
# written as the number n; only the eight in the corpus with a gap or a range
# -- Sirach 26 ends "19-27", 28, 29 -- spell their labels out:
#
# {"GEN": [31, 25, 24, ...], "SIR": [..., ["1", ..., "19-27", "28", "29"]]}
#
# That is 6 KB for 38,000 verses against 180 KB of spelled-out labels, which is
# the difference between a file the worker can parse every time it wakes and
# one it would have to find somewhere to keep.
#
# Chapter numbers are not in the file at all: they are the array index plus
# one. render_verses checks that, rather than assuming it.
VERSES_FILE = "verses.json"
def chapter_verses(chapter):
"""The verse labels of one chapter, in document order.
Blocks only. Footnotes hang inside the chapter section too, but a note's
content never carries a verse node -- and if one ever did, it would be a
citation rather than a place in the text.
"""
labels = []
def walk(nodes):
@@ -844,20 +635,12 @@ def chapter_verses(chapter):
walk(block.get("content", []))
return labels
def render_verses(books, out_dir):
"""Write the reference map. Returns its path and the verses in it.
See the note above for the encoding.
"""
out = {}
total = 0
for book in books:
numbers = [c["number"] for c in book["chapters"]]
if numbers != list(range(1, len(numbers) + 1)):
# The map addresses chapters by position, so a book that skips or
# repeats one would silently mislabel every option after the gap.
# Better to stop the build than to ship a picker that lies.
raise RuntimeError("%s: chapters are not 1..%d (%s)"
% (book["code"], len(numbers), numbers))
chapters = []
@@ -874,11 +657,6 @@ def render_verses(books, out_dir):
fh.write("\n")
return path, total
# ==========================================================================
# Entry point
# ==========================================================================
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -912,6 +690,5 @@ def main(argv=None):
print("wrote %s/" % args.out.rstrip("/"), file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main())