#!/usr/bin/env python3 """Compile the eBible.org HTML export into the fragments the app serves. One step, source to output. It reads a directory of eBible .htm files -- a flat set of per-chapter pages, ``JHN03.htm``, plus a site index -- and writes books/GEN.html and so on, plus books/index.html. There used to be two scripts and a 15 MB bible.json between them. The JSON was worth having when the corpus was going to be queried; nothing queries it, so it was a second representation to keep in step for no one's benefit. The parse tree now lives for as long as one process run. The two halves are still clearly separated below, because they are genuinely different jobs and the seam is where a second output format would attach: READING eBible's generated markup is a flat sequence of ``
`` blocks whose class is the USFM paragraph marker (``p``, ``q2``, ``s``, ...), with verse numbers, footnote markers and character styles as inline spans. This half turns that into a tree of blocks and inline nodes. Navigation and the copyright footer are dropped; footnotes are kept. WRITING That tree becomes one semantic fragment per book: a and one root element carrying data-template, not a document. The shell around it is assembled from the nunjucks templates in assets/templates/ -- by the service worker, or by boot.html when there can never be one -- so chrome changes never require running this. The markup carries structure and no presentation: every USFM distinction survives as a class or data attribute, so the whole corpus can be restyled from the stylesheet without being regenerated. This is the only build step in the project, and books/ is all it owns. Everything hand-written lives in assets/ and is served straight from there. Grouping worth knowing about: consecutive poetry lines are wrapped in a <div class="poetry"> that a blank-line block closes, so stanzas are real elements rather than spacing; consecutive list items become one <ul>. python3 utils/compile_bible.py source -o books """ from __future__ import annotations import argparse import html import os import re import sys from html.parser import HTMLParser # ========================================================================== # READING -- eBible .htm into a tree of blocks and inline nodes # ========================================================================== # Chapter files are a three-character book code plus a zero-padded chapter # number (three digits for Psalms). CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$") # Links between chapters: "JOB05.htm#V13", "JHN03.htm", or a bare "#V7". HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$") # Block-level div classes -> semantic block type. BLOCK_TYPES = { "p": "paragraph", # normal prose paragraph "m": "paragraph", # continuation, no first-line indent "nb": "paragraph", # no break with previous "pi": "paragraph", # indented "mi": "paragraph", # indented, no first-line indent "pc": "paragraph", # centered "ip": "paragraph", # introduction paragraph "q": "line", # poetry line "q2": "line", "q3": "line", "qs": "line", # Selah "b": "break", # blank line between stanzas "d": "title", # descriptive title (psalm superscription, acrostic letter) "s": "heading", # section heading "ms": "heading", # major section heading ("BOOK 3") "is": "heading", # introduction heading "sp": "speaker", # speaker label (Song of Solomon) "li": "item", # list item "ili": "item", # introduction list item (glossary entries) "mt": "booktitle", "mt2": "booktitle", "mt3": "booktitle", } # Nesting depth for poetry lines and indented paragraphs. INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1} # Heading / title rank. LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2} # Character-style spans -> inline node type. ``None`` means unwrap the span and # keep its children inline (``ft`` is just "footnote body text"). INLINE = { "wj": "wj", # words of Jesus "k": "keyword", # glossary keyword "bk": "bookref", # book name cited in a note "fq": "fq", # the quoted words the footnote is about "fqa": "fqa", # an alternative rendering "fl": "fl", # footnote label "fine": "fine", "xt": "xt", # cross-reference target "ft": None, } # Divs whose entire subtree is navigation or licensing, not content. DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"} def parse_href(href): """Turn an intra-corpus link into a structured reference, or None.""" m = HREF_RE.match(href or "") if not m or not any(m.groups()): return None book, chapter, verse = m.groups() ref = {} if book: ref["book"] = book ref["chapter"] = int(chapter) if verse: ref["verse"] = int(verse) return ref or None # -------------------------------------------------------------------------- # Whitespace # -------------------------------------------------------------------------- def normalize(nodes): """Collapse whitespace across the whole inline tree, in document order. Runs of whitespace become a single space and spaces are not duplicated across element boundaries, so "word</span> <span>next" keeps exactly one. """ state = {"prev_space": True} # True at the start suppresses a leading space def walk(lst): out = [] for node in lst: if node["t"] == "text": text = re.sub(r"[\s ]+", " ", node["v"]) if state["prev_space"]: text = text.lstrip(" ") if text: state["prev_space"] = text.endswith(" ") out.append({"t": "text", "v": text}) elif "c" in node: node["c"] = walk(node["c"]) if node["c"]: out.append(node) else: state["prev_space"] = False out.append(node) return out result = walk(nodes) rstrip_last(result) return result def rstrip_last(nodes): """Drop the trailing space on a tree's final text node. Only a genuinely trailing node is stripped. A space before a block's closing verse or note marker separates it from the preceding word and has to survive, so the walk stops at the last node rather than looking past markers for something to strip. """ if not nodes: return False node = nodes[-1] if node["t"] == "text": node["v"] = node["v"].rstrip() if not node["v"]: nodes.pop() return rstrip_last(nodes) return True if "c" in node: return rstrip_last(node["c"]) return False def plain_text(nodes, skip=("noteref",)): """Flatten an inline tree to readable text, dropping note markers.""" parts = [] for node in nodes: if node["t"] in skip: continue if node["t"] == "text": parts.append(node["v"]) elif node["t"] == "br": parts.append(" ") elif "c" in node: parts.append(plain_text(node["c"], skip)) return "".join(parts) # -------------------------------------------------------------------------- # Chapter parser # -------------------------------------------------------------------------- class ChapterParser(HTMLParser): """Parse one chapter file into blocks + notes. Content is appended to ``self.sink``, the inline-node list currently being filled. A stack of frames records what to restore (and what to finish) as each element closes; ``sink is None`` discards text, which is how navigation and the copyright footer are dropped. """ def __init__(self): super().__init__(convert_charrefs=True) self.blocks = [] self.notes = [] self.titles = [] # mt/mt2/mt3, hoisted to the book self.chapter_label = None self.is_toc = False # a chapter-0 navigation page self.in_main = False self.stack = [] self.sink = None # -- frame helpers ---------------------------------------------------- def _push(self, tag, sink, on_close=None, **extra): frame = {"tag": tag, "prev": self.sink, "close": on_close} frame.update(extra) self.stack.append(frame) self.sink = sink return frame def _emit(self, node): if self.sink is not None: self.sink.append(node) # -- HTMLParser hooks ------------------------------------------------- def handle_starttag(self, tag, attrs): attrs = dict(attrs) classes = (attrs.get("class") or "").split() cls = classes[0] if classes else "" if tag in ("br", "hr", "img", "meta", "link", "input"): if tag == "br": self._emit({"t": "br"}) return if not self.in_main: # Everything before <div class="main"> is <head> and top nav. if tag == "div" and "main" in classes: self.in_main = True self._push(tag, None) return if cls in DROP_CLASSES: if cls in ("toc", "toc1", "toc2"): self.is_toc = True self._push(tag, None) return if tag == "div" and cls == "chapterlabel": buf = [] self._push(tag, buf, self._close_chapter_label, nodes=buf) return if tag == "div" and cls in BLOCK_TYPES: block = {"type": BLOCK_TYPES[cls], "style": cls, "content": []} if cls in INDENT: block["indent"] = INDENT[cls] if cls in LEVEL: block["level"] = LEVEL[cls] self._push(tag, block["content"], self._close_block, block=block) return if tag == "p" and cls in ("f", "x"): note = { "id": attrs.get("id"), "kind": "footnote" if cls == "f" else "xref", "marker": None, "ref": None, "verse": None, "content": [], } self._push(tag, note["content"], self._close_note, note=note) return if tag == "span" and "verse" in classes: buf = [] self._push(tag, buf, self._close_verse, nodes=buf) return if tag == "span" and cls == "notemark": # The note's own marker inside the footnote list. buf = [] self._push(tag, buf, self._close_note_marker, nodes=buf) return if tag == "a" and cls == "notemark": # An in-text marker linking down to the note; its nested # <span class="popup"> duplicates the note body and is dropped. node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")} self._emit(node) buf = [] self._push(tag, buf, self._close_noteref, node=node, nodes=buf) return if tag == "a" and cls == "notebackref": buf = [] self._push(tag, buf, self._close_backref, nodes=buf) return if tag == "span" and cls in INLINE: kind = INLINE[cls] if kind is None: self._push(tag, self.sink) # unwrap else: node = {"t": kind, "c": []} self._emit(node) self._push(tag, node["c"]) return if tag == "a": ref = parse_href(attrs.get("href")) if ref is not None and self.sink is not None: node = {"t": "ref", "c": []} node.update(ref) self._emit(node) self._push(tag, node["c"]) else: self._push(tag, self.sink) return # Anything else (nested spans, stray markup) is transparent. self._push(tag, self.sink) def handle_endtag(self, tag): if not self.in_main or not self.stack: return for i in range(len(self.stack) - 1, -1, -1): if self.stack[i]["tag"] == tag: break else: return # unmatched close tag; ignore while len(self.stack) > i: frame = self.stack.pop() self.sink = frame["prev"] if frame["close"]: frame["close"](frame) if tag == "div" and not self.stack: self.in_main = False # <div class="main"> closed def handle_data(self, data): if self.sink is not None and data: self.sink.append({"t": "text", "v": data}) # -- frame close handlers --------------------------------------------- def _close_chapter_label(self, frame): self.chapter_label = plain_text(frame["nodes"]).strip() def _close_block(self, frame): block = frame["block"] block["content"] = normalize(block["content"]) if block["type"] == "booktitle": if block["content"]: self.titles.append({ "style": block["style"], "level": block.get("level"), "text": plain_text(block["content"]), }) return if block["type"] == "break": block.pop("content") self.blocks.append(block) return if block["content"]: self.blocks.append(block) def _close_note(self, frame): note = frame["note"] note["content"] = normalize(note["content"]) note["text"] = plain_text(note["content"]) self.notes.append(note) def _close_verse(self, frame): label = plain_text(frame["nodes"]).replace(" ", " ").strip() node = {"t": "verse", "label": label} m = re.match(r"^(\d+)(?:[-–](\d+))?$", label) if m: node["n"] = int(m.group(1)) if m.group(2): node["end"] = int(m.group(2)) self._emit(node) def _close_note_marker(self, frame): marker = plain_text(frame["nodes"]).strip() for f in reversed(self.stack): if "note" in f: f["note"]["marker"] = marker return def _close_noteref(self, frame): frame["node"]["marker"] = plain_text(frame["nodes"]).strip() def _close_backref(self, frame): text = plain_text(frame["nodes"]).strip() for f in reversed(self.stack): if "note" in f: f["note"]["ref"] = text m = re.search(r":(\d+)", text) if m: f["note"]["verse"] = int(m.group(1)) return # -------------------------------------------------------------------------- # Corpus # -------------------------------------------------------------------------- class IndexParser(HTMLParser): """Read book order, display names and canon section from index.htm.""" # Link class on the site index marks which part of the canon a book is in. SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"} def __init__(self): super().__init__(convert_charrefs=True) self.books = [] # [{code, name, section, order}] self.title = None self._link = None self._in_title = False def handle_starttag(self, tag, attrs): attrs = dict(attrs) if tag == "title": self._in_title = True elif tag == "a": m = CHAPTER_RE.match(os.path.basename(attrs.get("href") or "")) section = self.SECTIONS.get((attrs.get("class") or "").strip()) if m and section: self._link = { "code": m.group(1), "section": section, "order": len(self.books) + 1, "name": "", } def handle_endtag(self, tag): if tag == "title": self._in_title = False elif tag == "a" and self._link: self._link["name"] = self._link["name"].strip() self.books.append(self._link) self._link = None def handle_data(self, data): if self._in_title and self.title is None: self.title = data.strip() elif self._link is not None: self._link["name"] += data def parse_chapter(path): parser = ChapterParser() with open(path, encoding="utf-8-sig") as fh: parser.feed(fh.read()) parser.close() return parser def extract(source_dir): index_path = os.path.join(source_dir, "index.htm") index = IndexParser() with open(index_path, encoding="utf-8-sig") as fh: index.feed(fh.read()) meta = {b["code"]: b for b in index.books} # Group chapter files by book code. by_book = {} for name in sorted(os.listdir(source_dir)): m = CHAPTER_RE.match(name) if m: by_book.setdefault(m.group(1), []).append((int(m.group(2)), name)) books = [] skipped = [] for code in sorted(by_book, key=lambda c: (meta.get(c, {}).get("order", 999), c)): info = meta.get(code, {"code": code, "name": code, "section": "other", "order": 999}) book = { "code": code, "name": info["name"], "section": info["section"], "titles": [], "chapters": [], } for number, name in sorted(by_book[code]): parsed = parse_chapter(os.path.join(source_dir, name)) if parsed.is_toc or not parsed.blocks: # Chapter-0 files are per-book tables of contents, not text. skipped.append(name) continue if parsed.titles and not book["titles"]: book["titles"] = parsed.titles label = parsed.chapter_label chapter = { "number": int(label) if label and label.isdigit() else number, "blocks": parsed.blocks, "notes": parsed.notes, } book["chapters"].append(chapter) if book["chapters"]: books.append(book) return { "translation": { "id": "engwebu", "name": index.title or "World English Bible Updated", "license": "Public Domain", "source": "eBible.org", }, "books": books, }, skipped # ========================================================================== # WRITING -- that tree into one HTML fragment per book # ========================================================================== # Inline node type -> (tag, class). ``cite`` and ``dfn`` carry real meaning # here: a book name cited in a note, and a glossary term at its definition. INLINE_TAGS = { "wj": ("span", "wj"), # words of Jesus "keyword": ("dfn", "k"), # glossary headword "bookref": ("cite", "bk"), # book title cited in a note "fq": ("span", "fq"), # the words a footnote is about "fqa": ("span", "fqa"), # an alternative rendering "fl": ("span", "fl"), # footnote label "fine": ("span", "fine"), "xt": ("span", "xt"), # cross-reference target } # Block type -> tag. Headings sit at h3 so the outline never skips a level: # h1 book, h2 chapter, h3 section. BLOCK_TAGS = { "paragraph": "p", "line": "p", "heading": "h3", "title": "p", "speaker": "p", "item": "li", } def esc(text): return html.escape(text, quote=False) def verse_id(chapter, label): """Page-unique verse anchor, e.g. v23_1 (or v11_15-16 for a range).""" return "v%d_%s" % (chapter, label) def ref_href(node, book, chapter): """Link an intra-corpus reference; whole books are single pages.""" target_book = node.get("book") or book target_ch = node.get("chapter", chapter) verse = node.get("verse") base = "" if target_book == book else target_book if verse: return "%s#%s" % (base, verse_id(target_ch, verse)) return "%s#ch%d" % (base, target_ch) def render_inline(nodes, book, chapter, out): for node in nodes: kind = node["t"] if kind == "text": out.append(esc(node["v"])) elif kind == "verse": # Trailing space keeps the number off the text before any CSS # lands; style .v with padding and it stays correct either way. out.append('<span class="v" id="%s">%s</span> ' % ( verse_id(chapter, node["label"]), esc(node["label"]))) elif kind == "noteref": n = node["id"].replace("FN", "") out.append('<a class="noteref" id="fnref%s_%s" href="#fn%s_%s">' "<sup>%s</sup></a>" % (chapter, n, chapter, n, esc(caller(node.get("marker"))))) elif kind == "br": out.append("<br>") elif kind == "ref": out.append('<a class="ref" href="%s">' % ref_href(node, book, chapter)) render_inline(node.get("c", []), book, chapter, out) out.append("</a>") elif kind in INLINE_TAGS: tag, cls = INLINE_TAGS[kind] out.append('<%s class="%s">' % (tag, cls)) render_inline(node.get("c", []), book, chapter, out) out.append("</%s>" % tag) else: # unknown node type: keep the words, drop the wrapper render_inline(node.get("c", []), book, chapter, out) def render_block(block, book, chapter, out): btype = block["type"] tag = BLOCK_TAGS.get(btype, "p") classes = [block["style"]] attrs = "" if "indent" in block: attrs += ' data-indent="%d"' % block["indent"] if btype == "line": classes.append("line") out.append(" <%s class=\"%s\"%s>" % (tag, " ".join(classes), attrs)) render_inline(block.get("content", []), book, chapter, out) out.append("</%s>\n" % tag) # Footnote callers arrive from the source as * † ‡ § ✡. The star is the only # one of them a browser may decide is an emoji: no serif font carries a glyph # for U+2721, so the fallback chain reaches the colour emoji font and drops a # purple tile into the middle of a verse. U+FE0E is the standardised way to ask # for the text glyph of the same character -- the symbol, not the emoji. There # is no separate text codepoint to switch to; this is the only lever. TEXT_PRESENTATION = "\ufe0e" EMOJI_CAPABLE = "\u2721" def caller(marker): """A footnote marker, asked to render as text rather than as an emoji.""" return "".join(c + TEXT_PRESENTATION if c in EMOJI_CAPABLE else c for c in (marker or "*")) def render_notes(notes, chapter, book, out): if not notes: return out.append(' <section class="footnotes">\n') out.append(' <h4 class="footnotes-title">Notes</h4>\n') out.append(" <ol>\n") for note in notes: n = (note["id"] or "").replace("FN", "") out.append(' <li class="note" id="fn%s_%s" data-kind="%s">' % (chapter, n, note["kind"])) out.append('<span class="marker">%s</span> ' % esc(caller(note.get("marker")))) if note.get("ref"): out.append('<a class="backref" href="#fnref%s_%s">%s</a> ' % (chapter, n, esc(note["ref"]))) out.append('<span class="ft">') render_inline(note["content"], book, chapter, out) out.append("</span></li>\n") out.append(" </ol>\n </section>\n") def render_chapter(chapter, book, out): num = chapter["number"] out.append(' <section class="chapter" id="ch%d" data-chapter="%d">\n' % (num, num)) out.append(' <h2 class="chapter-number">%d</h2>\n' % num) group = None # "poetry" | "list" | None def close_group(): nonlocal group if group == "poetry": out.append(" </div>\n") elif group == "list": out.append(" </ul>\n") group = None for block in chapter["blocks"]: btype = block["type"] if btype == "break": close_group() # a blank line ends the stanza continue want = {"line": "poetry", "item": "list"}.get(btype) if want != group: close_group() if want == "poetry": out.append(' <div class="poetry">\n') elif want == "list": out.append(' <ul class="list">\n') group = want render_block(block, book, num, out) close_group() render_notes(chapter["notes"], num, book, out) out.append(" </section>\n") def render_book(book, translation, out_dir): code, name = book["code"], book["name"] out = [] title = "%s — %s" % (name, translation["name"]) # A fragment: the title the shell will use, then one root element whose # data-template names the template that wraps it. out.append("<title>%s\n" % esc(title)) out.append('
\n' % (code, code, esc(name), book["section"])) # Header: the full multi-line title if the source had one. out.append('
\n') out.append("

") if book["titles"]: # Separated by a space, not run together. The spans record what USFM # called them -- mt2 and mt3 were lines above and below the main title # -- but they are set as one line here, and without the space between # them Genesis reads "The First Book of Moses,Commonly CalledGenesis". # The markup still says what each part is, so stacking them again is a # stylesheet away. out.append(" ".join('%s' % (part["style"], esc(part["text"])) for part in book["titles"])) else: out.append(esc(name)) out.append("

\n") # The title and nothing else. Both of the things that used to sit under it # are in the navbar now, where they are in reach from anywhere in the book # rather than only from the top of it: the chapter list became the # reference picker, which reads its chapters and verses straight off this # markup, and "All books" is what the navbar brand already does. In Psalms # the chapter list alone was a hundred and fifty links, carried in the # fragment and in every device's offline copy. out.append("
\n") out.append('
\n') for chapter in book["chapters"]: render_chapter(chapter, code, out) out.append("
\n") out.append('

%s. %s.

\n' % (esc(translation["name"]), esc(translation["license"]))) out.append("
\n") path = os.path.join(out_dir, "%s.html" % code) with open(path, "w", encoding="utf-8") as fh: fh.write("".join(out)) return path SECTION_LABELS = [ ("ot", "Old Testament"), ("nt", "New Testament"), ("deuterocanon", "Deuterocanon"), ("other", "Other"), ] def render_index(books, translation, out_dir): out = ["%s\n" % esc(translation["name"]), '
\n'] for key, label in SECTION_LABELS: group = [b for b in books if b["section"] == key] if not group: continue out.append('
\n' % key) out.append("

%s

\n
    \n" % esc(label)) for book in group: out.append('
  • %s ' '%d
    • \n' % (book["code"], esc(book["name"]), len(book["chapters"]))) out.append("
\n
\n") out.append(' \n' % (esc(translation["name"]), esc(translation["license"]))) out.append("
\n") path = os.path.join(out_dir, "index.html") with open(path, "w", encoding="utf-8") as fh: fh.write("".join(out)) return path # ========================================================================== # Entry point # ========================================================================== def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("source", nargs="?", default="source", help="directory of eBible .htm files (default: source)") ap.add_argument("-o", "--out", default="books", help="output directory (default: books)") args = ap.parse_args(argv) data, skipped = extract(args.source) os.makedirs(args.out, exist_ok=True) total = 0 for book in data["books"]: total += os.path.getsize(render_book(book, data["translation"], args.out)) render_index(data["books"], data["translation"], args.out) chapters = sum(len(b["chapters"]) for b in data["books"]) notes = sum(len(c["notes"]) for b in data["books"] for c in b["chapters"]) print("read %s/" % args.source.rstrip("/"), file=sys.stderr) print("books %d" % len(data["books"]), file=sys.stderr) print("chapters %d" % chapters, file=sys.stderr) print("notes %d" % notes, file=sys.stderr) print("skipped %d (contents pages)" % len(skipped), file=sys.stderr) print("html %.1f MB total, %.0f KB average" % (total / 1e6, total / len(data["books"]) / 1e3), file=sys.stderr) print("wrote %s/" % args.out.rstrip("/"), file=sys.stderr) return 0 if __name__ == "__main__": sys.exit(main())