919 lines
34 KiB
Python
919 lines
34 KiB
Python
#!/usr/bin/env python3
|
||
"""Compile the eBible.org HTML export into the fragments the app serves.
|
||
|
||
One step, source to output. It reads a directory of eBible .htm files -- a flat
|
||
set of per-chapter pages, ``JHN03.htm``, plus a site index -- and writes
|
||
bible/GEN.html and so on, plus bible/index.html.
|
||
|
||
There used to be two scripts and a 15 MB bible.json between them. The JSON was
|
||
worth having when the corpus was going to be queried; nothing queries it, so
|
||
it was a second representation to keep in step for no one's benefit. The parse
|
||
tree now lives for as long as one process run.
|
||
|
||
The two halves are still clearly separated below, because they are genuinely
|
||
different jobs and the seam is where a second output format would attach:
|
||
|
||
READING eBible's generated markup is a flat sequence of
|
||
``<div class='...'>`` blocks whose class is the USFM paragraph
|
||
marker (``p``, ``q2``, ``s``, ...), with verse numbers, footnote
|
||
markers and character styles as inline spans. This half turns
|
||
that into a tree of blocks and inline nodes. Navigation and the
|
||
copyright footer are dropped; footnotes are kept.
|
||
|
||
WRITING That tree becomes one semantic fragment per book: a <title> and
|
||
one root element carrying data-template, not a document. The
|
||
shell around it is assembled from the nunjucks templates in
|
||
assets/templates/ -- by the service worker, or by boot.html when
|
||
there can never be one -- so chrome changes never require running
|
||
this. The markup carries structure and no presentation: every
|
||
USFM distinction survives as a class or data attribute, so the
|
||
whole corpus can be restyled from the stylesheet without being
|
||
regenerated.
|
||
|
||
This is the only build step in the project, and bible/ is all it owns.
|
||
Its output is not committed: bible/ is in .gitignore, so a deploy runs this.
|
||
Everything hand-written lives in assets/ and is served straight from there.
|
||
|
||
Grouping worth knowing about: consecutive poetry lines are wrapped in a
|
||
<div class="poetry"> that a blank-line block closes, so stanzas are real
|
||
elements rather than spacing; consecutive list items become one <ul>.
|
||
|
||
python3 utils/compile_bible.py source -o bible
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import html
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from html.parser import HTMLParser
|
||
|
||
|
||
# ==========================================================================
|
||
# READING -- eBible .htm into a tree of blocks and inline nodes
|
||
# ==========================================================================
|
||
|
||
# Chapter files are a three-character book code plus a zero-padded chapter
|
||
# number (three digits for Psalms).
|
||
CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$")
|
||
|
||
# Links between chapters: "JOB05.htm#V13", "JHN03.htm", or a bare "#V7".
|
||
HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$")
|
||
|
||
# Block-level div classes -> semantic block type.
|
||
BLOCK_TYPES = {
|
||
"p": "paragraph", # normal prose paragraph
|
||
"m": "paragraph", # continuation, no first-line indent
|
||
"nb": "paragraph", # no break with previous
|
||
"pi": "paragraph", # indented
|
||
"mi": "paragraph", # indented, no first-line indent
|
||
"pc": "paragraph", # centered
|
||
"ip": "paragraph", # introduction paragraph
|
||
"q": "line", # poetry line
|
||
"q2": "line",
|
||
"q3": "line",
|
||
"qs": "line", # Selah
|
||
"b": "break", # blank line between stanzas
|
||
"d": "title", # descriptive title (psalm superscription, acrostic letter)
|
||
"s": "heading", # section heading
|
||
"ms": "heading", # major section heading ("BOOK 3")
|
||
"is": "heading", # introduction heading
|
||
"sp": "speaker", # speaker label (Song of Solomon)
|
||
"li": "item", # list item
|
||
"ili": "item", # introduction list item (glossary entries)
|
||
"mt": "booktitle",
|
||
"mt2": "booktitle",
|
||
"mt3": "booktitle",
|
||
}
|
||
|
||
# Nesting depth for poetry lines and indented paragraphs.
|
||
INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1}
|
||
|
||
# Heading / title rank.
|
||
LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2}
|
||
|
||
# Character-style spans -> inline node type. ``None`` means unwrap the span and
|
||
# keep its children inline (``ft`` is just "footnote body text").
|
||
INLINE = {
|
||
"wj": "wj", # words of Jesus
|
||
"k": "keyword", # glossary keyword
|
||
"bk": "bookref", # book name cited in a note
|
||
"fq": "fq", # the quoted words the footnote is about
|
||
"fqa": "fqa", # an alternative rendering
|
||
"fl": "fl", # footnote label
|
||
"fine": "fine",
|
||
"xt": "xt", # cross-reference target
|
||
"ft": None,
|
||
}
|
||
|
||
# Divs whose entire subtree is navigation or licensing, not content.
|
||
DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"}
|
||
|
||
|
||
def parse_href(href):
|
||
"""Turn an intra-corpus link into a structured reference, or None."""
|
||
m = HREF_RE.match(href or "")
|
||
if not m or not any(m.groups()):
|
||
return None
|
||
book, chapter, verse = m.groups()
|
||
ref = {}
|
||
if book:
|
||
ref["book"] = book
|
||
ref["chapter"] = int(chapter)
|
||
if verse:
|
||
ref["verse"] = int(verse)
|
||
return ref or None
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Whitespace
|
||
# --------------------------------------------------------------------------
|
||
|
||
def normalize(nodes):
|
||
"""Collapse whitespace across the whole inline tree, in document order.
|
||
|
||
Runs of whitespace become a single space and spaces are not duplicated
|
||
across element boundaries, so "word</span> <span>next" keeps exactly one.
|
||
"""
|
||
state = {"prev_space": True} # True at the start suppresses a leading space
|
||
|
||
def walk(lst):
|
||
out = []
|
||
for node in lst:
|
||
if node["t"] == "text":
|
||
text = re.sub(r"[\s ]+", " ", node["v"])
|
||
if state["prev_space"]:
|
||
text = text.lstrip(" ")
|
||
if text:
|
||
state["prev_space"] = text.endswith(" ")
|
||
out.append({"t": "text", "v": text})
|
||
elif "c" in node:
|
||
node["c"] = walk(node["c"])
|
||
if node["c"]:
|
||
out.append(node)
|
||
else:
|
||
state["prev_space"] = False
|
||
out.append(node)
|
||
return out
|
||
|
||
result = walk(nodes)
|
||
rstrip_last(result)
|
||
return result
|
||
|
||
|
||
def rstrip_last(nodes):
|
||
"""Drop the trailing space on a tree's final text node.
|
||
|
||
Only a genuinely trailing node is stripped. A space before a block's
|
||
closing verse or note marker separates it from the preceding word and
|
||
has to survive, so the walk stops at the last node rather than looking
|
||
past markers for something to strip.
|
||
"""
|
||
if not nodes:
|
||
return False
|
||
node = nodes[-1]
|
||
if node["t"] == "text":
|
||
node["v"] = node["v"].rstrip()
|
||
if not node["v"]:
|
||
nodes.pop()
|
||
return rstrip_last(nodes)
|
||
return True
|
||
if "c" in node:
|
||
return rstrip_last(node["c"])
|
||
return False
|
||
|
||
|
||
def plain_text(nodes, skip=("noteref",)):
|
||
"""Flatten an inline tree to readable text, dropping note markers."""
|
||
parts = []
|
||
for node in nodes:
|
||
if node["t"] in skip:
|
||
continue
|
||
if node["t"] == "text":
|
||
parts.append(node["v"])
|
||
elif node["t"] == "br":
|
||
parts.append(" ")
|
||
elif "c" in node:
|
||
parts.append(plain_text(node["c"], skip))
|
||
return "".join(parts)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Chapter parser
|
||
# --------------------------------------------------------------------------
|
||
|
||
class ChapterParser(HTMLParser):
|
||
"""Parse one chapter file into blocks + notes.
|
||
|
||
Content is appended to ``self.sink``, the inline-node list currently being
|
||
filled. A stack of frames records what to restore (and what to finish) as
|
||
each element closes; ``sink is None`` discards text, which is how
|
||
navigation and the copyright footer are dropped.
|
||
"""
|
||
|
||
def __init__(self):
|
||
super().__init__(convert_charrefs=True)
|
||
self.blocks = []
|
||
self.notes = []
|
||
self.titles = [] # mt/mt2/mt3, hoisted to the book
|
||
self.chapter_label = None
|
||
self.is_toc = False # a chapter-0 navigation page
|
||
|
||
self.in_main = False
|
||
self.stack = []
|
||
self.sink = None
|
||
|
||
# -- frame helpers ----------------------------------------------------
|
||
|
||
def _push(self, tag, sink, on_close=None, **extra):
|
||
frame = {"tag": tag, "prev": self.sink, "close": on_close}
|
||
frame.update(extra)
|
||
self.stack.append(frame)
|
||
self.sink = sink
|
||
return frame
|
||
|
||
def _emit(self, node):
|
||
if self.sink is not None:
|
||
self.sink.append(node)
|
||
|
||
# -- HTMLParser hooks -------------------------------------------------
|
||
|
||
def handle_starttag(self, tag, attrs):
|
||
attrs = dict(attrs)
|
||
classes = (attrs.get("class") or "").split()
|
||
cls = classes[0] if classes else ""
|
||
|
||
if tag in ("br", "hr", "img", "meta", "link", "input"):
|
||
if tag == "br":
|
||
self._emit({"t": "br"})
|
||
return
|
||
|
||
if not self.in_main:
|
||
# Everything before <div class="main"> is <head> and top nav.
|
||
if tag == "div" and "main" in classes:
|
||
self.in_main = True
|
||
self._push(tag, None)
|
||
return
|
||
|
||
if cls in DROP_CLASSES:
|
||
if cls in ("toc", "toc1", "toc2"):
|
||
self.is_toc = True
|
||
self._push(tag, None)
|
||
return
|
||
|
||
if tag == "div" and cls == "chapterlabel":
|
||
buf = []
|
||
self._push(tag, buf, self._close_chapter_label, nodes=buf)
|
||
return
|
||
|
||
if tag == "div" and cls in BLOCK_TYPES:
|
||
block = {"type": BLOCK_TYPES[cls], "style": cls, "content": []}
|
||
if cls in INDENT:
|
||
block["indent"] = INDENT[cls]
|
||
if cls in LEVEL:
|
||
block["level"] = LEVEL[cls]
|
||
self._push(tag, block["content"], self._close_block, block=block)
|
||
return
|
||
|
||
if tag == "p" and cls in ("f", "x"):
|
||
note = {
|
||
"id": attrs.get("id"),
|
||
"kind": "footnote" if cls == "f" else "xref",
|
||
"marker": None,
|
||
"ref": None,
|
||
"verse": None,
|
||
"content": [],
|
||
}
|
||
self._push(tag, note["content"], self._close_note, note=note)
|
||
return
|
||
|
||
if tag == "span" and "verse" in classes:
|
||
buf = []
|
||
self._push(tag, buf, self._close_verse, nodes=buf)
|
||
return
|
||
|
||
if tag == "span" and cls == "notemark":
|
||
# The note's own marker inside the footnote list.
|
||
buf = []
|
||
self._push(tag, buf, self._close_note_marker, nodes=buf)
|
||
return
|
||
|
||
if tag == "a" and cls == "notemark":
|
||
# An in-text marker linking down to the note; its nested
|
||
# <span class="popup"> duplicates the note body and is dropped.
|
||
node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")}
|
||
self._emit(node)
|
||
buf = []
|
||
self._push(tag, buf, self._close_noteref, node=node, nodes=buf)
|
||
return
|
||
|
||
if tag == "a" and cls == "notebackref":
|
||
buf = []
|
||
self._push(tag, buf, self._close_backref, nodes=buf)
|
||
return
|
||
|
||
if tag == "span" and cls in INLINE:
|
||
kind = INLINE[cls]
|
||
if kind is None:
|
||
self._push(tag, self.sink) # unwrap
|
||
else:
|
||
node = {"t": kind, "c": []}
|
||
self._emit(node)
|
||
self._push(tag, node["c"])
|
||
return
|
||
|
||
if tag == "a":
|
||
ref = parse_href(attrs.get("href"))
|
||
if ref is not None and self.sink is not None:
|
||
node = {"t": "ref", "c": []}
|
||
node.update(ref)
|
||
self._emit(node)
|
||
self._push(tag, node["c"])
|
||
else:
|
||
self._push(tag, self.sink)
|
||
return
|
||
|
||
# Anything else (nested spans, stray markup) is transparent.
|
||
self._push(tag, self.sink)
|
||
|
||
def handle_endtag(self, tag):
|
||
if not self.in_main or not self.stack:
|
||
return
|
||
for i in range(len(self.stack) - 1, -1, -1):
|
||
if self.stack[i]["tag"] == tag:
|
||
break
|
||
else:
|
||
return # unmatched close tag; ignore
|
||
while len(self.stack) > i:
|
||
frame = self.stack.pop()
|
||
self.sink = frame["prev"]
|
||
if frame["close"]:
|
||
frame["close"](frame)
|
||
if tag == "div" and not self.stack:
|
||
self.in_main = False # <div class="main"> closed
|
||
|
||
def handle_data(self, data):
|
||
if self.sink is not None and data:
|
||
self.sink.append({"t": "text", "v": data})
|
||
|
||
# -- frame close handlers ---------------------------------------------
|
||
|
||
def _close_chapter_label(self, frame):
|
||
self.chapter_label = plain_text(frame["nodes"]).strip()
|
||
|
||
def _close_block(self, frame):
|
||
block = frame["block"]
|
||
block["content"] = normalize(block["content"])
|
||
if block["type"] == "booktitle":
|
||
if block["content"]:
|
||
self.titles.append({
|
||
"style": block["style"],
|
||
"level": block.get("level"),
|
||
"text": plain_text(block["content"]),
|
||
})
|
||
return
|
||
if block["type"] == "break":
|
||
block.pop("content")
|
||
self.blocks.append(block)
|
||
return
|
||
if block["content"]:
|
||
self.blocks.append(block)
|
||
|
||
def _close_note(self, frame):
|
||
note = frame["note"]
|
||
note["content"] = normalize(note["content"])
|
||
note["text"] = plain_text(note["content"])
|
||
self.notes.append(note)
|
||
|
||
def _close_verse(self, frame):
|
||
label = plain_text(frame["nodes"]).replace(" ", " ").strip()
|
||
node = {"t": "verse", "label": label}
|
||
m = re.match(r"^(\d+)(?:[-–](\d+))?$", label)
|
||
if m:
|
||
node["n"] = int(m.group(1))
|
||
if m.group(2):
|
||
node["end"] = int(m.group(2))
|
||
self._emit(node)
|
||
|
||
def _close_note_marker(self, frame):
|
||
marker = plain_text(frame["nodes"]).strip()
|
||
for f in reversed(self.stack):
|
||
if "note" in f:
|
||
f["note"]["marker"] = marker
|
||
return
|
||
|
||
def _close_noteref(self, frame):
|
||
frame["node"]["marker"] = plain_text(frame["nodes"]).strip()
|
||
|
||
def _close_backref(self, frame):
|
||
text = plain_text(frame["nodes"]).strip()
|
||
for f in reversed(self.stack):
|
||
if "note" in f:
|
||
f["note"]["ref"] = text
|
||
m = re.search(r":(\d+)", text)
|
||
if m:
|
||
f["note"]["verse"] = int(m.group(1))
|
||
return
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Corpus
|
||
# --------------------------------------------------------------------------
|
||
|
||
class IndexParser(HTMLParser):
|
||
"""Read book order, display names and canon section from index.htm."""
|
||
|
||
# Link class on the site index marks which part of the canon a book is in.
|
||
SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"}
|
||
|
||
def __init__(self):
|
||
super().__init__(convert_charrefs=True)
|
||
self.books = [] # [{code, name, section, order}]
|
||
self.title = None
|
||
self._link = None
|
||
self._in_title = False
|
||
|
||
def handle_starttag(self, tag, attrs):
|
||
attrs = dict(attrs)
|
||
if tag == "title":
|
||
self._in_title = True
|
||
elif tag == "a":
|
||
m = CHAPTER_RE.match(os.path.basename(attrs.get("href") or ""))
|
||
section = self.SECTIONS.get((attrs.get("class") or "").strip())
|
||
if m and section:
|
||
self._link = {
|
||
"code": m.group(1),
|
||
"section": section,
|
||
"order": len(self.books) + 1,
|
||
"name": "",
|
||
}
|
||
|
||
def handle_endtag(self, tag):
|
||
if tag == "title":
|
||
self._in_title = False
|
||
elif tag == "a" and self._link:
|
||
self._link["name"] = self._link["name"].strip()
|
||
self.books.append(self._link)
|
||
self._link = None
|
||
|
||
def handle_data(self, data):
|
||
if self._in_title and self.title is None:
|
||
self.title = data.strip()
|
||
elif self._link is not None:
|
||
self._link["name"] += data
|
||
|
||
|
||
def parse_chapter(path):
|
||
parser = ChapterParser()
|
||
with open(path, encoding="utf-8-sig") as fh:
|
||
parser.feed(fh.read())
|
||
parser.close()
|
||
return parser
|
||
|
||
|
||
def extract(source_dir):
|
||
index_path = os.path.join(source_dir, "index.htm")
|
||
index = IndexParser()
|
||
with open(index_path, encoding="utf-8-sig") as fh:
|
||
index.feed(fh.read())
|
||
|
||
meta = {b["code"]: b for b in index.books}
|
||
|
||
# Group chapter files by book code.
|
||
by_book = {}
|
||
for name in sorted(os.listdir(source_dir)):
|
||
m = CHAPTER_RE.match(name)
|
||
if m:
|
||
by_book.setdefault(m.group(1), []).append((int(m.group(2)), name))
|
||
|
||
books = []
|
||
skipped = []
|
||
for code in sorted(by_book, key=lambda c: (meta.get(c, {}).get("order", 999), c)):
|
||
info = meta.get(code, {"code": code, "name": code, "section": "other",
|
||
"order": 999})
|
||
book = {
|
||
"code": code,
|
||
"name": info["name"],
|
||
"section": info["section"],
|
||
"titles": [],
|
||
"chapters": [],
|
||
}
|
||
for number, name in sorted(by_book[code]):
|
||
parsed = parse_chapter(os.path.join(source_dir, name))
|
||
if parsed.is_toc or not parsed.blocks:
|
||
# Chapter-0 files are per-book tables of contents, not text.
|
||
skipped.append(name)
|
||
continue
|
||
if parsed.titles and not book["titles"]:
|
||
book["titles"] = parsed.titles
|
||
label = parsed.chapter_label
|
||
chapter = {
|
||
"number": int(label) if label and label.isdigit() else number,
|
||
"blocks": parsed.blocks,
|
||
"notes": parsed.notes,
|
||
}
|
||
book["chapters"].append(chapter)
|
||
if book["chapters"]:
|
||
books.append(book)
|
||
|
||
return {
|
||
"translation": {
|
||
"id": "engwebu",
|
||
"name": index.title or "World English Bible Updated",
|
||
"license": "Public Domain",
|
||
"source": "eBible.org",
|
||
},
|
||
"books": books,
|
||
}, skipped
|
||
|
||
|
||
# ==========================================================================
|
||
# WRITING -- that tree into one HTML fragment per book
|
||
# ==========================================================================
|
||
|
||
# Inline node type -> (tag, class). ``cite`` and ``dfn`` carry real meaning
|
||
# here: a book name cited in a note, and a glossary term at its definition.
|
||
INLINE_TAGS = {
|
||
"wj": ("span", "wj"), # words of Jesus
|
||
"keyword": ("dfn", "k"), # glossary headword
|
||
"bookref": ("cite", "bk"), # book title cited in a note
|
||
"fq": ("span", "fq"), # the words a footnote is about
|
||
"fqa": ("span", "fqa"), # an alternative rendering
|
||
"fl": ("span", "fl"), # footnote label
|
||
"fine": ("span", "fine"),
|
||
"xt": ("span", "xt"), # cross-reference target
|
||
}
|
||
|
||
# Block type -> tag. Headings sit at h3 so the outline never skips a level:
|
||
# h1 book, h2 chapter, h3 section.
|
||
BLOCK_TAGS = {
|
||
"paragraph": "p",
|
||
"line": "p",
|
||
"heading": "h3",
|
||
"title": "p",
|
||
"speaker": "p",
|
||
"item": "li",
|
||
}
|
||
|
||
|
||
|
||
def esc(text):
|
||
return html.escape(text, quote=False)
|
||
|
||
|
||
def verse_id(chapter, label):
|
||
"""Page-unique verse anchor, e.g. v23_1 (or v11_15-16 for a range)."""
|
||
return "v%d_%s" % (chapter, label)
|
||
|
||
|
||
def ref_href(node, book, chapter):
|
||
"""Link an intra-corpus reference; whole books are single pages."""
|
||
target_book = node.get("book") or book
|
||
target_ch = node.get("chapter", chapter)
|
||
verse = node.get("verse")
|
||
base = "" if target_book == book else target_book
|
||
if verse:
|
||
return "%s#%s" % (base, verse_id(target_ch, verse))
|
||
return "%s#ch%d" % (base, target_ch)
|
||
|
||
|
||
def render_inline(nodes, book, chapter, out):
|
||
for node in nodes:
|
||
kind = node["t"]
|
||
if kind == "text":
|
||
out.append(esc(node["v"]))
|
||
elif kind == "verse":
|
||
# Trailing space keeps the number off the text before any CSS
|
||
# lands; style .v with padding and it stays correct either way.
|
||
#
|
||
# A control, said here rather than by a pass over the page once it
|
||
# has landed. role is what announces it and what earns Reboot's
|
||
# [role=button] pointer cursor, tabindex is what puts it in the tab
|
||
# order, and the label is what a screen reader says in place of a
|
||
# bare number. All three are true of every verse in the corpus and
|
||
# none of them depends on anything only the browser knows, so
|
||
# there is nothing here for a script to work out: app.js used to
|
||
# set them on load, after every htmx swap and after every history
|
||
# restore, walking a whole book each time. The click itself never
|
||
# needed that pass -- it is delegated at #content in base.njk.
|
||
label = esc(node["label"])
|
||
out.append('<span class="v" id="%s" role="button" tabindex="0" '
|
||
'aria-label="Verse %s, options">%s</span> '
|
||
% (verse_id(chapter, node["label"]), label, label))
|
||
elif kind == "noteref":
|
||
n = node["id"].replace("FN", "")
|
||
out.append('<a class="noteref" id="fnref%s_%s" href="#fn%s_%s">'
|
||
"<sup>%s</sup></a>" % (chapter, n, chapter, n,
|
||
esc(caller(node.get("marker")))))
|
||
elif kind == "br":
|
||
out.append("<br>")
|
||
elif kind == "ref":
|
||
out.append('<a class="ref" href="%s">' % ref_href(node, book, chapter))
|
||
render_inline(node.get("c", []), book, chapter, out)
|
||
out.append("</a>")
|
||
elif kind in INLINE_TAGS:
|
||
tag, cls = INLINE_TAGS[kind]
|
||
out.append('<%s class="%s">' % (tag, cls))
|
||
render_inline(node.get("c", []), book, chapter, out)
|
||
out.append("</%s>" % tag)
|
||
else: # unknown node type: keep the words, drop the wrapper
|
||
render_inline(node.get("c", []), book, chapter, out)
|
||
|
||
|
||
def render_block(block, book, chapter, out):
|
||
btype = block["type"]
|
||
tag = BLOCK_TAGS.get(btype, "p")
|
||
classes = [block["style"]]
|
||
attrs = ""
|
||
if "indent" in block:
|
||
attrs += ' data-indent="%d"' % block["indent"]
|
||
if btype == "line":
|
||
classes.append("line")
|
||
out.append(" <%s class=\"%s\"%s>" % (tag, " ".join(classes), attrs))
|
||
render_inline(block.get("content", []), book, chapter, out)
|
||
out.append("</%s>\n" % tag)
|
||
|
||
|
||
# Footnote callers arrive from the source as * † ‡ § ✡. The star is the only
|
||
# one of them a browser may decide is an emoji: no serif font carries a glyph
|
||
# for U+2721, so the fallback chain reaches the colour emoji font and drops a
|
||
# purple tile into the middle of a verse. U+FE0E is the standardised way to ask
|
||
# for the text glyph of the same character -- the symbol, not the emoji. There
|
||
# is no separate text codepoint to switch to; this is the only lever.
|
||
TEXT_PRESENTATION = "\ufe0e"
|
||
EMOJI_CAPABLE = "\u2721"
|
||
|
||
|
||
def caller(marker):
|
||
"""A footnote marker, asked to render as text rather than as an emoji."""
|
||
return "".join(c + TEXT_PRESENTATION if c in EMOJI_CAPABLE else c
|
||
for c in (marker or "*"))
|
||
|
||
|
||
def render_notes(notes, chapter, book, out):
|
||
if not notes:
|
||
return
|
||
out.append(' <section class="footnotes">\n')
|
||
out.append(' <h4 class="footnotes-title">Notes</h4>\n')
|
||
out.append(" <ol>\n")
|
||
for note in notes:
|
||
n = (note["id"] or "").replace("FN", "")
|
||
out.append(' <li class="note" id="fn%s_%s" data-kind="%s">'
|
||
% (chapter, n, note["kind"]))
|
||
out.append('<span class="marker">%s</span> ' % esc(caller(note.get("marker"))))
|
||
if note.get("ref"):
|
||
out.append('<a class="backref" href="#fnref%s_%s">%s</a> '
|
||
% (chapter, n, esc(note["ref"])))
|
||
out.append('<span class="ft">')
|
||
render_inline(note["content"], book, chapter, out)
|
||
out.append("</span></li>\n")
|
||
out.append(" </ol>\n </section>\n")
|
||
|
||
|
||
def render_chapter(chapter, book, out):
|
||
num = chapter["number"]
|
||
out.append(' <section class="chapter" id="ch%d" data-chapter="%d">\n'
|
||
% (num, num))
|
||
out.append(' <h2 class="chapter-number">%d</h2>\n' % num)
|
||
|
||
group = None # "poetry" | "list" | None
|
||
|
||
def close_group():
|
||
nonlocal group
|
||
if group == "poetry":
|
||
out.append(" </div>\n")
|
||
elif group == "list":
|
||
out.append(" </ul>\n")
|
||
group = None
|
||
|
||
for block in chapter["blocks"]:
|
||
btype = block["type"]
|
||
if btype == "break":
|
||
close_group() # a blank line ends the stanza
|
||
continue
|
||
want = {"line": "poetry", "item": "list"}.get(btype)
|
||
if want != group:
|
||
close_group()
|
||
if want == "poetry":
|
||
out.append(' <div class="poetry">\n')
|
||
elif want == "list":
|
||
out.append(' <ul class="list">\n')
|
||
group = want
|
||
render_block(block, book, num, out)
|
||
close_group()
|
||
|
||
render_notes(chapter["notes"], num, book, out)
|
||
out.append(" </section>\n")
|
||
|
||
|
||
def render_book(book, translation, out_dir):
|
||
code, name = book["code"], book["name"]
|
||
out = []
|
||
title = "%s — %s" % (name, translation["name"])
|
||
# A fragment: the title the shell will use, then one root element whose
|
||
# data-template names the template that wraps it.
|
||
out.append("<title>%s</title>\n" % esc(title))
|
||
out.append('<article class="book" data-template="book" id="%s" data-book="%s" '
|
||
'data-name="%s" data-section="%s">\n'
|
||
% (code, code, esc(name), book["section"]))
|
||
|
||
# Header: the full multi-line title if the source had one.
|
||
out.append(' <header class="book-header">\n')
|
||
out.append(" <h1>")
|
||
if book["titles"]:
|
||
# Separated by a space, not run together. The spans record what USFM
|
||
# called them -- mt2 and mt3 were lines above and below the main title
|
||
# -- but they are set as one line here, and without the space between
|
||
# them Genesis reads "The First Book of Moses,Commonly CalledGenesis".
|
||
# The markup still says what each part is, so stacking them again is a
|
||
# stylesheet away.
|
||
out.append(" ".join('<span class="%s">%s</span>'
|
||
% (part["style"], esc(part["text"]))
|
||
for part in book["titles"]))
|
||
else:
|
||
out.append(esc(name))
|
||
out.append("</h1>\n")
|
||
# The title and nothing else. Both of the things that used to sit under it
|
||
# are in the navbar now, where they are in reach from anywhere in the book
|
||
# rather than only from the top of it: the chapter list became the
|
||
# reference picker, which reads its chapters and verses straight off this
|
||
# markup, and "All books" is what the navbar brand already does. In Psalms
|
||
# the chapter list alone was a hundred and fifty links, carried in the
|
||
# fragment and in every device's offline copy.
|
||
out.append(" </header>\n")
|
||
|
||
out.append(' <div class="chapters">\n')
|
||
for chapter in book["chapters"]:
|
||
render_chapter(chapter, code, out)
|
||
out.append(" </div>\n")
|
||
|
||
out.append(' <footer class="colophon"><p>%s. %s.</p></footer>\n'
|
||
% (esc(translation["name"]), esc(translation["license"])))
|
||
out.append("</article>\n")
|
||
|
||
path = os.path.join(out_dir, "%s.html" % code)
|
||
with open(path, "w", encoding="utf-8") as fh:
|
||
fh.write("".join(out))
|
||
return path
|
||
|
||
|
||
SECTION_LABELS = [
|
||
("ot", "Old Testament"),
|
||
("nt", "New Testament"),
|
||
("deuterocanon", "Deuterocanon"),
|
||
("other", "Other"),
|
||
]
|
||
|
||
|
||
def render_index(books, translation, out_dir):
|
||
out = ["<title>%s</title>\n" % esc(translation["name"]),
|
||
'<div class="index" data-template="index">\n']
|
||
for key, label in SECTION_LABELS:
|
||
group = [b for b in books if b["section"] == key]
|
||
if not group:
|
||
continue
|
||
out.append(' <section class="book-group" data-section="%s">\n' % key)
|
||
out.append(" <h2>%s</h2>\n <ul>\n" % esc(label))
|
||
for book in group:
|
||
out.append(' <li><a href="%s">%s</a> '
|
||
'<span class="count">%d</span></li>\n'
|
||
% (book["code"], esc(book["name"]), len(book["chapters"])))
|
||
out.append(" </ul>\n </section>\n")
|
||
out.append(' <footer class="colophon"><p>%s. %s.</p></footer>\n'
|
||
% (esc(translation["name"]), esc(translation["license"])))
|
||
out.append("</div>\n")
|
||
path = os.path.join(out_dir, "index.html")
|
||
with open(path, "w", encoding="utf-8") as fh:
|
||
fh.write("".join(out))
|
||
return path
|
||
|
||
|
||
# ==========================================================================
|
||
# WRITING -- the reference map
|
||
# ==========================================================================
|
||
|
||
# bible/verses.json: every book's chapters, and every chapter's verse labels.
|
||
#
|
||
# The navbar's reference picker is built from this, by the service worker. It
|
||
# used to be built in the browser by reading the book already on the page --
|
||
# the chapters were the sections and a chapter's verses were the anchors inside
|
||
# it -- which was free but only worked where there was a document to read. A
|
||
# worker has no DOM, and the picker now lives in a shell the worker composes
|
||
# before any document exists, so what the picker needs has to be a fact on
|
||
# disk rather than something recovered from markup.
|
||
#
|
||
# Written here, from the same tree the fragments are rendered from, so the two
|
||
# cannot disagree: every option this file produces names an anchor render_book
|
||
# wrote, because both come from the same verse nodes.
|
||
#
|
||
# The encoding is compact on purpose. A chapter whose verses run 1..n is
|
||
# written as the number n; only the eight in the corpus with a gap or a range
|
||
# -- Sirach 26 ends "19-27", 28, 29 -- spell their labels out:
|
||
#
|
||
# {"GEN": [31, 25, 24, ...], "SIR": [..., ["1", ..., "19-27", "28", "29"]]}
|
||
#
|
||
# That is 6 KB for 38,000 verses against 180 KB of spelled-out labels, which is
|
||
# the difference between a file the worker can parse every time it wakes and
|
||
# one it would have to find somewhere to keep.
|
||
#
|
||
# Chapter numbers are not in the file at all: they are the array index plus
|
||
# one. render_verses checks that, rather than assuming it.
|
||
|
||
VERSES_FILE = "verses.json"
|
||
|
||
|
||
def chapter_verses(chapter):
|
||
"""The verse labels of one chapter, in document order.
|
||
|
||
Blocks only. Footnotes hang inside the chapter section too, but a note's
|
||
content never carries a verse node -- and if one ever did, it would be a
|
||
citation rather than a place in the text.
|
||
"""
|
||
labels = []
|
||
|
||
def walk(nodes):
|
||
for node in nodes:
|
||
if node["t"] == "verse":
|
||
labels.append(node["label"])
|
||
elif "c" in node:
|
||
walk(node["c"])
|
||
|
||
for block in chapter["blocks"]:
|
||
walk(block.get("content", []))
|
||
return labels
|
||
|
||
|
||
def render_verses(books, out_dir):
|
||
"""Write the reference map. Returns its path and the verses in it.
|
||
|
||
See the note above for the encoding.
|
||
"""
|
||
out = {}
|
||
total = 0
|
||
for book in books:
|
||
numbers = [c["number"] for c in book["chapters"]]
|
||
if numbers != list(range(1, len(numbers) + 1)):
|
||
# The map addresses chapters by position, so a book that skips or
|
||
# repeats one would silently mislabel every option after the gap.
|
||
# Better to stop the build than to ship a picker that lies.
|
||
raise RuntimeError("%s: chapters are not 1..%d (%s)"
|
||
% (book["code"], len(numbers), numbers))
|
||
chapters = []
|
||
for chapter in book["chapters"]:
|
||
labels = chapter_verses(chapter)
|
||
plain = [str(i + 1) for i in range(len(labels))]
|
||
chapters.append(len(labels) if labels == plain else labels)
|
||
total += len(labels)
|
||
out[book["code"]] = chapters
|
||
|
||
path = os.path.join(out_dir, VERSES_FILE)
|
||
with open(path, "w", encoding="utf-8") as fh:
|
||
json.dump(out, fh, separators=(",", ":"), ensure_ascii=False)
|
||
fh.write("\n")
|
||
return path, total
|
||
|
||
|
||
# ==========================================================================
|
||
# Entry point
|
||
# ==========================================================================
|
||
|
||
def main(argv=None):
|
||
ap = argparse.ArgumentParser(description=__doc__,
|
||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
ap.add_argument("source", nargs="?", default="source",
|
||
help="directory of eBible .htm files (default: source)")
|
||
ap.add_argument("-o", "--out", default="bible",
|
||
help="output directory (default: bible)")
|
||
args = ap.parse_args(argv)
|
||
|
||
data, skipped = extract(args.source)
|
||
os.makedirs(args.out, exist_ok=True)
|
||
|
||
total = 0
|
||
for book in data["books"]:
|
||
total += os.path.getsize(render_book(book, data["translation"], args.out))
|
||
render_index(data["books"], data["translation"], args.out)
|
||
verses, count = render_verses(data["books"], args.out)
|
||
|
||
chapters = sum(len(b["chapters"]) for b in data["books"])
|
||
notes = sum(len(c["notes"]) for b in data["books"] for c in b["chapters"])
|
||
print("read %s/" % args.source.rstrip("/"), file=sys.stderr)
|
||
print("books %d" % len(data["books"]), file=sys.stderr)
|
||
print("chapters %d" % chapters, file=sys.stderr)
|
||
print("notes %d" % notes, file=sys.stderr)
|
||
print("skipped %d (contents pages)" % len(skipped), file=sys.stderr)
|
||
print("html %.1f MB total, %.0f KB average"
|
||
% (total / 1e6, total / len(data["books"]) / 1e3), file=sys.stderr)
|
||
print("verses %d in %s (%.1f KB)"
|
||
% (count, os.path.basename(verses), os.path.getsize(verses) / 1e3),
|
||
file=sys.stderr)
|
||
print("wrote %s/" % args.out.rstrip("/"), file=sys.stderr)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|