"""Compile the eBible.org HTML export into the fragments the app serves.""" from __future__ import annotations import argparse import html import json import os import re import sys from html.parser import HTMLParser CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$") HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$") BLOCK_TYPES = { "p": "paragraph", "m": "paragraph", "nb": "paragraph", "pi": "paragraph", "mi": "paragraph", "pc": "paragraph", "ip": "paragraph", "q": "line", "q2": "line", "q3": "line", "qs": "line", "b": "break", "d": "title", "s": "heading", "ms": "heading", "is": "heading", "sp": "speaker", "li": "item", "ili": "item", "mt": "booktitle", "mt2": "booktitle", "mt3": "booktitle", } INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1} LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2} INLINE = { "wj": "wj", "k": "keyword", "bk": "bookref", "fq": "fq", "fqa": "fqa", "fl": "fl", "fine": "fine", "xt": "xt", "ft": None, } DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"} def parse_href(href): m = HREF_RE.match(href or "") if not m or not any(m.groups()): return None book, chapter, verse = m.groups() ref = {} if book: ref["book"] = book ref["chapter"] = int(chapter) if verse: ref["verse"] = int(verse) return ref or None def normalize(nodes): state = {"prev_space": True} def walk(lst): out = [] for node in lst: if node["t"] == "text": text = re.sub(r"[\s ]+", " ", node["v"]) if state["prev_space"]: text = text.lstrip(" ") if text: state["prev_space"] = text.endswith(" ") out.append({"t": "text", "v": text}) elif "c" in node: node["c"] = walk(node["c"]) if node["c"]: out.append(node) else: state["prev_space"] = False out.append(node) return out result = walk(nodes) rstrip_last(result) return result def rstrip_last(nodes): if not nodes: return False node = nodes[-1] if node["t"] == "text": node["v"] = node["v"].rstrip() if not node["v"]: nodes.pop() return rstrip_last(nodes) return True if "c" in node: return rstrip_last(node["c"]) return False def plain_text(nodes, skip=("noteref",)): parts = [] for node in nodes: if node["t"] in skip: continue if node["t"] == "text": parts.append(node["v"]) elif node["t"] == "br": parts.append(" ") elif "c" in node: parts.append(plain_text(node["c"], skip)) return "".join(parts) class ChapterParser(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) self.blocks = [] self.notes = [] self.titles = [] self.chapter_label = None self.is_toc = False self.in_main = False self.stack = [] self.sink = None def _push(self, tag, sink, on_close=None, **extra): frame = {"tag": tag, "prev": self.sink, "close": on_close} frame.update(extra) self.stack.append(frame) self.sink = sink return frame def _emit(self, node): if self.sink is not None: self.sink.append(node) def handle_starttag(self, tag, attrs): attrs = dict(attrs) classes = (attrs.get("class") or "").split() cls = classes[0] if classes else "" if tag in ("br", "hr", "img", "meta", "link", "input"): if tag == "br": self._emit({"t": "br"}) return if not self.in_main: if tag == "div" and "main" in classes: self.in_main = True self._push(tag, None) return if cls in DROP_CLASSES: if cls in ("toc", "toc1", "toc2"): self.is_toc = True self._push(tag, None) return if tag == "div" and cls == "chapterlabel": buf = [] self._push(tag, buf, self._close_chapter_label, nodes=buf) return if tag == "div" and cls in BLOCK_TYPES: block = {"type": BLOCK_TYPES[cls], "style": cls, "content": []} if cls in INDENT: block["indent"] = INDENT[cls] if cls in LEVEL: block["level"] = LEVEL[cls] self._push(tag, block["content"], self._close_block, block=block) return if tag == "p" and cls in ("f", "x"): note = { "id": attrs.get("id"), "kind": "footnote" if cls == "f" else "xref", "marker": None, "ref": None, "verse": None, "content": [], } self._push(tag, note["content"], self._close_note, note=note) return if tag == "span" and "verse" in classes: buf = [] self._push(tag, buf, self._close_verse, nodes=buf) return if tag == "span" and cls == "notemark": buf = [] self._push(tag, buf, self._close_note_marker, nodes=buf) return if tag == "a" and cls == "notemark": node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")} self._emit(node) buf = [] self._push(tag, buf, self._close_noteref, node=node, nodes=buf) return if tag == "a" and cls == "notebackref": buf = [] self._push(tag, buf, self._close_backref, nodes=buf) return if tag == "span" and cls in INLINE: kind = INLINE[cls] if kind is None: self._push(tag, self.sink) else: node = {"t": kind, "c": []} self._emit(node) self._push(tag, node["c"]) return if tag == "a": ref = parse_href(attrs.get("href")) if ref is not None and self.sink is not None: node = {"t": "ref", "c": []} node.update(ref) self._emit(node) self._push(tag, node["c"]) else: self._push(tag, self.sink) return self._push(tag, self.sink) def handle_endtag(self, tag): if not self.in_main or not self.stack: return for i in range(len(self.stack) - 1, -1, -1): if self.stack[i]["tag"] == tag: break else: return while len(self.stack) > i: frame = self.stack.pop() self.sink = frame["prev"] if frame["close"]: frame["close"](frame) if tag == "div" and not self.stack: self.in_main = False def handle_data(self, data): if self.sink is not None and data: self.sink.append({"t": "text", "v": data}) def _close_chapter_label(self, frame): self.chapter_label = plain_text(frame["nodes"]).strip() def _close_block(self, frame): block = frame["block"] block["content"] = normalize(block["content"]) if block["type"] == "booktitle": if block["content"]: self.titles.append({ "style": block["style"], "level": block.get("level"), "text": plain_text(block["content"]), }) return if block["type"] == "break": block.pop("content") self.blocks.append(block) return if block["content"]: self.blocks.append(block) def _close_note(self, frame): note = frame["note"] note["content"] = normalize(note["content"]) note["text"] = plain_text(note["content"]) self.notes.append(note) def _close_verse(self, frame): label = plain_text(frame["nodes"]).replace(" ", " ").strip() node = {"t": "verse", "label": label} m = re.match(r"^(\d+)(?:[-–](\d+))?$", label) if m: node["n"] = int(m.group(1)) if m.group(2): node["end"] = int(m.group(2)) self._emit(node) def _close_note_marker(self, frame): marker = plain_text(frame["nodes"]).strip() for f in reversed(self.stack): if "note" in f: f["note"]["marker"] = marker return def _close_noteref(self, frame): frame["node"]["marker"] = plain_text(frame["nodes"]).strip() def _close_backref(self, frame): text = plain_text(frame["nodes"]).strip() for f in reversed(self.stack): if "note" in f: f["note"]["ref"] = text m = re.search(r":(\d+)", text) if m: f["note"]["verse"] = int(m.group(1)) return class IndexParser(HTMLParser): SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"} def __init__(self): super().__init__(convert_charrefs=True) self.books = [] self.title = None self._link = None self._in_title = False def handle_starttag(self, tag, attrs): attrs = dict(attrs) if tag == "title": self._in_title = True elif tag == "a": m = CHAPTER_RE.match(os.path.basename(attrs.get("href") or "")) section = self.SECTIONS.get((attrs.get("class") or "").strip()) if m and section: self._link = { "code": m.group(1), "section": section, "order": len(self.books) + 1, "name": "", } def handle_endtag(self, tag): if tag == "title": self._in_title = False elif tag == "a" and self._link: self._link["name"] = self._link["name"].strip() self.books.append(self._link) self._link = None def handle_data(self, data): if self._in_title and self.title is None: self.title = data.strip() elif self._link is not None: self._link["name"] += data def parse_chapter(path): parser = ChapterParser() with open(path, encoding="utf-8-sig") as fh: parser.feed(fh.read()) parser.close() return parser def extract(source_dir): index_path = os.path.join(source_dir, "index.htm") index = IndexParser() with open(index_path, encoding="utf-8-sig") as fh: index.feed(fh.read()) meta = {b["code"]: b for b in index.books} by_book = {} for name in sorted(os.listdir(source_dir)): m = CHAPTER_RE.match(name) if m: by_book.setdefault(m.group(1), []).append((int(m.group(2)), name)) books = [] skipped = [] for code in sorted(by_book, key=lambda c: (meta.get(c, {}).get("order", 999), c)): info = meta.get(code, {"code": code, "name": code, "section": "other", "order": 999}) book = { "code": code, "name": info["name"], "section": info["section"], "titles": [], "chapters": [], } for number, name in sorted(by_book[code]): parsed = parse_chapter(os.path.join(source_dir, name)) if parsed.is_toc or not parsed.blocks: skipped.append(name) continue if parsed.titles and not book["titles"]: book["titles"] = parsed.titles label = parsed.chapter_label chapter = { "number": int(label) if label and label.isdigit() else number, "blocks": parsed.blocks, "notes": parsed.notes, } book["chapters"].append(chapter) if book["chapters"]: books.append(book) return { "translation": { "id": "engwebu", "name": index.title or "World English Bible Updated", "license": "Public Domain", "source": "eBible.org", }, "books": books, }, skipped INLINE_TAGS = { "wj": ("span", "wj"), "keyword": ("dfn", "k"), "bookref": ("cite", "bk"), "fq": ("span", "fq"), "fqa": ("span", "fqa"), "fl": ("span", "fl"), "fine": ("span", "fine"), "xt": ("span", "xt"), } BLOCK_TAGS = { "paragraph": "p", "line": "p", "heading": "h3", "title": "p", "speaker": "p", "item": "li", } def esc(text): return html.escape(text, quote=False) def verse_id(chapter, label): return "v%d_%s" % (chapter, label) def ref_href(node, book, chapter): target_book = node.get("book") or book target_ch = node.get("chapter", chapter) verse = node.get("verse") base = "" if target_book == book else target_book if verse: return "%s#%s" % (base, verse_id(target_ch, verse)) return "%s#ch%d" % (base, target_ch) DROPCAP_BLOCKS = ("paragraph", "line", "item") def with_dropcap(nodes): """Split the first letter of the first text off into a dropcap node. Returns the new node list, or None when the text does not start with a letter. The initials font has only A-Z and a-z, so a chapter opening on a quotation mark or a digit gets no drop cap rather than a fallback one. """ nodes = list(nodes) for i, node in enumerate(nodes): kind = node["t"] if kind == "verse": continue # A note caller would land between the letter and the rest of its word. if kind == "noteref": return None if kind == "text": text = node["v"].lstrip() if not text: continue if not ("A" <= text[0] <= "Z" or "a" <= text[0] <= "z"): return None nodes[i:i + 1] = [{"t": "dropcap", "v": text[0]}, dict(node, v=text[1:])] else: inner = with_dropcap(node.get("c", [])) if inner is None: return None nodes[i] = dict(node, c=inner) # The verse number before the drop cap would otherwise sit between # the floated letter and the rest of its word. for j in range(i): if nodes[j]["t"] == "verse": nodes[j] = dict(nodes[j], dropcap=True) return nodes return None def render_inline(nodes, book, chapter, out): for node in nodes: kind = node["t"] if kind == "text": out.append(esc(node["v"])) elif kind == "dropcap": out.append('%s' % esc(node["v"])) elif kind == "verse": label = esc(node["label"]) vid = verse_id(chapter, node["label"]) # A link to itself: the click both puts the verse in the address # bar, which is what the options dialog offers to copy, and opens # that dialog. Being a real link is also what gives it the # keyboard and context menu a reader expects. out.append('%s' % (vid, vid, label, label)) # A non-breaking space keeps the number on the same line as the # word it introduces, as the source does. Before a drop cap the # number is hidden and the space would only push the word away. if not node.get("dropcap"): out.append(" ") elif kind == "noteref": n = node["id"].replace("FN", "") out.append('' "%s" % (chapter, n, chapter, n, esc(caller(node.get("marker"))))) elif kind == "br": out.append("
") elif kind == "ref": # htmx does not scroll to a fragment, so a reference into another # book has to be a real navigation for the browser to do it. href = ref_href(node, book, chapter) boost = "" if href.startswith("#") else ' hx-boost="false"' out.append('' % (href, boost)) render_inline(node.get("c", []), book, chapter, out) out.append("") elif kind in INLINE_TAGS: tag, cls = INLINE_TAGS[kind] out.append('<%s class="%s">' % (tag, cls)) render_inline(node.get("c", []), book, chapter, out) out.append("" % tag) else: render_inline(node.get("c", []), book, chapter, out) def render_block(block, book, chapter, out): btype = block["type"] tag = BLOCK_TAGS.get(btype, "p") classes = [block["style"]] attrs = "" if "indent" in block: attrs += ' data-indent="%d"' % block["indent"] if btype == "line": classes.append("line") out.append(" <%s class=\"%s\"%s>" % (tag, " ".join(classes), attrs)) render_inline(block.get("content", []), book, chapter, out) out.append("\n" % tag) TEXT_PRESENTATION = "\ufe0e" EMOJI_CAPABLE = "\u2721" def caller(marker): return "".join(c + TEXT_PRESENTATION if c in EMOJI_CAPABLE else c for c in (marker or "*")) def render_notes(notes, chapter, book, out): if not notes: return out.append('
\n') out.append('

Notes

\n') out.append("
    \n") for note in notes: n = (note["id"] or "").replace("FN", "") out.append('
  1. ' % (chapter, n, note["kind"])) out.append('%s ' % esc(caller(note.get("marker")))) if note.get("ref"): out.append('%s ' % (chapter, n, esc(note["ref"]))) out.append('') render_inline(note["content"], book, chapter, out) out.append("
    2. \n") out.append("
\n
\n") def render_chapter(chapter, book, out): num = chapter["number"] out.append('
\n' % (num, num)) out.append('

' '%d

\n' % (num, num, num)) group = None def close_group(): nonlocal group if group == "poetry": out.append(" \n") elif group == "list": out.append(" \n") group = None dropcap = True for block in chapter["blocks"]: btype = block["type"] if dropcap and btype in DROPCAP_BLOCKS: dropcap = False content = with_dropcap(block.get("content", [])) if content is not None: block = dict(block, content=content) if btype == "break": close_group() continue want = {"line": "poetry", "item": "list"}.get(btype) if want != group: close_group() if want == "poetry": out.append('
\n') elif want == "list": out.append('
\n") def render_book(book, translation, out_dir): code, name = book["code"], book["name"] out = [] title = "%s — %s" % (name, translation["name"]) out.append("%s\n" % esc(title)) out.append('
\n' % (code, code, esc(name), book["section"])) out.append('
\n') out.append("

") if book["titles"]: out.append(" ".join('%s' % (part["style"], esc(part["text"])) for part in book["titles"])) else: out.append(esc(name)) out.append("

\n") out.append("
\n") out.append('
\n') for chapter in book["chapters"]: render_chapter(chapter, code, out) out.append("
\n") out.append(' \n' % (esc(translation["name"]), esc(translation["license"]))) out.append("
\n") path = os.path.join(out_dir, "%s.html" % code) with open(path, "w", encoding="utf-8") as fh: fh.write("".join(out)) return path SECTION_LABELS = [ ("ot", "Old Testament"), ("nt", "New Testament"), ("deuterocanon", "Deuterocanon"), ("other", "Other"), ] def render_index(books, translation, out_dir): out = ["%s\n" % esc(translation["name"]), '
\n'] for key, label in SECTION_LABELS: group = [b for b in books if b["section"] == key] if not group: continue out.append('
\n' % key) out.append("

%s

\n \n
\n") out.append(' \n' % (esc(translation["name"]), esc(translation["license"]))) out.append("
\n") path = os.path.join(out_dir, "index.html") with open(path, "w", encoding="utf-8") as fh: fh.write("".join(out)) return path VERSES_FILE = "verses.json" def chapter_verses(chapter): labels = [] def walk(nodes): for node in nodes: if node["t"] == "verse": labels.append(node["label"]) elif "c" in node: walk(node["c"]) for block in chapter["blocks"]: walk(block.get("content", [])) return labels def render_verses(books, out_dir): out = {} total = 0 for book in books: numbers = [c["number"] for c in book["chapters"]] if numbers != list(range(1, len(numbers) + 1)): raise RuntimeError("%s: chapters are not 1..%d (%s)" % (book["code"], len(numbers), numbers)) chapters = [] for chapter in book["chapters"]: labels = chapter_verses(chapter) plain = [str(i + 1) for i in range(len(labels))] chapters.append(len(labels) if labels == plain else labels) total += len(labels) out[book["code"]] = chapters path = os.path.join(out_dir, VERSES_FILE) with open(path, "w", encoding="utf-8") as fh: json.dump(out, fh, separators=(",", ":"), ensure_ascii=False) fh.write("\n") return path, total def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("source", nargs="?", default="source", help="directory of eBible .htm files (default: source)") ap.add_argument("-o", "--out", default="bible", help="output directory (default: bible)") args = ap.parse_args(argv) data, skipped = extract(args.source) os.makedirs(args.out, exist_ok=True) total = 0 for book in data["books"]: total += os.path.getsize(render_book(book, data["translation"], args.out)) render_index(data["books"], data["translation"], args.out) verses, count = render_verses(data["books"], args.out) chapters = sum(len(b["chapters"]) for b in data["books"]) notes = sum(len(c["notes"]) for b in data["books"] for c in b["chapters"]) print("read %s/" % args.source.rstrip("/"), file=sys.stderr) print("books %d" % len(data["books"]), file=sys.stderr) print("chapters %d" % chapters, file=sys.stderr) print("notes %d" % notes, file=sys.stderr) print("skipped %d (contents pages)" % len(skipped), file=sys.stderr) print("html %.1f MB total, %.0f KB average" % (total / 1e6, total / len(data["books"]) / 1e3), file=sys.stderr) print("verses %d in %s (%.1f KB)" % (count, os.path.basename(verses), os.path.getsize(verses) / 1e3), file=sys.stderr) print("wrote %s/" % args.out.rstrip("/"), file=sys.stderr) return 0 if __name__ == "__main__": sys.exit(main())