WIP

extract_bible.py

@@ -0,0 +1,605 @@
+#!/usr/bin/env python3
+"""Extract the eBible.org HTML Bible export into structured JSON.
+
+The source is a flat directory of generated chapter files (``JHN03.htm``) plus
+per-book and site index pages. Each chapter file is a flat sequence of
+``<div class='...'>`` blocks whose class is the USFM paragraph marker (``p``,
+``q2``, ``s``, ...), with verse numbers, footnote markers and character styles
+as inline spans. Footnotes and cross references are collected in a trailing
+``<div class="footnote">``.
+
+This script turns that into a JSON tree of blocks and inline nodes, so the
+markup can be re-rendered from templates rather than scraped again. Navigation
+lists and the copyright footer are dropped; footnotes are kept.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from html.parser import HTMLParser
+
+# Chapter files are a three-character book code plus a zero-padded chapter
+# number (three digits for Psalms).
+CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$")
+
+# Links between chapters: "JOB05.htm#V13", "JHN03.htm", or a bare "#V7".
+HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$")
+
+# Block-level div classes -> semantic block type.
+BLOCK_TYPES = {
+    "p": "paragraph",    # normal prose paragraph
+    "m": "paragraph",    # continuation, no first-line indent
+    "nb": "paragraph",   # no break with previous
+    "pi": "paragraph",   # indented
+    "mi": "paragraph",   # indented, no first-line indent
+    "pc": "paragraph",   # centered
+    "ip": "paragraph",   # introduction paragraph
+    "q": "line",         # poetry line
+    "q2": "line",
+    "q3": "line",
+    "qs": "line",        # Selah
+    "b": "break",        # blank line between stanzas
+    "d": "title",        # descriptive title (psalm superscription, acrostic letter)
+    "s": "heading",      # section heading
+    "ms": "heading",     # major section heading ("BOOK 3")
+    "is": "heading",     # introduction heading
+    "sp": "speaker",     # speaker label (Song of Solomon)
+    "li": "item",        # list item
+    "ili": "item",       # introduction list item (glossary entries)
+    "mt": "booktitle",
+    "mt2": "booktitle",
+    "mt3": "booktitle",
+}
+
+# Nesting depth for poetry lines and indented paragraphs.
+INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1}
+
+# Heading / title rank.
+LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2}
+
+# Character-style spans -> inline node type. ``None`` means unwrap the span and
+# keep its children inline (``ft`` is just "footnote body text").
+INLINE = {
+    "wj": "wj",        # words of Jesus
+    "k": "keyword",    # glossary keyword
+    "bk": "bookref",   # book name cited in a note
+    "fq": "fq",        # the quoted words the footnote is about
+    "fqa": "fqa",      # an alternative rendering
+    "fl": "fl",        # footnote label
+    "fine": "fine",
+    "xt": "xt",        # cross-reference target
+    "ft": None,
+}
+
+# Divs whose entire subtree is navigation or licensing, not content.
+DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"}
+
+
+def parse_href(href):
+    """Turn an intra-corpus link into a structured reference, or None."""
+    m = HREF_RE.match(href or "")
+    if not m or not any(m.groups()):
+        return None
+    book, chapter, verse = m.groups()
+    ref = {}
+    if book:
+        ref["book"] = book
+        ref["chapter"] = int(chapter)
+    if verse:
+        ref["verse"] = int(verse)
+    return ref or None
+
+
+# --------------------------------------------------------------------------
+# Whitespace
+# --------------------------------------------------------------------------
+
+def normalize(nodes):
+    """Collapse whitespace across the whole inline tree, in document order.
+
+    Runs of whitespace become a single space and spaces are not duplicated
+    across element boundaries, so "word</span> <span>next" keeps exactly one.
+    """
+    state = {"prev_space": True}  # True at the start suppresses a leading space
+
+    def walk(lst):
+        out = []
+        for node in lst:
+            if node["t"] == "text":
+                text = re.sub(r"[\s ]+", " ", node["v"])
+                if state["prev_space"]:
+                    text = text.lstrip(" ")
+                if text:
+                    state["prev_space"] = text.endswith(" ")
+                    out.append({"t": "text", "v": text})
+            elif "c" in node:
+                node["c"] = walk(node["c"])
+                if node["c"]:
+                    out.append(node)
+            else:
+                state["prev_space"] = False
+                out.append(node)
+        return out
+
+    result = walk(nodes)
+    rstrip_last(result)
+    return result
+
+
+def rstrip_last(nodes):
+    """Drop the trailing space on a tree's final text node.
+
+    Only a genuinely trailing node is stripped. A space before a block's
+    closing verse or note marker separates it from the preceding word and
+    has to survive, so the walk stops at the last node rather than looking
+    past markers for something to strip.
+    """
+    if not nodes:
+        return False
+    node = nodes[-1]
+    if node["t"] == "text":
+        node["v"] = node["v"].rstrip()
+        if not node["v"]:
+            nodes.pop()
+            return rstrip_last(nodes)
+        return True
+    if "c" in node:
+        return rstrip_last(node["c"])
+    return False
+
+
+def plain_text(nodes, skip=("noteref",)):
+    """Flatten an inline tree to readable text, dropping note markers."""
+    parts = []
+    for node in nodes:
+        if node["t"] in skip:
+            continue
+        if node["t"] == "text":
+            parts.append(node["v"])
+        elif node["t"] == "br":
+            parts.append(" ")
+        elif "c" in node:
+            parts.append(plain_text(node["c"], skip))
+    return "".join(parts)
+
+
+# --------------------------------------------------------------------------
+# Chapter parser
+# --------------------------------------------------------------------------
+
+class ChapterParser(HTMLParser):
+    """Parse one chapter file into blocks + notes.
+
+    Content is appended to ``self.sink``, the inline-node list currently being
+    filled. A stack of frames records what to restore (and what to finish) as
+    each element closes; ``sink is None`` discards text, which is how
+    navigation and the copyright footer are dropped.
+    """
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.blocks = []
+        self.notes = []
+        self.titles = []          # mt/mt2/mt3, hoisted to the book
+        self.chapter_label = None
+        self.is_toc = False       # a chapter-0 navigation page
+
+        self.in_main = False
+        self.stack = []
+        self.sink = None
+
+    # -- frame helpers ----------------------------------------------------
+
+    def _push(self, tag, sink, on_close=None, **extra):
+        frame = {"tag": tag, "prev": self.sink, "close": on_close}
+        frame.update(extra)
+        self.stack.append(frame)
+        self.sink = sink
+        return frame
+
+    def _emit(self, node):
+        if self.sink is not None:
+            self.sink.append(node)
+
+    # -- HTMLParser hooks -------------------------------------------------
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        classes = (attrs.get("class") or "").split()
+        cls = classes[0] if classes else ""
+
+        if tag in ("br", "hr", "img", "meta", "link", "input"):
+            if tag == "br":
+                self._emit({"t": "br"})
+            return
+
+        if not self.in_main:
+            # Everything before <div class="main"> is <head> and top nav.
+            if tag == "div" and "main" in classes:
+                self.in_main = True
+                self._push(tag, None)
+            return
+
+        if cls in DROP_CLASSES:
+            if cls in ("toc", "toc1", "toc2"):
+                self.is_toc = True
+            self._push(tag, None)
+            return
+
+        if tag == "div" and cls == "chapterlabel":
+            buf = []
+            self._push(tag, buf, self._close_chapter_label, nodes=buf)
+            return
+
+        if tag == "div" and cls in BLOCK_TYPES:
+            block = {"type": BLOCK_TYPES[cls], "style": cls, "content": []}
+            if cls in INDENT:
+                block["indent"] = INDENT[cls]
+            if cls in LEVEL:
+                block["level"] = LEVEL[cls]
+            self._push(tag, block["content"], self._close_block, block=block)
+            return
+
+        if tag == "p" and cls in ("f", "x"):
+            note = {
+                "id": attrs.get("id"),
+                "kind": "footnote" if cls == "f" else "xref",
+                "marker": None,
+                "ref": None,
+                "verse": None,
+                "content": [],
+            }
+            self._push(tag, note["content"], self._close_note, note=note)
+            return
+
+        if tag == "span" and "verse" in classes:
+            buf = []
+            self._push(tag, buf, self._close_verse, nodes=buf)
+            return
+
+        if tag == "span" and cls == "notemark":
+            # The note's own marker inside the footnote list.
+            buf = []
+            self._push(tag, buf, self._close_note_marker, nodes=buf)
+            return
+
+        if tag == "a" and cls == "notemark":
+            # An in-text marker linking down to the note; its nested
+            # <span class="popup"> duplicates the note body and is dropped.
+            node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")}
+            self._emit(node)
+            buf = []
+            self._push(tag, buf, self._close_noteref, node=node, nodes=buf)
+            return
+
+        if tag == "a" and cls == "notebackref":
+            buf = []
+            self._push(tag, buf, self._close_backref, nodes=buf)
+            return
+
+        if tag == "span" and cls in INLINE:
+            kind = INLINE[cls]
+            if kind is None:
+                self._push(tag, self.sink)  # unwrap
+            else:
+                node = {"t": kind, "c": []}
+                self._emit(node)
+                self._push(tag, node["c"])
+            return
+
+        if tag == "a":
+            ref = parse_href(attrs.get("href"))
+            if ref is not None and self.sink is not None:
+                node = {"t": "ref", "c": []}
+                node.update(ref)
+                self._emit(node)
+                self._push(tag, node["c"])
+            else:
+                self._push(tag, self.sink)
+            return
+
+        # Anything else (nested spans, stray markup) is transparent.
+        self._push(tag, self.sink)
+
+    def handle_endtag(self, tag):
+        if not self.in_main or not self.stack:
+            return
+        for i in range(len(self.stack) - 1, -1, -1):
+            if self.stack[i]["tag"] == tag:
+                break
+        else:
+            return  # unmatched close tag; ignore
+        while len(self.stack) > i:
+            frame = self.stack.pop()
+            self.sink = frame["prev"]
+            if frame["close"]:
+                frame["close"](frame)
+        if tag == "div" and not self.stack:
+            self.in_main = False  # <div class="main"> closed
+
+    def handle_data(self, data):
+        if self.sink is not None and data:
+            self.sink.append({"t": "text", "v": data})
+
+    # -- frame close handlers ---------------------------------------------
+
+    def _close_chapter_label(self, frame):
+        self.chapter_label = plain_text(frame["nodes"]).strip()
+
+    def _close_block(self, frame):
+        block = frame["block"]
+        block["content"] = normalize(block["content"])
+        if block["type"] == "booktitle":
+            if block["content"]:
+                self.titles.append({
+                    "style": block["style"],
+                    "level": block.get("level"),
+                    "text": plain_text(block["content"]),
+                })
+            return
+        if block["type"] == "break":
+            block.pop("content")
+            self.blocks.append(block)
+            return
+        if block["content"]:
+            self.blocks.append(block)
+
+    def _close_note(self, frame):
+        note = frame["note"]
+        note["content"] = normalize(note["content"])
+        note["text"] = plain_text(note["content"])
+        self.notes.append(note)
+
+    def _close_verse(self, frame):
+        label = plain_text(frame["nodes"]).replace(" ", " ").strip()
+        node = {"t": "verse", "label": label}
+        m = re.match(r"^(\d+)(?:[-–](\d+))?$", label)
+        if m:
+            node["n"] = int(m.group(1))
+            if m.group(2):
+                node["end"] = int(m.group(2))
+        self._emit(node)
+
+    def _close_note_marker(self, frame):
+        marker = plain_text(frame["nodes"]).strip()
+        for f in reversed(self.stack):
+            if "note" in f:
+                f["note"]["marker"] = marker
+                return
+
+    def _close_noteref(self, frame):
+        frame["node"]["marker"] = plain_text(frame["nodes"]).strip()
+
+    def _close_backref(self, frame):
+        text = plain_text(frame["nodes"]).strip()
+        for f in reversed(self.stack):
+            if "note" in f:
+                f["note"]["ref"] = text
+                m = re.search(r":(\d+)", text)
+                if m:
+                    f["note"]["verse"] = int(m.group(1))
+                return
+
+
+# --------------------------------------------------------------------------
+# Corpus
+# --------------------------------------------------------------------------
+
+class IndexParser(HTMLParser):
+    """Read book order, display names and canon section from index.htm."""
+
+    # Link class on the site index marks which part of the canon a book is in.
+    SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"}
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.books = []       # [{code, name, section, order}]
+        self.title = None
+        self._link = None
+        self._in_title = False
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if tag == "title":
+            self._in_title = True
+        elif tag == "a":
+            m = CHAPTER_RE.match(os.path.basename(attrs.get("href") or ""))
+            section = self.SECTIONS.get((attrs.get("class") or "").strip())
+            if m and section:
+                self._link = {
+                    "code": m.group(1),
+                    "section": section,
+                    "order": len(self.books) + 1,
+                    "name": "",
+                }
+
+    def handle_endtag(self, tag):
+        if tag == "title":
+            self._in_title = False
+        elif tag == "a" and self._link:
+            self._link["name"] = self._link["name"].strip()
+            self.books.append(self._link)
+            self._link = None
+
+    def handle_data(self, data):
+        if self._in_title and self.title is None:
+            self.title = data.strip()
+        elif self._link is not None:
+            self._link["name"] += data
+
+
+def verse_index(blocks):
+    """Flat {verse label: text} map, for search and direct verse lookup.
+
+    Verse markers are inline, so a verse's text runs from its marker to the
+    next one and may span several blocks (a poetry couplet, for instance).
+    """
+    verses = {}
+    current = None
+    parts = []
+
+    def flush():
+        if current is not None:
+            text = re.sub(r"\s+", " ", "".join(parts)).strip()
+            if text:
+                verses[current] = (verses.get(current, "") + " " + text).strip()
+
+    def walk(nodes):
+        nonlocal current
+        for node in nodes:
+            if node["t"] == "verse":
+                flush()
+                parts.clear()
+                current = node["label"]
+            elif node["t"] == "noteref":
+                continue
+            elif node["t"] == "text":
+                parts.append(node["v"])
+            elif node["t"] == "br":
+                parts.append(" ")
+            elif "c" in node:
+                walk(node["c"])
+
+    for block in blocks:
+        content = block.get("content")
+        if not content:
+            continue
+        walk(content)
+        parts.append(" ")  # blocks do not run words together
+    flush()
+    return verses
+
+
+def parse_chapter(path):
+    parser = ChapterParser()
+    with open(path, encoding="utf-8-sig") as fh:
+        parser.feed(fh.read())
+    parser.close()
+    return parser
+
+
+def extract(source_dir, with_verse_text=True):
+    index_path = os.path.join(source_dir, "index.htm")
+    index = IndexParser()
+    with open(index_path, encoding="utf-8-sig") as fh:
+        index.feed(fh.read())
+
+    meta = {b["code"]: b for b in index.books}
+
+    # Group chapter files by book code.
+    by_book = {}
+    for name in sorted(os.listdir(source_dir)):
+        m = CHAPTER_RE.match(name)
+        if m:
+            by_book.setdefault(m.group(1), []).append((int(m.group(2)), name))
+
+    books = []
+    skipped = []
+    for code in sorted(by_book, key=lambda c: (meta.get(c, {}).get("order", 999), c)):
+        info = meta.get(code, {"code": code, "name": code, "section": "other",
+                               "order": 999})
+        book = {
+            "code": code,
+            "name": info["name"],
+            "section": info["section"],
+            "titles": [],
+            "chapters": [],
+        }
+        for number, name in sorted(by_book[code]):
+            parsed = parse_chapter(os.path.join(source_dir, name))
+            if parsed.is_toc or not parsed.blocks:
+                # Chapter-0 files are per-book tables of contents, not text.
+                skipped.append(name)
+                continue
+            if parsed.titles and not book["titles"]:
+                book["titles"] = parsed.titles
+            label = parsed.chapter_label
+            chapter = {
+                "number": int(label) if label and label.isdigit() else number,
+                "blocks": parsed.blocks,
+                "notes": parsed.notes,
+            }
+            if with_verse_text:
+                chapter["verses"] = verse_index(parsed.blocks)
+            book["chapters"].append(chapter)
+        if book["chapters"]:
+            books.append(book)
+
+    return {
+        "translation": {
+            "id": "engwebu",
+            "name": index.title or "World English Bible Updated",
+            "license": "Public Domain",
+            "source": "eBible.org",
+        },
+        "books": books,
+    }, skipped
+
+
+def write_split(data, out_dir):
+    """One JSON per chapter plus an index, for chapter-at-a-time fetching."""
+    os.makedirs(out_dir, exist_ok=True)
+    index = {"translation": data["translation"], "books": []}
+    for book in data["books"]:
+        book_dir = os.path.join(out_dir, book["code"])
+        os.makedirs(book_dir, exist_ok=True)
+        for chapter in book["chapters"]:
+            payload = dict(chapter)
+            payload["book"] = book["code"]
+            path = os.path.join(book_dir, "%d.json" % chapter["number"])
+            with open(path, "w", encoding="utf-8") as fh:
+                json.dump(payload, fh, ensure_ascii=False, separators=(",", ":"))
+        index["books"].append({
+            "code": book["code"],
+            "name": book["name"],
+            "section": book["section"],
+            "titles": book["titles"],
+            "chapters": [c["number"] for c in book["chapters"]],
+        })
+    with open(os.path.join(out_dir, "index.json"), "w", encoding="utf-8") as fh:
+        json.dump(index, fh, ensure_ascii=False, indent=2)
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("source", nargs="?", default="source",
+                    help="directory of eBible .htm files (default: source)")
+    ap.add_argument("-o", "--output", default="bible.json",
+                    help="output JSON file (default: bible.json)")
+    ap.add_argument("--split", metavar="DIR",
+                    help="also write one JSON per chapter into DIR")
+    ap.add_argument("--no-verse-text", action="store_true",
+                    help="omit the flat per-chapter verse text map")
+    ap.add_argument("--pretty", action="store_true",
+                    help="indent the output JSON")
+    args = ap.parse_args(argv)
+
+    data, skipped = extract(args.source, with_verse_text=not args.no_verse_text)
+
+    dump = dict(indent=2) if args.pretty else dict(separators=(",", ":"))
+    with open(args.output, "w", encoding="utf-8") as fh:
+        json.dump(data, fh, ensure_ascii=False, **dump)
+
+    if args.split:
+        write_split(data, args.split)
+
+    chapters = sum(len(b["chapters"]) for b in data["books"])
+    notes = sum(len(c["notes"]) for b in data["books"] for c in b["chapters"])
+    blocks = sum(len(c["blocks"]) for b in data["books"] for c in b["chapters"])
+    size = os.path.getsize(args.output) / 1e6
+    print("books    %d" % len(data["books"]), file=sys.stderr)
+    print("chapters %d" % chapters, file=sys.stderr)
+    print("blocks   %d" % blocks, file=sys.stderr)
+    print("notes    %d" % notes, file=sys.stderr)
+    print("skipped  %d (contents pages)" % len(skipped), file=sys.stderr)
+    print("wrote    %s (%.1f MB)" % (args.output, size), file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())