606 lines
21 KiB
Python
606 lines
21 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract the eBible.org HTML Bible export into structured JSON.
|
||
|
||
The source is a flat directory of generated chapter files (``JHN03.htm``) plus
|
||
per-book and site index pages. Each chapter file is a flat sequence of
|
||
``<div class='...'>`` blocks whose class is the USFM paragraph marker (``p``,
|
||
``q2``, ``s``, ...), with verse numbers, footnote markers and character styles
|
||
as inline spans. Footnotes and cross references are collected in a trailing
|
||
``<div class="footnote">``.
|
||
|
||
This script turns that into a JSON tree of blocks and inline nodes, so the
|
||
markup can be re-rendered from templates rather than scraped again. Navigation
|
||
lists and the copyright footer are dropped; footnotes are kept.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from html.parser import HTMLParser
|
||
|
||
# Chapter files are a three-character book code plus a zero-padded chapter
|
||
# number (three digits for Psalms).
|
||
CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$")
|
||
|
||
# Links between chapters: "JOB05.htm#V13", "JHN03.htm", or a bare "#V7".
|
||
HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$")
|
||
|
||
# Block-level div classes -> semantic block type.
|
||
BLOCK_TYPES = {
|
||
"p": "paragraph", # normal prose paragraph
|
||
"m": "paragraph", # continuation, no first-line indent
|
||
"nb": "paragraph", # no break with previous
|
||
"pi": "paragraph", # indented
|
||
"mi": "paragraph", # indented, no first-line indent
|
||
"pc": "paragraph", # centered
|
||
"ip": "paragraph", # introduction paragraph
|
||
"q": "line", # poetry line
|
||
"q2": "line",
|
||
"q3": "line",
|
||
"qs": "line", # Selah
|
||
"b": "break", # blank line between stanzas
|
||
"d": "title", # descriptive title (psalm superscription, acrostic letter)
|
||
"s": "heading", # section heading
|
||
"ms": "heading", # major section heading ("BOOK 3")
|
||
"is": "heading", # introduction heading
|
||
"sp": "speaker", # speaker label (Song of Solomon)
|
||
"li": "item", # list item
|
||
"ili": "item", # introduction list item (glossary entries)
|
||
"mt": "booktitle",
|
||
"mt2": "booktitle",
|
||
"mt3": "booktitle",
|
||
}
|
||
|
||
# Nesting depth for poetry lines and indented paragraphs.
|
||
INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1}
|
||
|
||
# Heading / title rank.
|
||
LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2}
|
||
|
||
# Character-style spans -> inline node type. ``None`` means unwrap the span and
|
||
# keep its children inline (``ft`` is just "footnote body text").
|
||
INLINE = {
|
||
"wj": "wj", # words of Jesus
|
||
"k": "keyword", # glossary keyword
|
||
"bk": "bookref", # book name cited in a note
|
||
"fq": "fq", # the quoted words the footnote is about
|
||
"fqa": "fqa", # an alternative rendering
|
||
"fl": "fl", # footnote label
|
||
"fine": "fine",
|
||
"xt": "xt", # cross-reference target
|
||
"ft": None,
|
||
}
|
||
|
||
# Divs whose entire subtree is navigation or licensing, not content.
|
||
DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"}
|
||
|
||
|
||
def parse_href(href):
|
||
"""Turn an intra-corpus link into a structured reference, or None."""
|
||
m = HREF_RE.match(href or "")
|
||
if not m or not any(m.groups()):
|
||
return None
|
||
book, chapter, verse = m.groups()
|
||
ref = {}
|
||
if book:
|
||
ref["book"] = book
|
||
ref["chapter"] = int(chapter)
|
||
if verse:
|
||
ref["verse"] = int(verse)
|
||
return ref or None
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Whitespace
|
||
# --------------------------------------------------------------------------
|
||
|
||
def normalize(nodes):
|
||
"""Collapse whitespace across the whole inline tree, in document order.
|
||
|
||
Runs of whitespace become a single space and spaces are not duplicated
|
||
across element boundaries, so "word</span> <span>next" keeps exactly one.
|
||
"""
|
||
state = {"prev_space": True} # True at the start suppresses a leading space
|
||
|
||
def walk(lst):
|
||
out = []
|
||
for node in lst:
|
||
if node["t"] == "text":
|
||
text = re.sub(r"[\s ]+", " ", node["v"])
|
||
if state["prev_space"]:
|
||
text = text.lstrip(" ")
|
||
if text:
|
||
state["prev_space"] = text.endswith(" ")
|
||
out.append({"t": "text", "v": text})
|
||
elif "c" in node:
|
||
node["c"] = walk(node["c"])
|
||
if node["c"]:
|
||
out.append(node)
|
||
else:
|
||
state["prev_space"] = False
|
||
out.append(node)
|
||
return out
|
||
|
||
result = walk(nodes)
|
||
rstrip_last(result)
|
||
return result
|
||
|
||
|
||
def rstrip_last(nodes):
|
||
"""Drop the trailing space on a tree's final text node.
|
||
|
||
Only a genuinely trailing node is stripped. A space before a block's
|
||
closing verse or note marker separates it from the preceding word and
|
||
has to survive, so the walk stops at the last node rather than looking
|
||
past markers for something to strip.
|
||
"""
|
||
if not nodes:
|
||
return False
|
||
node = nodes[-1]
|
||
if node["t"] == "text":
|
||
node["v"] = node["v"].rstrip()
|
||
if not node["v"]:
|
||
nodes.pop()
|
||
return rstrip_last(nodes)
|
||
return True
|
||
if "c" in node:
|
||
return rstrip_last(node["c"])
|
||
return False
|
||
|
||
|
||
def plain_text(nodes, skip=("noteref",)):
|
||
"""Flatten an inline tree to readable text, dropping note markers."""
|
||
parts = []
|
||
for node in nodes:
|
||
if node["t"] in skip:
|
||
continue
|
||
if node["t"] == "text":
|
||
parts.append(node["v"])
|
||
elif node["t"] == "br":
|
||
parts.append(" ")
|
||
elif "c" in node:
|
||
parts.append(plain_text(node["c"], skip))
|
||
return "".join(parts)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Chapter parser
|
||
# --------------------------------------------------------------------------
|
||
|
||
class ChapterParser(HTMLParser):
|
||
"""Parse one chapter file into blocks + notes.
|
||
|
||
Content is appended to ``self.sink``, the inline-node list currently being
|
||
filled. A stack of frames records what to restore (and what to finish) as
|
||
each element closes; ``sink is None`` discards text, which is how
|
||
navigation and the copyright footer are dropped.
|
||
"""
|
||
|
||
def __init__(self):
|
||
super().__init__(convert_charrefs=True)
|
||
self.blocks = []
|
||
self.notes = []
|
||
self.titles = [] # mt/mt2/mt3, hoisted to the book
|
||
self.chapter_label = None
|
||
self.is_toc = False # a chapter-0 navigation page
|
||
|
||
self.in_main = False
|
||
self.stack = []
|
||
self.sink = None
|
||
|
||
# -- frame helpers ----------------------------------------------------
|
||
|
||
def _push(self, tag, sink, on_close=None, **extra):
|
||
frame = {"tag": tag, "prev": self.sink, "close": on_close}
|
||
frame.update(extra)
|
||
self.stack.append(frame)
|
||
self.sink = sink
|
||
return frame
|
||
|
||
def _emit(self, node):
|
||
if self.sink is not None:
|
||
self.sink.append(node)
|
||
|
||
# -- HTMLParser hooks -------------------------------------------------
|
||
|
||
def handle_starttag(self, tag, attrs):
|
||
attrs = dict(attrs)
|
||
classes = (attrs.get("class") or "").split()
|
||
cls = classes[0] if classes else ""
|
||
|
||
if tag in ("br", "hr", "img", "meta", "link", "input"):
|
||
if tag == "br":
|
||
self._emit({"t": "br"})
|
||
return
|
||
|
||
if not self.in_main:
|
||
# Everything before <div class="main"> is <head> and top nav.
|
||
if tag == "div" and "main" in classes:
|
||
self.in_main = True
|
||
self._push(tag, None)
|
||
return
|
||
|
||
if cls in DROP_CLASSES:
|
||
if cls in ("toc", "toc1", "toc2"):
|
||
self.is_toc = True
|
||
self._push(tag, None)
|
||
return
|
||
|
||
if tag == "div" and cls == "chapterlabel":
|
||
buf = []
|
||
self._push(tag, buf, self._close_chapter_label, nodes=buf)
|
||
return
|
||
|
||
if tag == "div" and cls in BLOCK_TYPES:
|
||
block = {"type": BLOCK_TYPES[cls], "style": cls, "content": []}
|
||
if cls in INDENT:
|
||
block["indent"] = INDENT[cls]
|
||
if cls in LEVEL:
|
||
block["level"] = LEVEL[cls]
|
||
self._push(tag, block["content"], self._close_block, block=block)
|
||
return
|
||
|
||
if tag == "p" and cls in ("f", "x"):
|
||
note = {
|
||
"id": attrs.get("id"),
|
||
"kind": "footnote" if cls == "f" else "xref",
|
||
"marker": None,
|
||
"ref": None,
|
||
"verse": None,
|
||
"content": [],
|
||
}
|
||
self._push(tag, note["content"], self._close_note, note=note)
|
||
return
|
||
|
||
if tag == "span" and "verse" in classes:
|
||
buf = []
|
||
self._push(tag, buf, self._close_verse, nodes=buf)
|
||
return
|
||
|
||
if tag == "span" and cls == "notemark":
|
||
# The note's own marker inside the footnote list.
|
||
buf = []
|
||
self._push(tag, buf, self._close_note_marker, nodes=buf)
|
||
return
|
||
|
||
if tag == "a" and cls == "notemark":
|
||
# An in-text marker linking down to the note; its nested
|
||
# <span class="popup"> duplicates the note body and is dropped.
|
||
node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")}
|
||
self._emit(node)
|
||
buf = []
|
||
self._push(tag, buf, self._close_noteref, node=node, nodes=buf)
|
||
return
|
||
|
||
if tag == "a" and cls == "notebackref":
|
||
buf = []
|
||
self._push(tag, buf, self._close_backref, nodes=buf)
|
||
return
|
||
|
||
if tag == "span" and cls in INLINE:
|
||
kind = INLINE[cls]
|
||
if kind is None:
|
||
self._push(tag, self.sink) # unwrap
|
||
else:
|
||
node = {"t": kind, "c": []}
|
||
self._emit(node)
|
||
self._push(tag, node["c"])
|
||
return
|
||
|
||
if tag == "a":
|
||
ref = parse_href(attrs.get("href"))
|
||
if ref is not None and self.sink is not None:
|
||
node = {"t": "ref", "c": []}
|
||
node.update(ref)
|
||
self._emit(node)
|
||
self._push(tag, node["c"])
|
||
else:
|
||
self._push(tag, self.sink)
|
||
return
|
||
|
||
# Anything else (nested spans, stray markup) is transparent.
|
||
self._push(tag, self.sink)
|
||
|
||
def handle_endtag(self, tag):
|
||
if not self.in_main or not self.stack:
|
||
return
|
||
for i in range(len(self.stack) - 1, -1, -1):
|
||
if self.stack[i]["tag"] == tag:
|
||
break
|
||
else:
|
||
return # unmatched close tag; ignore
|
||
while len(self.stack) > i:
|
||
frame = self.stack.pop()
|
||
self.sink = frame["prev"]
|
||
if frame["close"]:
|
||
frame["close"](frame)
|
||
if tag == "div" and not self.stack:
|
||
self.in_main = False # <div class="main"> closed
|
||
|
||
def handle_data(self, data):
|
||
if self.sink is not None and data:
|
||
self.sink.append({"t": "text", "v": data})
|
||
|
||
# -- frame close handlers ---------------------------------------------
|
||
|
||
def _close_chapter_label(self, frame):
|
||
self.chapter_label = plain_text(frame["nodes"]).strip()
|
||
|
||
def _close_block(self, frame):
|
||
block = frame["block"]
|
||
block["content"] = normalize(block["content"])
|
||
if block["type"] == "booktitle":
|
||
if block["content"]:
|
||
self.titles.append({
|
||
"style": block["style"],
|
||
"level": block.get("level"),
|
||
"text": plain_text(block["content"]),
|
||
})
|
||
return
|
||
if block["type"] == "break":
|
||
block.pop("content")
|
||
self.blocks.append(block)
|
||
return
|
||
if block["content"]:
|
||
self.blocks.append(block)
|
||
|
||
def _close_note(self, frame):
|
||
note = frame["note"]
|
||
note["content"] = normalize(note["content"])
|
||
note["text"] = plain_text(note["content"])
|
||
self.notes.append(note)
|
||
|
||
def _close_verse(self, frame):
|
||
label = plain_text(frame["nodes"]).replace(" ", " ").strip()
|
||
node = {"t": "verse", "label": label}
|
||
m = re.match(r"^(\d+)(?:[-–](\d+))?$", label)
|
||
if m:
|
||
node["n"] = int(m.group(1))
|
||
if m.group(2):
|
||
node["end"] = int(m.group(2))
|
||
self._emit(node)
|
||
|
||
def _close_note_marker(self, frame):
|
||
marker = plain_text(frame["nodes"]).strip()
|
||
for f in reversed(self.stack):
|
||
if "note" in f:
|
||
f["note"]["marker"] = marker
|
||
return
|
||
|
||
def _close_noteref(self, frame):
|
||
frame["node"]["marker"] = plain_text(frame["nodes"]).strip()
|
||
|
||
def _close_backref(self, frame):
|
||
text = plain_text(frame["nodes"]).strip()
|
||
for f in reversed(self.stack):
|
||
if "note" in f:
|
||
f["note"]["ref"] = text
|
||
m = re.search(r":(\d+)", text)
|
||
if m:
|
||
f["note"]["verse"] = int(m.group(1))
|
||
return
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Corpus
|
||
# --------------------------------------------------------------------------
|
||
|
||
class IndexParser(HTMLParser):
|
||
"""Read book order, display names and canon section from index.htm."""
|
||
|
||
# Link class on the site index marks which part of the canon a book is in.
|
||
SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"}
|
||
|
||
def __init__(self):
|
||
super().__init__(convert_charrefs=True)
|
||
self.books = [] # [{code, name, section, order}]
|
||
self.title = None
|
||
self._link = None
|
||
self._in_title = False
|
||
|
||
def handle_starttag(self, tag, attrs):
|
||
attrs = dict(attrs)
|
||
if tag == "title":
|
||
self._in_title = True
|
||
elif tag == "a":
|
||
m = CHAPTER_RE.match(os.path.basename(attrs.get("href") or ""))
|
||
section = self.SECTIONS.get((attrs.get("class") or "").strip())
|
||
if m and section:
|
||
self._link = {
|
||
"code": m.group(1),
|
||
"section": section,
|
||
"order": len(self.books) + 1,
|
||
"name": "",
|
||
}
|
||
|
||
def handle_endtag(self, tag):
|
||
if tag == "title":
|
||
self._in_title = False
|
||
elif tag == "a" and self._link:
|
||
self._link["name"] = self._link["name"].strip()
|
||
self.books.append(self._link)
|
||
self._link = None
|
||
|
||
def handle_data(self, data):
|
||
if self._in_title and self.title is None:
|
||
self.title = data.strip()
|
||
elif self._link is not None:
|
||
self._link["name"] += data
|
||
|
||
|
||
def verse_index(blocks):
|
||
"""Flat {verse label: text} map, for search and direct verse lookup.
|
||
|
||
Verse markers are inline, so a verse's text runs from its marker to the
|
||
next one and may span several blocks (a poetry couplet, for instance).
|
||
"""
|
||
verses = {}
|
||
current = None
|
||
parts = []
|
||
|
||
def flush():
|
||
if current is not None:
|
||
text = re.sub(r"\s+", " ", "".join(parts)).strip()
|
||
if text:
|
||
verses[current] = (verses.get(current, "") + " " + text).strip()
|
||
|
||
def walk(nodes):
|
||
nonlocal current
|
||
for node in nodes:
|
||
if node["t"] == "verse":
|
||
flush()
|
||
parts.clear()
|
||
current = node["label"]
|
||
elif node["t"] == "noteref":
|
||
continue
|
||
elif node["t"] == "text":
|
||
parts.append(node["v"])
|
||
elif node["t"] == "br":
|
||
parts.append(" ")
|
||
elif "c" in node:
|
||
walk(node["c"])
|
||
|
||
for block in blocks:
|
||
content = block.get("content")
|
||
if not content:
|
||
continue
|
||
walk(content)
|
||
parts.append(" ") # blocks do not run words together
|
||
flush()
|
||
return verses
|
||
|
||
|
||
def parse_chapter(path):
|
||
parser = ChapterParser()
|
||
with open(path, encoding="utf-8-sig") as fh:
|
||
parser.feed(fh.read())
|
||
parser.close()
|
||
return parser
|
||
|
||
|
||
def extract(source_dir, with_verse_text=True):
|
||
index_path = os.path.join(source_dir, "index.htm")
|
||
index = IndexParser()
|
||
with open(index_path, encoding="utf-8-sig") as fh:
|
||
index.feed(fh.read())
|
||
|
||
meta = {b["code"]: b for b in index.books}
|
||
|
||
# Group chapter files by book code.
|
||
by_book = {}
|
||
for name in sorted(os.listdir(source_dir)):
|
||
m = CHAPTER_RE.match(name)
|
||
if m:
|
||
by_book.setdefault(m.group(1), []).append((int(m.group(2)), name))
|
||
|
||
books = []
|
||
skipped = []
|
||
for code in sorted(by_book, key=lambda c: (meta.get(c, {}).get("order", 999), c)):
|
||
info = meta.get(code, {"code": code, "name": code, "section": "other",
|
||
"order": 999})
|
||
book = {
|
||
"code": code,
|
||
"name": info["name"],
|
||
"section": info["section"],
|
||
"titles": [],
|
||
"chapters": [],
|
||
}
|
||
for number, name in sorted(by_book[code]):
|
||
parsed = parse_chapter(os.path.join(source_dir, name))
|
||
if parsed.is_toc or not parsed.blocks:
|
||
# Chapter-0 files are per-book tables of contents, not text.
|
||
skipped.append(name)
|
||
continue
|
||
if parsed.titles and not book["titles"]:
|
||
book["titles"] = parsed.titles
|
||
label = parsed.chapter_label
|
||
chapter = {
|
||
"number": int(label) if label and label.isdigit() else number,
|
||
"blocks": parsed.blocks,
|
||
"notes": parsed.notes,
|
||
}
|
||
if with_verse_text:
|
||
chapter["verses"] = verse_index(parsed.blocks)
|
||
book["chapters"].append(chapter)
|
||
if book["chapters"]:
|
||
books.append(book)
|
||
|
||
return {
|
||
"translation": {
|
||
"id": "engwebu",
|
||
"name": index.title or "World English Bible Updated",
|
||
"license": "Public Domain",
|
||
"source": "eBible.org",
|
||
},
|
||
"books": books,
|
||
}, skipped
|
||
|
||
|
||
def write_split(data, out_dir):
|
||
"""One JSON per chapter plus an index, for chapter-at-a-time fetching."""
|
||
os.makedirs(out_dir, exist_ok=True)
|
||
index = {"translation": data["translation"], "books": []}
|
||
for book in data["books"]:
|
||
book_dir = os.path.join(out_dir, book["code"])
|
||
os.makedirs(book_dir, exist_ok=True)
|
||
for chapter in book["chapters"]:
|
||
payload = dict(chapter)
|
||
payload["book"] = book["code"]
|
||
path = os.path.join(book_dir, "%d.json" % chapter["number"])
|
||
with open(path, "w", encoding="utf-8") as fh:
|
||
json.dump(payload, fh, ensure_ascii=False, separators=(",", ":"))
|
||
index["books"].append({
|
||
"code": book["code"],
|
||
"name": book["name"],
|
||
"section": book["section"],
|
||
"titles": book["titles"],
|
||
"chapters": [c["number"] for c in book["chapters"]],
|
||
})
|
||
with open(os.path.join(out_dir, "index.json"), "w", encoding="utf-8") as fh:
|
||
json.dump(index, fh, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main(argv=None):
|
||
ap = argparse.ArgumentParser(description=__doc__,
|
||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
ap.add_argument("source", nargs="?", default="source",
|
||
help="directory of eBible .htm files (default: source)")
|
||
ap.add_argument("-o", "--output", default="bible.json",
|
||
help="output JSON file (default: bible.json)")
|
||
ap.add_argument("--split", metavar="DIR",
|
||
help="also write one JSON per chapter into DIR")
|
||
ap.add_argument("--no-verse-text", action="store_true",
|
||
help="omit the flat per-chapter verse text map")
|
||
ap.add_argument("--pretty", action="store_true",
|
||
help="indent the output JSON")
|
||
args = ap.parse_args(argv)
|
||
|
||
data, skipped = extract(args.source, with_verse_text=not args.no_verse_text)
|
||
|
||
dump = dict(indent=2) if args.pretty else dict(separators=(",", ":"))
|
||
with open(args.output, "w", encoding="utf-8") as fh:
|
||
json.dump(data, fh, ensure_ascii=False, **dump)
|
||
|
||
if args.split:
|
||
write_split(data, args.split)
|
||
|
||
chapters = sum(len(b["chapters"]) for b in data["books"])
|
||
notes = sum(len(c["notes"]) for b in data["books"] for c in b["chapters"])
|
||
blocks = sum(len(c["blocks"]) for b in data["books"] for c in b["chapters"])
|
||
size = os.path.getsize(args.output) / 1e6
|
||
print("books %d" % len(data["books"]), file=sys.stderr)
|
||
print("chapters %d" % chapters, file=sys.stderr)
|
||
print("blocks %d" % blocks, file=sys.stderr)
|
||
print("notes %d" % notes, file=sys.stderr)
|
||
print("skipped %d (contents pages)" % len(skipped), file=sys.stderr)
|
||
print("wrote %s (%.1f MB)" % (args.output, size), file=sys.stderr)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|