#!/usr/bin/env python3
"""Extract the eBible.org HTML Bible export into structured JSON.

The source is a flat directory of generated chapter files (``JHN03.htm``) plus
per-book and site index pages. Each chapter file is a flat sequence of
``<div class='...'>`` blocks whose class is the USFM paragraph marker (``p``,
``q2``, ``s``, ...), with verse numbers, footnote markers and character styles
as inline spans. Footnotes and cross references are collected in a trailing
``<div class="footnote">``.

This script turns that into a JSON tree of blocks and inline nodes, so the
markup can be re-rendered from templates rather than scraped again. Navigation
lists and the copyright footer are dropped; footnotes are kept.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
from html.parser import HTMLParser

# Chapter files are a three-character book code plus a zero-padded chapter
# number (three digits for Psalms).
CHAPTER_RE = re.compile(r"^([0-9A-Z]{3})(\d{2,3})\.htm$")

# Links between chapters: "JOB05.htm#V13", "JHN03.htm", or a bare "#V7".
HREF_RE = re.compile(r"^(?:([0-9A-Z]{3})(\d{2,3})\.htm)?(?:#V(\d+))?$")

# Block-level div classes -> semantic block type.
BLOCK_TYPES = {
    "p": "paragraph",    # normal prose paragraph
    "m": "paragraph",    # continuation, no first-line indent
    "nb": "paragraph",   # no break with previous
    "pi": "paragraph",   # indented
    "mi": "paragraph",   # indented, no first-line indent
    "pc": "paragraph",   # centered
    "ip": "paragraph",   # introduction paragraph
    "q": "line",         # poetry line
    "q2": "line",
    "q3": "line",
    "qs": "line",        # Selah
    "b": "break",        # blank line between stanzas
    "d": "title",        # descriptive title (psalm superscription, acrostic letter)
    "s": "heading",      # section heading
    "ms": "heading",     # major section heading ("BOOK 3")
    "is": "heading",     # introduction heading
    "sp": "speaker",     # speaker label (Song of Solomon)
    "li": "item",        # list item
    "ili": "item",       # introduction list item (glossary entries)
    "mt": "booktitle",
    "mt2": "booktitle",
    "mt3": "booktitle",
}

# Nesting depth for poetry lines and indented paragraphs.
INDENT = {"q": 1, "q2": 2, "q3": 3, "qs": 3, "pi": 1, "mi": 1, "li": 1, "ili": 1}

# Heading / title rank.
LEVEL = {"mt": 1, "mt2": 2, "mt3": 3, "ms": 1, "s": 2, "is": 2}

# Character-style spans -> inline node type. ``None`` means unwrap the span and
# keep its children inline (``ft`` is just "footnote body text").
INLINE = {
    "wj": "wj",        # words of Jesus
    "k": "keyword",    # glossary keyword
    "bk": "bookref",   # book name cited in a note
    "fq": "fq",        # the quoted words the footnote is about
    "fqa": "fqa",      # an alternative rendering
    "fl": "fl",        # footnote label
    "fine": "fine",
    "xt": "xt",        # cross-reference target
    "ft": None,
}

# Divs whose entire subtree is navigation or licensing, not content.
DROP_CLASSES = {"copyright", "tnav", "vnav", "popup", "toc", "toc1", "toc2"}


def parse_href(href):
    """Turn an intra-corpus link into a structured reference, or None."""
    m = HREF_RE.match(href or "")
    if not m or not any(m.groups()):
        return None
    book, chapter, verse = m.groups()
    ref = {}
    if book:
        ref["book"] = book
        ref["chapter"] = int(chapter)
    if verse:
        ref["verse"] = int(verse)
    return ref or None


# --------------------------------------------------------------------------
# Whitespace
# --------------------------------------------------------------------------

def normalize(nodes):
    """Collapse whitespace across the whole inline tree, in document order.

    Runs of whitespace become a single space and spaces are not duplicated
    across element boundaries, so "word</span> <span>next" keeps exactly one.
    """
    state = {"prev_space": True}  # True at the start suppresses a leading space

    def walk(lst):
        out = []
        for node in lst:
            if node["t"] == "text":
                text = re.sub(r"[\s ]+", " ", node["v"])
                if state["prev_space"]:
                    text = text.lstrip(" ")
                if text:
                    state["prev_space"] = text.endswith(" ")
                    out.append({"t": "text", "v": text})
            elif "c" in node:
                node["c"] = walk(node["c"])
                if node["c"]:
                    out.append(node)
            else:
                state["prev_space"] = False
                out.append(node)
        return out

    result = walk(nodes)
    rstrip_last(result)
    return result


def rstrip_last(nodes):
    """Drop the trailing space on a tree's final text node.

    Only a genuinely trailing node is stripped. A space before a block's
    closing verse or note marker separates it from the preceding word and
    has to survive, so the walk stops at the last node rather than looking
    past markers for something to strip.
    """
    if not nodes:
        return False
    node = nodes[-1]
    if node["t"] == "text":
        node["v"] = node["v"].rstrip()
        if not node["v"]:
            nodes.pop()
            return rstrip_last(nodes)
        return True
    if "c" in node:
        return rstrip_last(node["c"])
    return False


def plain_text(nodes, skip=("noteref",)):
    """Flatten an inline tree to readable text, dropping note markers."""
    parts = []
    for node in nodes:
        if node["t"] in skip:
            continue
        if node["t"] == "text":
            parts.append(node["v"])
        elif node["t"] == "br":
            parts.append(" ")
        elif "c" in node:
            parts.append(plain_text(node["c"], skip))
    return "".join(parts)


# --------------------------------------------------------------------------
# Chapter parser
# --------------------------------------------------------------------------

class ChapterParser(HTMLParser):
    """Parse one chapter file into blocks + notes.

    Content is appended to ``self.sink``, the inline-node list currently being
    filled. A stack of frames records what to restore (and what to finish) as
    each element closes; ``sink is None`` discards text, which is how
    navigation and the copyright footer are dropped.
    """

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.blocks = []
        self.notes = []
        self.titles = []          # mt/mt2/mt3, hoisted to the book
        self.chapter_label = None
        self.is_toc = False       # a chapter-0 navigation page

        self.in_main = False
        self.stack = []
        self.sink = None

    # -- frame helpers ----------------------------------------------------

    def _push(self, tag, sink, on_close=None, **extra):
        frame = {"tag": tag, "prev": self.sink, "close": on_close}
        frame.update(extra)
        self.stack.append(frame)
        self.sink = sink
        return frame

    def _emit(self, node):
        if self.sink is not None:
            self.sink.append(node)

    # -- HTMLParser hooks -------------------------------------------------

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        classes = (attrs.get("class") or "").split()
        cls = classes[0] if classes else ""

        if tag in ("br", "hr", "img", "meta", "link", "input"):
            if tag == "br":
                self._emit({"t": "br"})
            return

        if not self.in_main:
            # Everything before <div class="main"> is <head> and top nav.
            if tag == "div" and "main" in classes:
                self.in_main = True
                self._push(tag, None)
            return

        if cls in DROP_CLASSES:
            if cls in ("toc", "toc1", "toc2"):
                self.is_toc = True
            self._push(tag, None)
            return

        if tag == "div" and cls == "chapterlabel":
            buf = []
            self._push(tag, buf, self._close_chapter_label, nodes=buf)
            return

        if tag == "div" and cls in BLOCK_TYPES:
            block = {"type": BLOCK_TYPES[cls], "style": cls, "content": []}
            if cls in INDENT:
                block["indent"] = INDENT[cls]
            if cls in LEVEL:
                block["level"] = LEVEL[cls]
            self._push(tag, block["content"], self._close_block, block=block)
            return

        if tag == "p" and cls in ("f", "x"):
            note = {
                "id": attrs.get("id"),
                "kind": "footnote" if cls == "f" else "xref",
                "marker": None,
                "ref": None,
                "verse": None,
                "content": [],
            }
            self._push(tag, note["content"], self._close_note, note=note)
            return

        if tag == "span" and "verse" in classes:
            buf = []
            self._push(tag, buf, self._close_verse, nodes=buf)
            return

        if tag == "span" and cls == "notemark":
            # The note's own marker inside the footnote list.
            buf = []
            self._push(tag, buf, self._close_note_marker, nodes=buf)
            return

        if tag == "a" and cls == "notemark":
            # An in-text marker linking down to the note; its nested
            # <span class="popup"> duplicates the note body and is dropped.
            node = {"t": "noteref", "id": (attrs.get("href") or "").lstrip("#")}
            self._emit(node)
            buf = []
            self._push(tag, buf, self._close_noteref, node=node, nodes=buf)
            return

        if tag == "a" and cls == "notebackref":
            buf = []
            self._push(tag, buf, self._close_backref, nodes=buf)
            return

        if tag == "span" and cls in INLINE:
            kind = INLINE[cls]
            if kind is None:
                self._push(tag, self.sink)  # unwrap
            else:
                node = {"t": kind, "c": []}
                self._emit(node)
                self._push(tag, node["c"])
            return

        if tag == "a":
            ref = parse_href(attrs.get("href"))
            if ref is not None and self.sink is not None:
                node = {"t": "ref", "c": []}
                node.update(ref)
                self._emit(node)
                self._push(tag, node["c"])
            else:
                self._push(tag, self.sink)
            return

        # Anything else (nested spans, stray markup) is transparent.
        self._push(tag, self.sink)

    def handle_endtag(self, tag):
        if not self.in_main or not self.stack:
            return
        for i in range(len(self.stack) - 1, -1, -1):
            if self.stack[i]["tag"] == tag:
                break
        else:
            return  # unmatched close tag; ignore
        while len(self.stack) > i:
            frame = self.stack.pop()
            self.sink = frame["prev"]
            if frame["close"]:
                frame["close"](frame)
        if tag == "div" and not self.stack:
            self.in_main = False  # <div class="main"> closed

    def handle_data(self, data):
        if self.sink is not None and data:
            self.sink.append({"t": "text", "v": data})

    # -- frame close handlers ---------------------------------------------

    def _close_chapter_label(self, frame):
        self.chapter_label = plain_text(frame["nodes"]).strip()

    def _close_block(self, frame):
        block = frame["block"]
        block["content"] = normalize(block["content"])
        if block["type"] == "booktitle":
            if block["content"]:
                self.titles.append({
                    "style": block["style"],
                    "level": block.get("level"),
                    "text": plain_text(block["content"]),
                })
            return
        if block["type"] == "break":
            block.pop("content")
            self.blocks.append(block)
            return
        if block["content"]:
            self.blocks.append(block)

    def _close_note(self, frame):
        note = frame["note"]
        note["content"] = normalize(note["content"])
        note["text"] = plain_text(note["content"])
        self.notes.append(note)

    def _close_verse(self, frame):
        label = plain_text(frame["nodes"]).replace(" ", " ").strip()
        node = {"t": "verse", "label": label}
        m = re.match(r"^(\d+)(?:[-–](\d+))?$", label)
        if m:
            node["n"] = int(m.group(1))
            if m.group(2):
                node["end"] = int(m.group(2))
        self._emit(node)

    def _close_note_marker(self, frame):
        marker = plain_text(frame["nodes"]).strip()
        for f in reversed(self.stack):
            if "note" in f:
                f["note"]["marker"] = marker
                return

    def _close_noteref(self, frame):
        frame["node"]["marker"] = plain_text(frame["nodes"]).strip()

    def _close_backref(self, frame):
        text = plain_text(frame["nodes"]).strip()
        for f in reversed(self.stack):
            if "note" in f:
                f["note"]["ref"] = text
                m = re.search(r":(\d+)", text)
                if m:
                    f["note"]["verse"] = int(m.group(1))
                return


# --------------------------------------------------------------------------
# Corpus
# --------------------------------------------------------------------------

class IndexParser(HTMLParser):
    """Read book order, display names and canon section from index.htm."""

    # Link class on the site index marks which part of the canon a book is in.
    SECTIONS = {"oo": "ot", "nn": "nt", "aa": "deuterocanon", "xx": "other"}

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.books = []       # [{code, name, section, order}]
        self.title = None
        self._link = None
        self._in_title = False

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        if tag == "title":
            self._in_title = True
        elif tag == "a":
            m = CHAPTER_RE.match(os.path.basename(attrs.get("href") or ""))
            section = self.SECTIONS.get((attrs.get("class") or "").strip())
            if m and section:
                self._link = {
                    "code": m.group(1),
                    "section": section,
                    "order": len(self.books) + 1,
                    "name": "",
                }

    def handle_endtag(self, tag):
        if tag == "title":
            self._in_title = False
        elif tag == "a" and self._link:
            self._link["name"] = self._link["name"].strip()
            self.books.append(self._link)
            self._link = None

    def handle_data(self, data):
        if self._in_title and self.title is None:
            self.title = data.strip()
        elif self._link is not None:
            self._link["name"] += data


def verse_index(blocks):
    """Flat {verse label: text} map, for search and direct verse lookup.

    Verse markers are inline, so a verse's text runs from its marker to the
    next one and may span several blocks (a poetry couplet, for instance).
    """
    verses = {}
    current = None
    parts = []

    def flush():
        if current is not None:
            text = re.sub(r"\s+", " ", "".join(parts)).strip()
            if text:
                verses[current] = (verses.get(current, "") + " " + text).strip()

    def walk(nodes):
        nonlocal current
        for node in nodes:
            if node["t"] == "verse":
                flush()
                parts.clear()
                current = node["label"]
            elif node["t"] == "noteref":
                continue
            elif node["t"] == "text":
                parts.append(node["v"])
            elif node["t"] == "br":
                parts.append(" ")
            elif "c" in node:
                walk(node["c"])

    for block in blocks:
        content = block.get("content")
        if not content:
            continue
        walk(content)
        parts.append(" ")  # blocks do not run words together
    flush()
    return verses


def parse_chapter(path):
    parser = ChapterParser()
    with open(path, encoding="utf-8-sig") as fh:
        parser.feed(fh.read())
    parser.close()
    return parser


def extract(source_dir, with_verse_text=True):
    index_path = os.path.join(source_dir, "index.htm")
    index = IndexParser()
    with open(index_path, encoding="utf-8-sig") as fh:
        index.feed(fh.read())

    meta = {b["code"]: b for b in index.books}

    # Group chapter files by book code.
    by_book = {}
    for name in sorted(os.listdir(source_dir)):
        m = CHAPTER_RE.match(name)
        if m:
            by_book.setdefault(m.group(1), []).append((int(m.group(2)), name))

    books = []
    skipped = []
    for code in sorted(by_book, key=lambda c: (meta.get(c, {}).get("order", 999), c)):
        info = meta.get(code, {"code": code, "name": code, "section": "other",
                               "order": 999})
        book = {
            "code": code,
            "name": info["name"],
            "section": info["section"],
            "titles": [],
            "chapters": [],
        }
        for number, name in sorted(by_book[code]):
            parsed = parse_chapter(os.path.join(source_dir, name))
            if parsed.is_toc or not parsed.blocks:
                # Chapter-0 files are per-book tables of contents, not text.
                skipped.append(name)
                continue
            if parsed.titles and not book["titles"]:
                book["titles"] = parsed.titles
            label = parsed.chapter_label
            chapter = {
                "number": int(label) if label and label.isdigit() else number,
                "blocks": parsed.blocks,
                "notes": parsed.notes,
            }
            if with_verse_text:
                chapter["verses"] = verse_index(parsed.blocks)
            book["chapters"].append(chapter)
        if book["chapters"]:
            books.append(book)

    return {
        "translation": {
            "id": "engwebu",
            "name": index.title or "World English Bible Updated",
            "license": "Public Domain",
            "source": "eBible.org",
        },
        "books": books,
    }, skipped


def write_split(data, out_dir):
    """One JSON per chapter plus an index, for chapter-at-a-time fetching."""
    os.makedirs(out_dir, exist_ok=True)
    index = {"translation": data["translation"], "books": []}
    for book in data["books"]:
        book_dir = os.path.join(out_dir, book["code"])
        os.makedirs(book_dir, exist_ok=True)
        for chapter in book["chapters"]:
            payload = dict(chapter)
            payload["book"] = book["code"]
            path = os.path.join(book_dir, "%d.json" % chapter["number"])
            with open(path, "w", encoding="utf-8") as fh:
                json.dump(payload, fh, ensure_ascii=False, separators=(",", ":"))
        index["books"].append({
            "code": book["code"],
            "name": book["name"],
            "section": book["section"],
            "titles": book["titles"],
            "chapters": [c["number"] for c in book["chapters"]],
        })
    with open(os.path.join(out_dir, "index.json"), "w", encoding="utf-8") as fh:
        json.dump(index, fh, ensure_ascii=False, indent=2)


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("source", nargs="?", default="source",
                    help="directory of eBible .htm files (default: source)")
    ap.add_argument("-o", "--output", default="bible.json",
                    help="output JSON file (default: bible.json)")
    ap.add_argument("--split", metavar="DIR",
                    help="also write one JSON per chapter into DIR")
    ap.add_argument("--no-verse-text", action="store_true",
                    help="omit the flat per-chapter verse text map")
    ap.add_argument("--pretty", action="store_true",
                    help="indent the output JSON")
    args = ap.parse_args(argv)

    data, skipped = extract(args.source, with_verse_text=not args.no_verse_text)

    dump = dict(indent=2) if args.pretty else dict(separators=(",", ":"))
    with open(args.output, "w", encoding="utf-8") as fh:
        json.dump(data, fh, ensure_ascii=False, **dump)

    if args.split:
        write_split(data, args.split)

    chapters = sum(len(b["chapters"]) for b in data["books"])
    notes = sum(len(c["notes"]) for b in data["books"] for c in b["chapters"])
    blocks = sum(len(c["blocks"]) for b in data["books"] for c in b["chapters"])
    size = os.path.getsize(args.output) / 1e6
    print("books    %d" % len(data["books"]), file=sys.stderr)
    print("chapters %d" % chapters, file=sys.stderr)
    print("blocks   %d" % blocks, file=sys.stderr)
    print("notes    %d" % notes, file=sys.stderr)
    print("skipped  %d (contents pages)" % len(skipped), file=sys.stderr)
    print("wrote    %s (%.1f MB)" % (args.output, size), file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())