"""PDF converter: main pipeline orchestrating all extractors."""
from __future__ import annotations

from pathlib import Path
import re as _re
import fitz

from .base import BaseConverter
from ..types import (
    ConversionResult, PipelineConfig, DocumentBlock, BlockType,
    Table, Figure, Reference, BBox, TextLine,
)
from ..extractors.layout import (
    extract_lines_from_page, extract_spans_from_page,
    classify_lines, classify_page_with_columns,
    detect_page_columns, sort_blocks_reading_order,
    _dehyphenate, _is_allcaps_heading,
)
from ..extractors.tables import extract_tables, table_to_pipe
from ..extractors.equations import (
    extract_inline_math, is_display_equation, extract_display_equation,
    is_math_span, _equation_complexity,
)
from ..extractors.figures import (
    extract_images, extract_vector_figures,
    find_captions, associate_captions, figure_to_markdown,
)
from ..extractors.references import (
    find_references_section, extract_reference_entries,
    parse_reference, references_to_markdown,
)
from ..utils.fonts import build_font_hierarchy, FontHierarchy
from ..utils.geometry import merge_bboxes, vertical_distance


def _merge_consecutive_blocks(blocks: list[DocumentBlock]) -> list[DocumentBlock]:
    """Merge consecutive blocks of the same heading type on the same page.

    Also handles multi-line headings separated by intervening non-heading
    blocks from a different column (e.g. algorithm boxes alongside section
    headings in two-column layouts).
    """
    if not blocks:
        return blocks

    heading_types = {BlockType.TITLE, BlockType.HEADING, BlockType.SUBHEADING}

    def _should_merge_headings(prev: DocumentBlock, cur: DocumentBlock) -> bool:
        if not (cur.block_type in heading_types
                and prev.block_type == cur.block_type
                and cur.page_num == prev.page_num
                and cur.level == prev.level):
            return False
        # Don't merge if dominant font sizes differ significantly
        # (e.g., title at 13pt vs author names at 10pt)
        if prev.lines and cur.lines:
            prev_size = prev.lines[-1].dominant_size
            cur_size = cur.lines[0].dominant_size
            if abs(prev_size - cur_size) > 1.5:
                return False
        gap = cur.bbox.y0 - prev.bbox.y1
        same_row = (abs(cur.bbox.y0 - prev.bbox.y0) < 5
                     or prev.bbox.y0 <= cur.bbox.y0 <= prev.bbox.y1)
        return same_row or (0 <= gap < 25)

    # Common compound prefixes that form real hyphenated words (keep hyphen)
    _COMPOUND_PREFIXES = frozenset({
        "low", "high", "mid", "non", "pre", "post", "sub", "self", "full",
        "half", "fine", "long", "short", "cross", "well", "off", "out",
        "over", "under", "back", "two", "three", "one", "four", "five",
        "all", "anti", "auto", "co", "de", "ex", "re", "un", "semi",
        "multi", "bi", "tri", "super", "inter", "intra", "ultra",
        "pan", "poly", "neo", "mono", "meta", "hyper", "counter", "trans",
        "down", "up", "top", "near", "far", "left", "right", "open",
        "large", "small", "hard", "soft", "real", "task", "zero", "few",
    })

    def _merge_heading_text(prev_text: str, next_text: str) -> str:
        """Join heading lines with dehyphenation."""
        pt = prev_text.rstrip()
        nt = next_text.lstrip()
        if pt.endswith("-") and nt:
            if nt[0].islower():
                return pt[:-1] + nt  # dehyphenate: "experi-" + "ments" → "experiments"
            # Uppercase continuation: check if this is a line-break hyphenation
            # in a title-cased heading (e.g. "Lan-" + "Guage") vs a real
            # compound word (e.g. "Batch-" + "Normalized", "Low-" + "Rank").
            stub = pt.split()[-1].rstrip("-").lower() if pt.split() else ""
            if stub in _COMPOUND_PREFIXES or len(stub) >= 5:
                return pt + nt  # compound word: keep hyphen
            else:
                # Line-break: dehyphenate and normalize case of joined word.
                # "LAN" + "Guage" → "Language" (capitalize the combined word)
                prefix = pt[:pt.rfind(pt.split()[-1])]  # everything before the stub word
                joined = pt.split()[-1].rstrip("-") + nt.split()[0]
                rest = " ".join(nt.split()[1:])
                normalized = joined.capitalize()
                parts = [prefix.rstrip(), normalized]
                if rest:
                    parts.append(rest)
                return " ".join(p for p in parts if p)
        return pt + " " + nt

    merged: list[DocumentBlock] = [blocks[0]]

    i = 1
    while i < len(blocks):
        block = blocks[i]
        prev = merged[-1]

        if _should_merge_headings(prev, block):
            prev.text = _merge_heading_text(prev.text, block.text)
            prev.bbox = merge_bboxes([prev.bbox, block.bbox])
            prev.lines.extend(block.lines)
            i += 1
            continue

        # Check if an intervening non-heading block separates two heading
        # continuations (e.g. algorithm box text between two lines of a
        # section heading in a two-column layout).
        if (prev.block_type in heading_types
                and block.block_type not in heading_types
                and i + 1 < len(blocks)):
            next_block = blocks[i + 1]
            if _should_merge_headings(prev, next_block):
                # The intervening block is from a different column — keep it
                # but also merge the heading continuation.
                merged.append(block)  # keep the intervening block
                prev.text = _merge_heading_text(prev.text, next_block.text)
                prev.bbox = merge_bboxes([prev.bbox, next_block.bbox])
                prev.lines.extend(next_block.lines)
                i += 2
                continue

        merged.append(block)
        i += 1

    return merged


def _merge_cross_page_paragraphs(blocks: list[DocumentBlock]) -> list[DocumentBlock]:
    """Merge BODY blocks that were split at page boundaries mid-sentence."""
    if not blocks:
        return blocks

    merged: list[DocumentBlock] = [blocks[0]]

    for block in blocks[1:]:
        prev = merged[-1]
        # Merge if both are BODY and previous ends mid-sentence
        if (block.block_type == BlockType.BODY
                and prev.block_type == BlockType.BODY
                and block.page_num == prev.page_num + 1):
            prev_text = prev.text.rstrip()
            curr_text = block.text.lstrip()
            # Previous block ends mid-sentence (no terminal punctuation)
            if prev_text and not prev_text[-1] in '.!?:;' and curr_text and curr_text[0].islower():
                # Dehyphenate at boundary
                if prev_text.endswith('-') and curr_text[0].islower():
                    prev.text = prev_text[:-1] + curr_text
                else:
                    prev.text = prev_text + " " + curr_text
                prev.bbox = merge_bboxes([prev.bbox, block.bbox])
                prev.lines.extend(block.lines)
                continue
        merged.append(block)

    return merged


def _brackets_balanced(s: str) -> bool:
    """Check if parentheses, brackets, and braces are balanced."""
    depth = 0
    for ch in s:
        if ch in '([':
            depth += 1
        elif ch in ')]':
            depth -= 1
    return depth == 0


def _merge_adjacent_display_math(parts: list[str]) -> list[str]:
    """Merge consecutive $$...$$ block strings in a parts list.

    When a multi-line display equation is split across several lines (each
    producing a separate ``\\n$$...$$\\n`` entry), this function combines
    them into one block so the rendered markdown does not show dozens of
    tiny display equations.

    Also merges display equations with unbalanced brackets by absorbing
    subsequent display-math parts until brackets balance.
    """
    _display_re = _re.compile(
        r"^\n\$\$(.+?)\$\$([ \t]*\([\d.]+\))?\n$", _re.DOTALL
    )

    merged: list[str] = []
    for part in parts:
        m = _display_re.match(part)
        if m and merged:
            prev = merged[-1]
            pm = _display_re.match(prev)
            if pm:
                prev_content = pm.group(1).strip()
                curr_content = m.group(1).strip()
                eq_num = m.group(2) or pm.group(2) or ""
                merged[-1] = f"\n$${prev_content} {curr_content}$${eq_num}\n"
                continue
            # Merge when there's a trivial intervening part (whitespace,
            # equation number, or very short non-math text)
            if len(merged) >= 2:
                intervening = merged[-1].strip()
                prev2 = merged[-2]
                pm2 = _display_re.match(prev2)
                if pm2 and (not intervening or len(intervening) < 10):
                    prev_content = pm2.group(1).strip()
                    curr_content = m.group(1).strip()
                    eq_num = m.group(2) or pm2.group(2) or ""
                    merged[-2] = f"\n$${prev_content} {curr_content}$${eq_num}\n"
                    merged.pop(-1)
                    continue
        merged.append(part)

    # Second pass: merge display equations with unbalanced brackets
    balanced: list[str] = []
    for part in merged:
        if balanced:
            prev = balanced[-1]
            pm = _display_re.match(prev)
            m = _display_re.match(part)
            if pm and m and not _brackets_balanced(pm.group(1)):
                prev_content = pm.group(1).strip()
                curr_content = m.group(1).strip()
                eq_num = m.group(2) or pm.group(2) or ""
                balanced[-1] = f"\n$${prev_content} {curr_content}$${eq_num}\n"
                continue
        balanced.append(part)

    return balanced


def _is_metadata_line(text: str) -> bool:
    """Detect metadata lines that shouldn't be headings."""
    t = text.strip()
    if _re.match(r"^arXiv:\d", t):
        return True
    if _re.match(r"^\d{4}\.\d{4,5}", t):
        return True
    # Math/diagram fragments (high ratio of symbols to letters)
    letters = sum(1 for c in t if c.isalpha())
    symbols = sum(1 for c in t if c in '−·!→←↔≤≥=+×÷∑∏∫∇∂−')
    if len(t) > 5 and symbols > letters:
        return True
    return False


_AFFIL_KW = frozenset({
    "university", "institute", "department", "school", "college",
    "lab", "laboratory", "research", "center", "centre",
    "corporation", "inc.", "ltd.", "google", "meta", "microsoft",
    "deepmind", "openai", "facebook", "brain", "technologies",
})


_SENTENCE_STARTERS = frozenset({
    "the", "a", "an", "this", "these", "those", "we", "our", "in", "on",
    "for", "to", "it", "its", "is", "are", "was", "were", "has", "have",
    "by", "with", "from", "as", "at", "however", "moreover", "furthermore",
    "recently", "given", "let", "consider", "note", "since", "because",
    "when", "while", "although", "each", "every", "both", "all", "most",
    "many", "several", "some", "such", "here", "there",
})


def _is_author_line(text: str) -> bool:
    """Heuristic: does this text look like author names/affiliations (not body text)?

    Requires strong signals AND must not look like a body-text sentence.
    """
    t = text.strip()
    if not t:
        return False
    # Reject if text starts like a regular sentence
    first_word = t.split()[0].lower().rstrip(".,;:")
    if first_word in _SENTENCE_STARTERS:
        return False
    has_markers = bool(_re.search(r"[*†‡§¶∗⋆★✦✧]", t))
    has_email = bool(_re.search(r"\S+@\S+\.\S+", t))
    # Reject long texts that contain multiple sentences — BUT allow if
    # footnote markers or emails are present (author lists with middle
    # initials like "Tom B. Brown∗" trigger the ". [A-Z]" pattern falsely)
    if len(t) > 200 and not has_markers and not has_email:
        if len(_re.findall(r'\.\s+[A-Z]', t)) >= 2:
            return False
    # Contains email → definitely author metadata
    if has_email:
        return True
    # Contains footnote markers (common in author lines: ∗ † ‡ § ¶)
    if has_markers:
        return True
    # Contains affiliation keywords
    lower = t.lower()
    if any(kw in lower for kw in _AFFIL_KW):
        return True
    return False


# Strong affiliation indicators: any one of these in a segment means it's not a name.
# These are institutional/organizational terms that never appear in a person's name.
_STRONG_AFFIL = frozenset({
    # From _AFFIL_KW (organizational terms)
    "university", "institute", "department", "school", "college",
    "lab", "laboratory", "research", "center", "centre",
    "corporation", "inc.", "ltd.", "google", "meta", "microsoft",
    "deepmind", "openai", "facebook", "brain", "technologies",
    # Publication / preprint venues
    "arxiv", "preprint", "abstract", "proceedings", "conference",
    "ieee", "acm", "icml", "iclr", "neurips", "nips", "cvpr",
    "aaai", "emnlp", "naacl",
    # Countries / major cities / regions that appear in affiliations
    "germany", "usa", "china", "france", "canada", "japan",
    "uk", "india", "korea", "australia", "italy", "spain",
    "munich", "heidelberg", "zurich", "toronto", "ontario",
    "california", "massachusetts", "washington",
    # Other strong affiliation signals
    "http", "https", "www", "equal", "contribution", "corresponding",
    "international", "national",
    # Academic and regional institution words
    "academia", "sinica", "taiwan",
    # Specific institution names / abbreviations
    "johns", "hopkins", "carnegie", "mellon",
    "caltech", "eth", "epfl", "cmu", "uc",
    "berkeley", "stanford", "mit", "cambridge", "oxford", "princeton",
    # Company / org names that appear standalone
    "runway", "iwr",
})

# Weak affiliation indicators: two or more of these in the same segment → not a name.
# Individually they are too generic (appear in paper titles, field names, etc.).
_WEAK_AFFIL = frozenset({
    "work", "done", "author", "team",
    "ai", "ml",
    "florida", "lauderdale", "fort", "york",
    "language", "computing", "vision", "natural", "processing",
    "technical", "sciences", "engineering", "electrical",
    "computer", "science", "systems", "information",
    "geographic", "geospatial", "analytics", "biological", "biomedical",
    "signalling", "signaling", "studies", "page", "home",
    "machine", "learning", "deep", "foundation", "models", "model",
    "intelligence", "artificial", "data", "neural", "network",
    "new",
})

# Name particles: lowercase words that are legitimate parts of person names.
# A segment containing only these (plus capitalized words) is still a name candidate.
_NAME_PARTICLES = frozenset({
    "de", "von", "van", "der", "el", "al", "bin", "dos",
    "da", "di", "du", "la", "le", "del", "den", "ter",
})

# Kept for backwards compatibility with _is_author_line and any external callers.
_AFFIL_WORDS = _STRONG_AFFIL | _WEAK_AFFIL


def _split_capitalized_names(segment: str) -> list[str]:
    """Split a long segment of consecutive capitalized words into person names.

    Handles patterns like "Kaiming He Xiangyu Zhang Shaoqing Ren Jian Sun"
    by grouping into 2-word or 3-word names (when middle initials present).
    """
    words = segment.split()
    if len(words) <= 3:
        return [segment]
    # Check that most words start with uppercase
    caps_count = sum(1 for w in words if w[0].isupper())
    if caps_count < len(words) * 0.7:
        return []  # not a name list

    names: list[str] = []
    i = 0
    while i < len(words):
        w = words[i]
        if not w[0].isupper():
            i += 1
            continue
        # Collect first name + any middle initials + last name
        # e.g., "Ricky T. Q. Chen" → 4 words, "Tom B. Brown" → 3 words
        parts = [w]
        j = i + 1
        # Gather consecutive initials (single letter like "B." or "Q.")
        while j < len(words) and len(words[j].rstrip(".")) == 1 and words[j][0].isupper():
            parts.append(words[j])
            j += 1
        # If we have initials, next uppercase word is the last name
        if len(parts) > 1 and j < len(words) and words[j][0].isupper():
            parts.append(words[j])
            names.append(" ".join(parts))
            i = j + 1
        elif i + 1 < len(words) and words[i + 1][0].isupper():
            # 2-word name: "Kaiming He"
            names.append(f"{words[i]} {words[i+1]}")
            i += 2
        else:
            # Single stranded word — attach to previous name if it exists
            if names:
                names[-1] += f" {w}"
            i += 1
    return names


def _segment_is_affiliation(seg: str) -> bool:
    """Return True if the segment looks like an affiliation rather than a person name.

    Uses a tiered approach:
    - Any STRONG affiliation word → definitely an affiliation.
    - Two or more WEAK affiliation words → likely an affiliation phrase.
    - Name particles (de, von, van, …) are not counted against the segment.
    """
    words_lower = [w.lower().rstrip(".,;:") for w in seg.split()]
    strong_hits = sum(1 for w in words_lower if w in _STRONG_AFFIL)
    if strong_hits > 0:
        return True
    weak_hits = sum(1 for w in words_lower
                    if w in _WEAK_AFFIL and w not in _NAME_PARTICLES)
    return weak_hits >= 2


def _extract_names_from_text(raw: str) -> list[str]:
    """Extract person names from text containing author info mixed with affiliations.

    Uses a tiered affiliation-detection strategy:
    - STRONG affiliation words (organizational terms) immediately mark a segment as
      non-name, regardless of context.
    - WEAK affiliation words (generic field/topic terms) only filter a segment when
      two or more are present, preventing false positives from single common words.
    - Name particles (de, von, van, der, …) are explicitly allowed inside names.

    Long runs of consecutive capitalized words are split into individual names.
    """
    # Detect corporate/team authors before stripping (e.g., "DeepSeek-AI", "OpenAI")
    corporate_authors: list[str] = []
    _corp_pattern = _re.compile(
        r"\b((?:[A-Z][a-zA-Z]*[-](?:AI|ML|NLP|Lab|Labs|Team|Research))"
        r"|(?:OpenAI|DeepMind|Google Brain|Meta AI|Microsoft Research))\b"
    )
    for m in _corp_pattern.finditer(raw):
        corporate_authors.append(m.group(1))

    # Strip emails, URLs, footnote markers
    raw = _re.sub(r"\S+@\S+\.\S+", "  ", raw)
    raw = _re.sub(r"https?://\S+", "  ", raw)
    raw = _re.sub(r"[*†‡§¶∗⋆★✦✧]+", " ", raw)
    # Replace superscript-style affiliation markers with commas to separate names
    # e.g., "Yaron Lipman1,2 Ricky T. Q. Chen1" → "Yaron Lipman, Ricky T. Q. Chen"
    raw = _re.sub(r"(\w)(\d+(?:,\d+)*)\s+(?=[A-Z])", r"\1, ", raw)
    # Strip remaining trailing digits from words (footnote refs like "Rombach1")
    raw = _re.sub(r"(\w)(\d+)(?=\s|,|$)", r"\1", raw)
    # Split digit-letter compounds like "1Geographic" → " Geographic"
    raw = _re.sub(r"(\d)([A-Z])", r"\1 \2", raw)
    # Strip standalone footnote reference numbers (e.g., "Juhász 1" → "Juhász")
    raw = _re.sub(r"\s+\d{1,2}(?=\s|,|$)", " ", raw)

    # First pass: replace STRONG affiliation words with a separator so they
    # immediately break segments without needing to evaluate whole-segment context.
    words = raw.split()
    cleaned_words: list[str] = []
    for w in words:
        if w.lower().rstrip(".,;:") in _STRONG_AFFIL:
            cleaned_words.append("  ")  # separator
        else:
            cleaned_words.append(w)
    raw = " ".join(cleaned_words)

    # Split on commas, semicolons, " and ", double+ spaces, newlines
    segments = _re.split(r"[,;\n]+|\s{2,}|\s+and\s+", raw)

    names: list[str] = []
    for seg in segments:
        seg = seg.strip()
        if not seg or len(seg) < 3:
            continue
        words = seg.split()
        # Segments with 4+ words: try splitting into individual names
        if len(words) > 3:
            split = _split_capitalized_names(seg)
            if split:
                names.extend(n for n in split if not _segment_is_affiliation(n))
            continue
        # Must start with uppercase letter
        if not seg[0].isupper():
            continue
        # Skip single words (not a person name)
        if len(words) < 2:
            continue
        # Skip segments with special characters (curly braces, ampersand, etc.)
        if _re.search(r"[{}()\[\]<>&]", seg):
            continue
        # Must be mostly alphabetic
        alpha_chars = sum(1 for c in seg if c.isalpha())
        if alpha_chars < 3 or alpha_chars < len(seg.replace(" ", "")) * 0.6:
            continue
        # Skip segments where most words are ≤2 chars AND none look like
        # initials (e.g., postal codes "M5 S G") — but allow "G. E. Hinton"
        short_words = sum(1 for w in words if len(w) <= 2)
        has_initial = any(len(w) <= 2 and w.endswith('.') for w in words)
        if short_words > len(words) * 0.5 and not has_initial:
            continue
        # Apply weak affiliation filter at segment level (2+ weak words = skip)
        if _segment_is_affiliation(seg):
            continue
        names.append(seg)

    # Add corporate/team authors detected earlier (if not already in names)
    for corp in corporate_authors:
        if corp not in names:
            names.append(corp)

    return names


def _is_allcaps_name_line(text: str) -> bool:
    """Check if text looks like an ALL-CAPS person name (ACM format).

    Matches patterns like "SHRIYANK SOMVANSHI" or "SYED AAQIB JAVED".
    Returns False for common section headings and single words.
    """
    words = text.strip().split()
    if not words or len(words) < 2:
        return False
    # All words must be uppercase alphabetic (allow "PH.D." style suffixes)
    alpha_words = [w.rstrip(".,;:") for w in words if w.rstrip(".,;:").replace(".", "").isalpha()]
    if len(alpha_words) < 2:
        return False
    if not all(w.isupper() for w in alpha_words):
        return False
    # Must not be a common section heading
    common = {"ABSTRACT", "INTRODUCTION", "CONCLUSION", "CONCLUSIONS",
              "REFERENCES", "ACKNOWLEDGEMENTS", "ACKNOWLEDGMENTS",
              "RELATED WORK", "BACKGROUND", "METHODS", "RESULTS",
              "DISCUSSION", "APPENDIX", "SUPPLEMENTARY"}
    if text.strip().rstrip(".,;:") in common:
        return False
    return True


def _extract_allcaps_name_from_line(text: str) -> str | None:
    """Extract an ALL-CAPS person name from the beginning of a line.

    Handles ACM-format lines like:
      "SHRIYANK SOMVANSHI, Texas State University, TX"
      "SUBASISH DAS, PH.D., Texas State University, TX"

    Returns title-cased name like "Shriyank Somvanshi", or None if not found.
    """
    # Match 2-4 ALL-CAPS words at the start, followed by comma or end-of-string
    m = _re.match(
        r"^([A-Z][A-Z]+(?:\s+[A-Z][A-Z.]+){1,3})"
        r"(?:\s*,|\s*$)",
        text.strip(),
    )
    if not m:
        return None
    name_part = m.group(1).strip()
    # Exclude honorific suffixes from the name (only at the end, not beginning)
    _SUFFIXES = {"PHD", "PH.D", "JR", "SR", "II", "III", "IV", "MD"}
    name_words = name_part.split()
    # Strip suffixes from the end only
    while name_words and name_words[-1].rstrip(".,") in _SUFFIXES:
        name_words.pop()
    if len(name_words) < 2:
        return None
    # Check it's not a common section heading
    rejoined = " ".join(name_words)
    if not _is_allcaps_name_line(rejoined):
        return None
    # Convert to title case
    return " ".join(w.capitalize() for w in name_words)


def _is_short_name_heading(text: str) -> bool:
    """Check if a heading looks like a person name (1-4 capitalized words, no common headings)."""
    t = text.strip()
    words = t.split()
    if not words or len(words) > 4:
        return False
    _HEADING_WORDS = {"abstract", "introduction", "conclusion", "references",
                      "acknowledgements", "acknowledgments", "appendix",
                      "related", "work", "method", "methods", "results",
                      "discussion", "background", "overview", "summary"}
    if t.lower() in _HEADING_WORDS or words[0].lower() in _HEADING_WORDS:
        return False
    # All words should start with uppercase
    if not all(w[0].isupper() for w in words if w):
        return False
    # Must be mostly alphabetic
    alpha = sum(1 for c in t if c.isalpha())
    return alpha >= 3 and len(t) < 40


def _reclassify_name_headings(blocks: list[DocumentBlock]) -> list[DocumentBlock]:
    """Reclassify heading blocks on page 0 that are author names as METADATA.

    Some papers (e.g. LoRA) have author names as bold-at-body-size text without
    footnote markers. The layout classifier sees these as headings, but they're
    really metadata that should be merged with the author block.

    Strategy: find METADATA blocks on page 0 (author lines with markers). Any
    adjacent heading blocks that look like person names are also author metadata.
    """
    # Find first METADATA block on page 0
    meta_idx = None
    for i, b in enumerate(blocks):
        if b.page_num == 0 and b.block_type == BlockType.METADATA:
            meta_idx = i
            break
        if b.page_num > 0:
            break
    if meta_idx is None:
        return blocks

    # Scan forward from first METADATA to find the author zone end.
    # Stop at Abstract heading, non-name heading, or page boundary.
    end_idx = len(blocks)
    for i in range(meta_idx + 1, len(blocks)):
        b = blocks[i]
        if b.page_num > 0:
            end_idx = i
            break
        if b.block_type == BlockType.ABSTRACT:
            end_idx = i
            break
        if (b.block_type == BlockType.HEADING
                and not _is_short_name_heading(b.text)):
            end_idx = i
            break

    result = list(blocks)
    for i in range(meta_idx, end_idx):
        b = result[i]
        if (b.block_type == BlockType.HEADING
                and b.page_num == 0
                and _is_short_name_heading(b.text)):
            result[i] = DocumentBlock(
                block_type=BlockType.METADATA,
                text=b.text, bbox=b.bbox, page_num=b.page_num,
                lines=b.lines, metadata=b.metadata,
            )
    return result


def _parse_authors(blocks: list[DocumentBlock], title_block: DocumentBlock | None) -> list[str]:
    """Extract author names from METADATA, BODY, and name-like HEADING blocks between title and abstract."""
    if not title_block:
        return []

    # Find abstract position — first try explicit "Abstract" heading
    abstract_y = None
    for b in blocks:
        if (b.block_type == BlockType.HEADING
                and b.page_num == title_block.page_num
                and b.text.strip().lower() == "abstract"):
            abstract_y = b.bbox.y0
            break

    # Check for ABSTRACT block (bold-opening-paragraph detection)
    if abstract_y is None:
        for b in blocks:
            if (b.block_type == BlockType.ABSTRACT
                    and b.page_num == title_block.page_num
                    and b.bbox.y0 > title_block.bbox.y1):
                abstract_y = b.bbox.y0
                break

    # Fallback: if no "Abstract" heading, use the first HEADING on the same page
    # (e.g., "Introduction" in ACM format where abstract has no heading)
    if abstract_y is None:
        for b in blocks:
            if (b.block_type == BlockType.HEADING
                    and b.page_num == title_block.page_num
                    and b.bbox.y0 > title_block.bbox.y1):
                abstract_y = b.bbox.y0
                break

    # Collect METADATA blocks + author-like BODY blocks between title and abstract
    meta_texts: list[str] = []
    meta_blocks: list[DocumentBlock] = []
    name_headings: list[str] = []
    allcaps_body_blocks: list[DocumentBlock] = []
    for b in blocks:
        if b.page_num != title_block.page_num:
            continue
        if b.bbox.y0 <= title_block.bbox.y1:
            continue
        if abstract_y and b.bbox.y0 >= abstract_y:
            continue
        if b.block_type == BlockType.METADATA:
            meta_texts.append(b.text.strip())
            meta_blocks.append(b)
        elif b.block_type == BlockType.FOOTNOTE and _is_author_line(b.text):
            # Author lines may be classified as footnotes when their font
            # is smaller than body size (e.g., Chinchilla paper)
            meta_texts.append(b.text.strip())
            meta_blocks.append(b)
        elif b.block_type == BlockType.BODY and _is_author_line(b.text):
            meta_texts.append(b.text.strip())
        elif b.block_type == BlockType.HEADING and _is_short_name_heading(b.text):
            name_headings.append(b.text.strip())
        elif b.block_type == BlockType.BODY:
            # Track BODY blocks for ALL-CAPS fallback
            allcaps_body_blocks.append(b)

    # If we found name-like headings and no other author text, use them directly
    if not meta_texts and name_headings:
        return name_headings

    # Try ALL-CAPS author names first (ACM format)
    # These appear as BODY or METADATA blocks whose lines match "FIRSTNAME LASTNAME, Affiliation"
    names: list[str] = []
    for b in (meta_blocks + allcaps_body_blocks):
        acm_names: list[str] = []
        for line in b.lines:
            name = _extract_allcaps_name_from_line(line.text)
            if name:
                acm_names.append(name)
            else:
                break
        if acm_names:
            names = acm_names
            break

    # General extraction from meta text
    # Process each block separately first (preserves multi-word names like
    # "Hong-Yuan Mark Liao"), then fall back to joined text if no names found
    if not names and meta_texts:
        for mt in meta_texts:
            names.extend(_extract_names_from_text(mt))
        if not names:
            names = _extract_names_from_text(" ".join(meta_texts))

    # Fallback: check if the title block text itself contains author names
    # (happens when title + authors are merged in the same block, e.g. ViT)
    if not names and title_block and _re.search(r"[∗†‡⋆]", title_block.text):
        title_text = title_block.text
        # Find "Name∗," pattern — the ∗, signals start of comma-separated author list
        bound = _re.search(r"[∗†‡⋆]\s*,", title_text)
        if bound:
            # Walk backward from the marker to find the start of the first author name
            pos = bound.start()
            before = title_text[:pos]
            # Find the first author: 2-3 capitalized words right before the marker
            words_before = before.split()
            # Take last 2-3 words that look like a name (all start with uppercase)
            first_author_words: list[str] = []
            for w in reversed(words_before):
                if w and w[0].isupper() and w[0].isalpha():
                    first_author_words.insert(0, w)
                    if len(first_author_words) >= 2:
                        break
                else:
                    break
            if first_author_words:
                start_word = first_author_words[0]
                start_pos = before.rfind(start_word)
                if start_pos >= 0:
                    author_text = title_text[start_pos:]
                    extracted = _extract_names_from_text(author_text)
                    if len(extracted) >= 2:
                        names = extracted

    # Deduplicate while preserving order
    seen: set[str] = set()
    deduped: list[str] = []
    for n in names:
        if n not in seen:
            seen.add(n)
            deduped.append(n)
    return deduped


class PdfConverter(BaseConverter):
    """Convert research paper PDFs to Markdown."""

    def supports(self, path: Path) -> bool:
        return path.suffix.lower() == ".pdf"

    def _report_progress(self, stage: str, current: int, total: int) -> None:
        """Report progress via the configured callback, if any."""
        if self.config.progress_callback:
            try:
                self.config.progress_callback(stage, current, total)
            except Exception:
                pass  # never let callback errors break the pipeline

    @staticmethod
    def _detect_repeated_headers(doc: fitz.Document) -> set[str]:
        """Detect text that repeats across 3+ pages in header/footer regions."""
        from collections import Counter
        margin_texts: Counter[str] = Counter()
        for page in doc:
            h = page.rect.height
            lines = extract_lines_from_page(page)
            seen_on_page: set[str] = set()
            for l in lines:
                t = l.text.strip()
                if not t or len(t) < 3 or t in seen_on_page:
                    continue
                if l.bbox.y0 < 60 or l.bbox.y0 > h - 60:
                    seen_on_page.add(t)
                    margin_texts[t] += 1
        threshold = max(3, len(doc) * 0.4)
        return {t for t, c in margin_texts.items() if c >= threshold and not t.isdigit()}

    def convert(self, path: Path) -> ConversionResult:
        try:
            doc = fitz.open(str(path))
        except Exception as e:
            raise ValueError(f"Cannot open PDF: {e}") from e
        try:
            if doc.is_encrypted:
                raise ValueError("PDF is encrypted/password-protected")
            if doc.page_count == 0:
                raise ValueError("PDF has no pages")
            return self._convert_doc(doc)
        finally:
            doc.close()

    @staticmethod
    def _render_vector_figure(page: fitz.Page, bbox: BBox, dpi: int = 150) -> bytes | None:
        """Render a vector figure region to PNG bytes."""
        try:
            clip = fitz.Rect(bbox.x0, bbox.y0, bbox.x1, bbox.y1)
            pix = page.get_pixmap(clip=clip, dpi=dpi)
            return pix.tobytes("png")
        except Exception:
            return None

    @staticmethod
    def _is_scanned_pdf(doc: fitz.Document) -> bool:
        """Detect scanned PDFs by sampling pages for text content."""
        sample_pages = min(3, len(doc))
        for i in range(sample_pages):
            text = doc[i].get_text("text")
            if len(text.strip()) >= 50:
                return False
        return True

    @staticmethod
    def _ocr_page(page: fitz.Page, dpi: int = 300, lang: str = "eng") -> str:
        """Extract text from a scanned page using OCR.

        Tries PyMuPDF's built-in OCR first, falls back to pytesseract.
        Returns empty string if OCR is not available.
        """
        # Try PyMuPDF's built-in Tesseract integration
        try:
            tp = page.get_textpage_ocr(dpi=dpi, language=lang, full=True)
            text = page.get_text("text", textpage=tp)
            if text.strip():
                return text
        except Exception:
            pass

        # Fallback: pytesseract via pixmap
        try:
            import pytesseract
            from PIL import Image
            import io
            pix = page.get_pixmap(dpi=dpi)
            img = Image.open(io.BytesIO(pix.tobytes("png")))
            return pytesseract.image_to_string(img, lang=lang)
        except ImportError:
            return ""
        except Exception:
            return ""

    def _convert_doc(self, doc: fitz.Document) -> ConversionResult:
        # Check for scanned PDF
        if self.config.ocr_enabled and self._is_scanned_pdf(doc):
            return self._convert_scanned_doc(doc)

        # Stage 1: Extract all spans for font analysis
        all_spans = []
        page_lines_cache: list[list] = []
        for page in doc:
            all_spans.extend(extract_spans_from_page(page))

        hierarchy = build_font_hierarchy(all_spans, self.config)
        body_font = hierarchy.body_font
        page_count = len(doc)
        self._report_progress("analyzing", 0, page_count)

        # Pre-scan: detect repeated headers/footers across pages
        repeated_headers = self._detect_repeated_headers(doc)

        # Stage 2-3: Per-page extraction
        all_blocks: list[DocumentBlock] = []
        all_tables: list[Table] = []
        all_figures: list[Figure] = []
        all_captions: list[DocumentBlock] = []
        page_widths: list[float] = []

        for page_num, page in enumerate(doc):
            self._report_progress("extracting", page_num + 1, page_count)
            page_width = page.rect.width
            page_height = page.rect.height
            page_widths.append(page_width)

            lines = extract_lines_from_page(page)

            # Filter out repeated headers/footers — only in margin regions
            # (y < 60 or y > h-60) to avoid removing actual titles that match
            # running headers
            if repeated_headers:
                _h = page.rect.height
                lines = [
                    l for l in lines
                    if l.text.strip() not in repeated_headers
                    or (60 <= l.bbox.y0 <= _h - 60)
                ]

            # Tables first - we need their bboxes for exclusion
            table_bboxes: list[BBox] = []
            if self.config.extract_tables:
                tables = extract_tables(page, lines, page_num)
                all_tables.extend(tables)
                table_bboxes = [t.bbox for t in tables if t.bbox]

            # Figures (raster + vector)
            if self.config.extract_figures:
                figures = extract_images(page, page_num)
                # Vector graphics detection + rendering
                if self.config.detect_vector_figures:
                    vec_figs = extract_vector_figures(page, page_num, table_bboxes)
                    # Deduplicate: skip vector figures that overlap existing raster figures
                    for vf in vec_figs:
                        if not any(vf.bbox.overlaps(rf.bbox, threshold=0.3)
                                   for rf in figures if rf.bbox):
                            # Render vector region to PNG
                            vf.image_data = self._render_vector_figure(page, vf.bbox)
                            figures.append(vf)
                # Render clustered (multi-tile) figures as composite images
                for fig in figures:
                    if fig.xref is None and not fig.is_vector and fig.bbox:
                        fig.image_data = self._render_vector_figure(page, fig.bbox)
                captions = find_captions(lines, page_num)
                figures = associate_captions(figures, captions)
                all_figures.extend(figures)
                all_captions.extend(captions)

            # Build exclusion zones (tables + caption bboxes + vector figure bboxes)
            exclude_bboxes = list(table_bboxes)
            if self.config.extract_figures:
                for cap in captions:
                    exclude_bboxes.append(cap.bbox)
                for fig in figures:
                    if fig.is_vector and fig.bbox:
                        exclude_bboxes.append(fig.bbox)

            # Classify with column awareness
            blocks = classify_page_with_columns(
                lines, hierarchy, page_num, page_height, page_width,
                self.config, exclude_bboxes,
            )
            all_blocks.extend(blocks)

        # Pre-merge: reclassify heading blocks on page 0 between title and
        # abstract that look like author names (bold-at-body-size name lines
        # without footnote markers, e.g. LoRA paper).
        all_blocks = _reclassify_name_headings(all_blocks)

        # Post-process: merge split headings, filter metadata
        all_blocks = _merge_consecutive_blocks(all_blocks)
        all_blocks = [
            b if not (b.block_type in {BlockType.TITLE, BlockType.HEADING, BlockType.SUBHEADING}
                      and _is_metadata_line(b.text))
            else DocumentBlock(
                block_type=BlockType.METADATA,
                text=b.text, bbox=b.bbox, page_num=b.page_num,
                lines=b.lines, metadata=b.metadata,
            )
            for b in all_blocks
        ]

        # Handle title detection and arXiv cover pages
        title_block = None
        for b in all_blocks:
            if b.block_type == BlockType.TITLE:
                title_block = b
                break

        # Title on page 2+ is almost certainly a figure/chart label —
        # look for a heading on pages 0-1 to promote instead
        if title_block and title_block.page_num > 1:
            for b in all_blocks:
                if b.block_type == BlockType.HEADING and b.page_num <= 1:
                    # Demote all false TITLE blocks
                    for t in all_blocks:
                        if t.block_type == BlockType.TITLE:
                            t.block_type = BlockType.BODY
                    # Promote the early heading
                    b.block_type = BlockType.TITLE
                    b.level = 1
                    title_block = b
                    break
            else:
                # No early heading — just demote the late title
                title_block.block_type = BlockType.BODY
                title_block = None

        if title_block and title_block.page_num > 0:
            # Title is on page 1+ — page 0 is likely an arXiv cover page.
            # Check both block text AND raw page text (sidebar may have been
            # filtered from blocks but still present in the PDF).
            has_arxiv = any("arxiv:" in b.text.lower() for b in all_blocks if b.page_num == 0)
            if not has_arxiv:
                # Check raw page text for arXiv stamp (may have been filtered
                # from blocks due to sidebar detection)
                page0_text = doc[0].get_text().lower() if len(doc) > 0 else ""
                has_arxiv = "arxiv:" in page0_text
            # Also detect cover pages where page 0 duplicates the title
            title_words = set(title_block.text.lower().split())
            has_dup_title = any(
                b.page_num == 0 and b.block_type == BlockType.HEADING
                and len(set(b.text.lower().split()) & title_words) >= len(title_words) * 0.7
                for b in all_blocks
            )
            if has_arxiv or has_dup_title:
                all_blocks = [b for b in all_blocks if b.page_num != 0]
        elif not title_block:
            # No TITLE block found — promote first HEADING on pages 0-1
            for b in all_blocks:
                if b.block_type == BlockType.HEADING and b.page_num <= 1:
                    b.block_type = BlockType.TITLE
                    b.level = 1
                    title_block = b
                    break

        # Demote all TITLE blocks after the real title to HEADING
        # (common in journal papers where title and section headings share the same font size)
        demoted_title_blocks = False
        if title_block:
            for b in all_blocks:
                if b is title_block:
                    continue
                if b.block_type == BlockType.TITLE:
                    text = b.text.strip()
                    # Bold text ending with comma is mid-sentence, not a heading
                    if text.endswith(",") or text.endswith(";"):
                        b.block_type = BlockType.BODY
                    else:
                        b.block_type = BlockType.HEADING
                        b.level = 2
                        demoted_title_blocks = True

        # If we demoted TITLE→HEADING (level 2), bump heading_sizes-level
        # blocks to SUBHEADING (level 3) to maintain hierarchy
        if demoted_title_blocks and hierarchy.heading_sizes:
            for b in all_blocks:
                if b is title_block:
                    continue
                if b.block_type == BlockType.HEADING and b.level == 2 and b.lines:
                    block_size = round(b.lines[0].dominant_size * 2) / 2
                    for hs in hierarchy.heading_sizes:
                        if abs(block_size - hs) < 0.5:
                            b.block_type = BlockType.SUBHEADING
                            b.level = 3
                            break

        # Heading hierarchy refinement: when both allcaps headings AND
        # non-allcaps bold-at-body-size headings exist at H2, demote the
        # non-allcaps ones to H3 (subsections). This fixes ICLR-format
        # papers like LoRA where "INTRODUCTION" and "Problem Statement"
        # both get level 2 because they share the same font size.
        if hierarchy.bold_heading_at_body_size:
            h2_blocks = [b for b in all_blocks
                         if b.block_type == BlockType.HEADING and b.level == 2]
            if h2_blocks:
                # Check original line text (before title-casing) for allcaps
                allcaps_h2 = [b for b in h2_blocks
                              if b.lines and _is_allcaps_heading(b.lines[0].text.strip())]
                noncaps_h2 = [b for b in h2_blocks if b not in allcaps_h2]
                # Only demote if we have both types (at least 2 allcaps, 1+ noncaps)
                if len(allcaps_h2) >= 2 and noncaps_h2:
                    for b in noncaps_h2:
                        b.block_type = BlockType.SUBHEADING
                        b.level = 3

        # Remove body blocks that appear before the title on the same page
        # (e.g., permission notices, conference headers)
        if title_block:
            all_blocks = [
                b for b in all_blocks
                if not (b.page_num == title_block.page_num
                        and b.bbox.y0 < title_block.bbox.y0
                        and b.block_type == BlockType.BODY)
            ]

        # Reclassify headings AND body blocks between title and Abstract as METADATA
        # (typically author names, affiliations, footnote text)
        if title_block:
            abstract_y = None
            for b in all_blocks:
                if (b.block_type == BlockType.HEADING
                        and b.page_num == title_block.page_num
                        and b.text.strip().lower() == "abstract"):
                    abstract_y = b.bbox.y0
                    break
            if abstract_y:
                for b in all_blocks:
                    if (b.page_num == title_block.page_num
                            and title_block.bbox.y1 < b.bbox.y0 < abstract_y
                            and b.block_type in (BlockType.HEADING, BlockType.SUBHEADING, BlockType.BODY)):
                        b.block_type = BlockType.METADATA

        # Detect bold-opening-paragraph as abstract (papers without "Abstract" heading)
        if title_block:
            has_abstract_heading = any(
                b.block_type == BlockType.HEADING
                and b.text.strip().lower() == "abstract"
                for b in all_blocks
            )
            if not has_abstract_heading:
                for b_idx, b in enumerate(all_blocks):
                    if b.block_type != BlockType.BODY:
                        continue
                    if b.page_num > title_block.page_num + 1:
                        break
                    if b.bbox.y0 < title_block.bbox.y1:
                        continue
                    if not b.lines:
                        break
                    # Find a contiguous run of bold lines (skip non-bold
                    # author/affiliation lines that may precede the abstract)
                    pre_lines: list[TextLine] = []
                    bold_lines: list[TextLine] = []
                    rest_lines: list[TextLine] = []
                    phase = "pre"  # pre → bold → rest
                    for line in b.lines:
                        text_spans = [s for s in line.spans if s.text.strip()]
                        if not text_spans:
                            continue
                        line_bold = all(s.is_bold for s in text_spans)
                        if phase == "pre":
                            if line_bold:
                                phase = "bold"
                                bold_lines.append(line)
                            else:
                                pre_lines.append(line)
                        elif phase == "bold":
                            if line_bold:
                                bold_lines.append(line)
                            else:
                                phase = "rest"
                                rest_lines.append(line)
                        else:
                            rest_lines.append(line)
                    # Need at least 2 bold lines totaling 50+ chars
                    bold_text = " ".join(l.text for l in bold_lines).strip()
                    if len(bold_lines) >= 2 and len(bold_text) > 50:
                        # Split: pre → METADATA, bold → ABSTRACT, rest → BODY
                        new_blocks: list[DocumentBlock] = []
                        if pre_lines:
                            pre_text = " ".join(l.text for l in pre_lines).strip()
                            new_blocks.append(DocumentBlock(
                                block_type=BlockType.METADATA,
                                text=pre_text,
                                bbox=merge_bboxes([l.bbox for l in pre_lines]),
                                page_num=b.page_num, lines=pre_lines,
                            ))
                        abs_block = DocumentBlock(
                            block_type=BlockType.ABSTRACT,
                            text=bold_text,
                            bbox=merge_bboxes([l.bbox for l in bold_lines]),
                            page_num=b.page_num, lines=bold_lines,
                        )
                        new_blocks.append(abs_block)
                        if rest_lines:
                            rest_text = " ".join(l.text for l in rest_lines).strip()
                            new_blocks.append(DocumentBlock(
                                block_type=BlockType.BODY,
                                text=rest_text,
                                bbox=merge_bboxes([l.bbox for l in rest_lines]),
                                page_num=b.page_num, lines=rest_lines,
                            ))
                        # Replace the original block with the split blocks
                        all_blocks[b_idx:b_idx + 1] = new_blocks
                        break

                    # Springer LNCS format: "Abstract. <text>" inline in a
                    # body block — bold "Abstract." keyword followed by text
                    for li, line in enumerate(b.lines):
                        lt = line.text.strip()
                        if _re.match(r'^Abstract[.\s]', lt, _re.IGNORECASE):
                            # Found inline abstract keyword
                            pre = b.lines[:li]
                            abs_start = b.lines[li:]
                            abs_text = " ".join(l.text for l in abs_start).strip()
                            # Strip "Abstract." or "Abstract " prefix
                            abs_text = _re.sub(r'^Abstract\.?\s*', '', abs_text, flags=_re.IGNORECASE)
                            if len(abs_text) > 50:
                                new_blocks = []
                                if pre:
                                    mt = " ".join(l.text for l in pre).strip()
                                    new_blocks.append(DocumentBlock(
                                        block_type=BlockType.METADATA, text=mt,
                                        bbox=merge_bboxes([l.bbox for l in pre]),
                                        page_num=b.page_num, lines=pre,
                                    ))
                                new_blocks.append(DocumentBlock(
                                    block_type=BlockType.ABSTRACT, text=abs_text,
                                    bbox=merge_bboxes([l.bbox for l in abs_start]),
                                    page_num=b.page_num, lines=abs_start,
                                ))
                                all_blocks[b_idx:b_idx + 1] = new_blocks
                                break
                    else:
                        # Only try ACM format if no inline abstract found
                        pass

                    if any(b.block_type == BlockType.ABSTRACT for b in all_blocks):
                        break

                    # ACM format: non-bold abstract between ALL-CAPS author
                    # lines and "CCS Concepts" / "Keywords" / "ACM Reference"
                    _ACM_BOUNDARIES = {"ccs concepts", "additional key words",
                                       "acm reference format", "categories and subject"}
                    meta_lines: list[TextLine] = []
                    abs_lines: list[TextLine] = []
                    post_lines: list[TextLine] = []
                    acm_phase = "meta"
                    for line in b.lines:
                        lt = line.text.strip()
                        if not lt:
                            continue
                        lt_low = lt.lower()
                        if acm_phase == "meta":
                            # Author lines: ALL CAPS names with affiliations
                            if _is_allcaps_name_line(lt) or _is_author_line(lt):
                                meta_lines.append(line)
                            elif any(lt_low.startswith(bnd) for bnd in _ACM_BOUNDARIES):
                                acm_phase = "post"
                                post_lines.append(line)
                            else:
                                acm_phase = "abs"
                                abs_lines.append(line)
                        elif acm_phase == "abs":
                            if any(lt_low.startswith(bnd) for bnd in _ACM_BOUNDARIES):
                                acm_phase = "post"
                                post_lines.append(line)
                            else:
                                abs_lines.append(line)
                        else:
                            post_lines.append(line)

                    abs_text = " ".join(l.text for l in abs_lines).strip()
                    if len(abs_lines) >= 3 and len(abs_text) > 100 and meta_lines:
                        new_blocks = []
                        if meta_lines:
                            mt = " ".join(l.text for l in meta_lines).strip()
                            new_blocks.append(DocumentBlock(
                                block_type=BlockType.METADATA, text=mt,
                                bbox=merge_bboxes([l.bbox for l in meta_lines]),
                                page_num=b.page_num, lines=meta_lines,
                            ))
                        new_blocks.append(DocumentBlock(
                            block_type=BlockType.ABSTRACT, text=abs_text,
                            bbox=merge_bboxes([l.bbox for l in abs_lines]),
                            page_num=b.page_num, lines=abs_lines,
                        ))
                        if post_lines:
                            pt = " ".join(l.text for l in post_lines).strip()
                            new_blocks.append(DocumentBlock(
                                block_type=BlockType.METADATA, text=pt,
                                bbox=merge_bboxes([l.bbox for l in post_lines]),
                                page_num=b.page_num, lines=post_lines,
                            ))
                        all_blocks[b_idx:b_idx + 1] = new_blocks

                    break  # only check the first BODY block

        # Merge paragraphs split at page boundaries
        all_blocks = _merge_cross_page_paragraphs(all_blocks)

        # Stage 4: Process references
        self._report_progress("references", page_count, page_count)
        all_refs: list[Reference] = []
        ref_section_idx = find_references_section(all_blocks)

        if ref_section_idx is not None and self.config.extract_references:
            # Collect blocks until the next heading (e.g. Appendix)
            _stop_types = {BlockType.HEADING, BlockType.TITLE, BlockType.SUBHEADING}
            ref_blocks = []
            for b in all_blocks[ref_section_idx + 1:]:
                if b.block_type in _stop_types:
                    break
                ref_blocks.append(b)
            ref_lines = []
            for b in ref_blocks:
                ref_lines.extend(b.lines)
            raw_entries = extract_reference_entries(ref_lines)
            all_refs = [parse_reference(e) for e in raw_entries]
            for b in ref_blocks:
                b.block_type = BlockType.REFERENCE

        # Associate table captions by number matching
        table_captions = [c for c in all_captions if c.metadata.get("type") in ("table", "tab")]
        _associate_table_captions(all_tables, table_captions)

        # Stage 5: Assemble Markdown
        self._report_progress("assembling", page_count, page_count)
        page_width = page_widths[0] if page_widths else 612
        markdown = self._assemble_markdown(
            all_blocks, all_tables, all_figures, all_refs,
            hierarchy, page_width, ref_section_idx, body_font,
        )
        # Final cleanup
        markdown = _dehyphenate(markdown)
        # Strip Unicode replacement characters and control characters
        markdown = markdown.replace('\ufffd', '')
        markdown = _re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', markdown)
        # Collapse 3+ consecutive blank lines to 2
        markdown = _re.sub(r'\n{4,}', '\n\n\n', markdown)
        # Remove trailing whitespace on lines
        markdown = '\n'.join(line.rstrip() for line in markdown.split('\n'))

        # Extract metadata
        title = ""
        abstract = ""
        for b in all_blocks:
            if b.block_type == BlockType.TITLE and not title:
                title = b.text
            if b.block_type == BlockType.ABSTRACT and not abstract:
                abstract = b.text

        authors = _parse_authors(all_blocks, title_block)

        # Clean author names from title if they were merged into the same block
        if authors and title:
            for author in authors:
                pos = title.find(author)
                if pos > 0:
                    # Truncate title before the first author name
                    title = title[:pos].rstrip()
                    # Also strip trailing footnote markers (∗†‡⋆) and whitespace
                    title = _re.sub(r'[\s∗†‡⋆,]+$', '', title)
                    # Fix the markdown heading to match
                    old_heading = f"# {title_block.text}"
                    if old_heading in markdown:
                        markdown = markdown.replace(old_heading, f"# {title}", 1)
                    break

        return ConversionResult(
            markdown=markdown,
            title=title,
            authors=authors,
            abstract=abstract,
            tables=all_tables,
            figures=all_figures,
            references=all_refs,
            page_count=len(page_widths),
        )

    def _convert_scanned_doc(self, doc: fitz.Document) -> ConversionResult:
        """Convert a scanned PDF using OCR fallback."""
        parts: list[str] = []
        warnings: list[str] = ["Scanned PDF detected; using OCR (quality may vary)"]

        for page_num, page in enumerate(doc):
            text = self._ocr_page(
                page,
                dpi=self.config.ocr_dpi,
                lang=self.config.ocr_lang,
            )
            if text.strip():
                parts.append(text.strip())
            else:
                if page_num == 0:
                    warnings.append("OCR produced no text; is Tesseract installed?")

        markdown = "\n\n".join(parts)
        markdown = _dehyphenate(markdown)

        # Try to extract title from first non-empty line
        title = ""
        for line in markdown.split("\n"):
            line = line.strip()
            if line and len(line) > 5:
                title = line
                break

        return ConversionResult(
            markdown=markdown,
            title=title,
            page_count=len(doc),
            warnings=warnings,
        )

    def _assemble_markdown(
        self,
        blocks: list[DocumentBlock],
        tables: list[Table],
        figures: list[Figure],
        refs: list[Reference],
        hierarchy: FontHierarchy,
        page_width: float,
        ref_section_idx: int | None,
        body_font: str = "",
    ) -> str:
        """Assemble all blocks into final Markdown."""
        parts: list[str] = []
        used_tables: set[int] = set()
        used_figures: set[int] = set()
        caption_blocks: set[int] = set()

        for i, b in enumerate(blocks):
            if b.block_type == BlockType.CAPTION:
                caption_blocks.add(i)

        for i, block in enumerate(blocks):
            if i in caption_blocks:
                continue

            if block.block_type == BlockType.TITLE:
                parts.append(f"# {block.text}\n")

            elif block.block_type == BlockType.HEADING:
                htxt = block.text.rstrip(".")
                parts.append(f"\n## {htxt}\n")
                if htxt.strip().lower() == "abstract":
                    for j in range(i + 1, min(i + 5, len(blocks))):
                        if blocks[j].block_type == BlockType.BODY:
                            blocks[j].block_type = BlockType.ABSTRACT
                            break

            elif block.block_type == BlockType.SUBHEADING:
                level = block.level
                prefix = "#" * min(level, 6)
                htxt = block.text.rstrip(".")
                parts.append(f"\n{prefix} {htxt}\n")

            elif block.block_type == BlockType.BODY:
                text = self._process_body_text(block, hierarchy, page_width, body_font)
                parts.append(f"\n{text}\n")

            elif block.block_type == BlockType.ABSTRACT:
                text = self._process_body_text(block, hierarchy, page_width, body_font)
                parts.append(f"\n> {text}\n")

            elif block.block_type == BlockType.LIST_ITEM:
                text = block.text.strip()
                if text and text[0] in "\u2022\u2023\u25e6\u2043\u2219\u25cf\u25cb\u2013\u2014":
                    text = "- " + text[1:].strip()
                elif not text.startswith("- "):
                    text = _re.sub(r"^(\d{1,3})[.)]\s", r"\1. ", text)
                parts.append(text)

            elif block.block_type in (BlockType.FOOTNOTE, BlockType.METADATA, BlockType.REFERENCE):
                continue

            # Insert tables near their position
            for t_idx, table in enumerate(tables):
                if t_idx in used_tables:
                    continue
                if table.page_num == block.page_num and table.bbox and block.bbox:
                    if table.bbox.y0 <= block.bbox.y1 + 50:
                        if table.caption:
                            parts.append(f"\n**{table.caption}**\n")
                        parts.append(f"\n{table_to_pipe(table)}\n")
                        used_tables.add(t_idx)

            # Insert figures near their position
            for f_idx, fig in enumerate(figures):
                if f_idx in used_figures:
                    continue
                if fig.page_num == block.page_num and fig.bbox and block.bbox:
                    if fig.bbox.y0 <= block.bbox.y1 + 50:
                        parts.append(f"\n{figure_to_markdown(fig)}\n")
                        used_figures.add(f_idx)

        # Remaining tables/figures
        for t_idx, table in enumerate(tables):
            if t_idx not in used_tables:
                if table.caption:
                    parts.append(f"\n**{table.caption}**\n")
                parts.append(f"\n{table_to_pipe(table)}\n")

        for f_idx, fig in enumerate(figures):
            if f_idx not in used_figures:
                parts.append(f"\n{figure_to_markdown(fig)}\n")

        # References (heading already in block loop)
        if refs:
            parts.append(f"\n{references_to_markdown(refs)}\n")

        return "\n".join(parts)

    def _process_body_text(
        self,
        block: DocumentBlock,
        hierarchy: FontHierarchy,
        page_width: float,
        body_font: str = "",
    ) -> str:
        """Process body text: inline math, bold, italic, dehyphenation."""
        if not block.lines:
            return block.text

        if self.config.extract_equations:
            parts = []
            for line in block.lines:
                if is_display_equation(line, page_width, hierarchy.body_size, body_font):
                    # Check complexity for image fallback
                    if self.config.math_fallback_to_image:
                        math_spans = [s for s in line.spans if is_math_span(s, body_font)]
                        score = _equation_complexity(math_spans, hierarchy.body_size)
                        if score >= 3:
                            y_pos = int(line.bbox.y0)
                            placeholder = f"![equation](equation_p{block.page_num}_{y_pos}.png)"
                            parts.append(f"\n{placeholder}\n")
                            continue
                    eq = extract_display_equation(line, hierarchy.body_size, body_font)
                    if eq:
                        parts.append(f"\n{eq}\n")
                        continue
                processed = extract_inline_math(line, hierarchy.body_size, body_font)
                parts.append(processed)

            # Merge consecutive $$...$$ blocks on adjacent lines into one
            parts = _merge_adjacent_display_math(parts)

            # Merge inline math continuation into preceding display equation
            # e.g. $$...softmax(QK^T / sqrt(dk)$$  $)V$ → merged display eq
            merged_parts: list[str] = []
            for p in parts:
                stripped = p.strip()
                if (merged_parts
                        and merged_parts[-1].strip().endswith('$$')
                        and stripped.startswith('$') and not stripped.startswith('$$')
                        and len(stripped) < 30):
                    inline_content = _re.match(r'^\$([^$]+)\$', stripped)
                    if inline_content:
                        prev = merged_parts[-1].rstrip()
                        if prev.endswith('$$'):
                            merged_parts[-1] = prev[:-2] + inline_content.group(1) + '$$'
                            continue
                merged_parts.append(p)
            parts = merged_parts

            text = " ".join(parts)
        else:
            text = " ".join(l.text for l in block.lines)

        # Dehyphenate
        text = _dehyphenate(text)

        # Ensure spaces around inline math: word$x$ → word $x$, $x$word → $x$ word
        text = _re.sub(r'(\w)\$(?!\$)', r'\1 $', text)
        text = _re.sub(r'(?<!\$)\$([^$]+)\$(\w)', r'$\1$ \2', text)
        # Trim whitespace inside inline $...$ delimiters: $ x $ → $x$
        def _trim_math(m: _re.Match) -> str:
            return f"${m.group(1).strip()}$"
        text = _re.sub(r'(?<!\$)\$([^$]+?)\$(?!\$)', _trim_math, text)
        # Remove empty inline math fragments (but not $$ display delimiters)
        text = _re.sub(r'(?<!\$)\$\s+\$(?!\$)', '', text)
        # Merge $x$ bareword → $x_{\text{bareword}}$ for common ML subscript patterns
        _SUB_WORDS = (
            r'model|ff|head|embed|vocab|key|query|value|pos|max|min|'
            r'out|enc|dec|src|tgt|mask|norm|attn|self|cross|drop|'
            r'steps|step|num|warmup|total|hidden|input|output|layer'
        )
        text = _re.sub(
            rf'\$([a-zA-Z])\$\s?({_SUB_WORDS})\b',
            r'$\1_{\\text{\2}}$',
            text,
        )
        # Fix R^{d}$ model → R^{d_{\text{model}}}$ (superscript + subscript text)
        text = _re.sub(
            rf'\^\{{([a-zA-Z])\}}\$\s?({_SUB_WORDS})\b',
            r'^{\1_{\\text{\2}}}$',
            text,
        )
        # Fix ...d}$ model → ...d_{\text{model}}}$ (letter at end of brace group)
        text = _re.sub(
            rf'([a-zA-Z])\}}\$\s?({_SUB_WORDS})\b',
            r'\1_{\\text{\2}}}$',
            text,
        )
        # Also fix d$model inside longer math: $...= d$model → $...= d_{\text{model}}$
        text = _re.sub(
            rf'([a-zA-Z])\$({_SUB_WORDS})\b',
            r'\1_{\\text{\2}}$',
            text,
        )

        # Apply bold/italic for non-math spans (only for substantial spans)
        for line in block.lines:
            for span in line.spans:
                if is_math_span(span, body_font):
                    continue
                raw = span.text
                stripped = raw.strip()
                if not stripped or len(stripped) < 2:
                    continue
                # Skip if it would nest markdown markers
                if stripped.startswith('*') or stripped.endswith('*'):
                    continue
                if span.is_bold and span.is_italic:
                    text = text.replace(raw, f"***{stripped}***", 1)
                elif span.is_bold:
                    text = text.replace(raw, f"**{stripped}**", 1)
                elif span.is_italic and len(stripped) > 2:
                    text = text.replace(raw, f"*{stripped}*", 1)

        # Clean up whitespace
        text = _re.sub(r"[ \t]+", " ", text).strip()
        return text


def _associate_table_captions(tables: list[Table], captions: list[DocumentBlock]) -> None:
    """Associate captions with tables, preferring number matching over proximity.

    Supports:
    - Sequential Arabic numbering (Table 1, Table 2, ...)
    - Roman numeral numbering (Table I, TABLE II, ...)
    - Compound numbering (Table 3.1, ...)
    - Proximity fallback on the same page when number matching fails
    """
    # Build caption number -> caption map
    num_to_cap: dict[int, DocumentBlock] = {}
    for cap in captions:
        num = cap.metadata.get("number")
        if num and num > 0:
            num_to_cap[num] = cap

    used_caps: set[int] = set()  # object ids of captions already assigned

    # First pass: match by sequential numbering (table 1 = first table, etc.)
    for idx, table in enumerate(tables):
        table_num = idx + 1
        if table_num in num_to_cap:
            cap = num_to_cap[table_num]
            table.caption = _dehyphenate(cap.text)
            used_caps.add(id(cap))

    # Second pass: tables still without a caption — try proximity on same page.
    # This handles Roman-numeral tables, reordered tables, and cases where the
    # sequential assumption broke down.
    for table in tables:
        if table.caption:
            continue
        if not table.bbox:
            continue
        best_cap = None
        best_dist = float("inf")
        for cap in captions:
            if id(cap) in used_caps:
                continue  # already assigned to another table
            if cap.page_num != table.page_num:
                continue
            dist = vertical_distance(table.bbox, cap.bbox)
            if dist < best_dist:
                best_dist = dist
                best_cap = cap
        if best_cap is not None and best_dist < 200:
            table.caption = _dehyphenate(best_cap.text)
            used_caps.add(id(best_cap))
