"""Integration tests against test PDFs."""
import pytest
from pathlib import Path

TEST_PDF = Path("/Volumes/EXTERNAL_USB/2603.04390v1.pdf")
TEST_PAPERS = Path("/Volumes/EXTERNAL_USB/test_papers")

PAPERS = {
    "attention": TEST_PAPERS / "attention.pdf",
    "resnet": TEST_PAPERS / "resnet.pdf",
    "bert": TEST_PAPERS / "bert.pdf",
    "gpt3": TEST_PAPERS / "gpt3.pdf",
    "ddpm": TEST_PAPERS / "ddpm.pdf",
    "vit": TEST_PAPERS / "vit.pdf",
    "llama": TEST_PAPERS / "llama.pdf",
    "ldm": TEST_PAPERS / "ldm.pdf",
    "kan_survey": TEST_PAPERS / "kan_survey.pdf",
    "mamba": TEST_PAPERS / "mamba.pdf",
    "batch_norm": TEST_PAPERS / "batch_norm.pdf",
    "dropout": TEST_PAPERS / "dropout.pdf",
    "yolo_ieee": TEST_PAPERS / "yolo_ieee.pdf",
    "webgis": TEST_PDF,
    "codex": TEST_PAPERS / "alphafold2.pdf",
    "sklearn": TEST_PAPERS / "jmlr_sklearn.pdf",
    "adam": TEST_PAPERS / "pmlr_adam.pdf",
    "unet": TEST_PAPERS / "elsevier_unet.pdf",
    "gan": TEST_PAPERS / "neurips_gan.pdf",
}

pytestmark = pytest.mark.skipif(
    not TEST_PDF.exists(),
    reason="Test PDFs not available",
)


def _convert(name):
    from paper2md import convert
    return convert(PAPERS[name])


# === WebGIS paper (original) ===

def test_webgis_title():
    r = _convert("webgis")
    assert r.markdown.strip().startswith("# ")
    assert "Dual-Helix" in r.title


def test_webgis_sections():
    r = _convert("webgis")
    h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
    assert "introduction" in h2s
    assert "background" in h2s or "related work" in h2s


def test_webgis_tables():
    r = _convert("webgis")
    assert len(r.tables) >= 4
    assert "| " in r.markdown


def test_webgis_figures():
    r = _convert("webgis")
    assert len(r.figures) >= 5


def test_webgis_references():
    r = _convert("webgis")
    assert len(r.references) >= 30
    with_year = [ref for ref in r.references if ref.year]
    assert len(with_year) >= 20


def test_webgis_pages():
    r = _convert("webgis")
    assert r.page_count == 56


def test_webgis_no_page_numbers():
    r = _convert("webgis")
    standalone = [l.strip() for l in r.markdown.split("\n") if l.strip().isdigit() and len(l.strip()) <= 2]
    assert len(standalone) < 10


# === Author extraction (cross-paper) ===

def test_authors_attention():
    r = _convert("attention")
    assert len(r.authors) >= 2, f"Expected >=2 authors, got {r.authors}"
    assert any("Vaswani" in a for a in r.authors)


def test_authors_llama():
    r = _convert("llama")
    assert len(r.authors) >= 5, f"Expected >=5 authors, got {r.authors}"
    assert any("Touvron" in a for a in r.authors)


def test_authors_ddpm():
    r = _convert("ddpm")
    assert len(r.authors) >= 2, f"Expected >=2 authors, got {r.authors}"


def test_authors_resnet():
    r = _convert("resnet")
    assert len(r.authors) >= 3, f"Expected >=3 authors, got {r.authors}"
    assert any("He" in a for a in r.authors)


def test_authors_bert():
    r = _convert("bert")
    assert len(r.authors) >= 3, f"Expected >=3 authors, got {r.authors}"
    assert any("Devlin" in a for a in r.authors)


def test_authors_gpt3():
    r = _convert("gpt3")
    assert len(r.authors) >= 10, f"Expected >=10 authors, got {r.authors}"
    assert any("Brown" in a for a in r.authors)


def test_authors_vit():
    r = _convert("vit")
    assert len(r.authors) >= 5, f"Expected >=5 authors, got {r.authors}"
    assert any("Dosovitskiy" in a for a in r.authors)


def test_authors_ldm():
    r = _convert("ldm")
    assert len(r.authors) >= 3, f"Expected >=3 authors, got {r.authors}"
    assert any("Rombach" in a for a in r.authors)


def test_authors_kan_survey():
    """ACM format with all-caps author names."""
    r = _convert("kan_survey")
    assert len(r.authors) >= 4, f"Expected >=4 authors, got {r.authors}"
    assert any("Somvanshi" in a for a in r.authors)
    # Names should be title-cased, not ALL CAPS
    for name in r.authors:
        assert not name.isupper(), f"Author name should not be all caps: {name}"


def test_authors_webgis():
    r = _convert("webgis")
    assert len(r.authors) >= 2, f"Expected >=2 authors, got {r.authors}"


# === Vector figure detection ===

def test_vector_figures_resnet():
    """ResNet has vector charts/diagrams that should be detected."""
    r = _convert("resnet")
    vec = [f for f in r.figures if f.is_vector]
    assert len(vec) >= 1, "Expected vector figures in ResNet"


def test_vector_figures_rendered():
    """Vector figures should have PNG image data rendered."""
    r = _convert("resnet")
    vec = [f for f in r.figures if f.is_vector]
    if vec:
        # At least one should have image_data
        with_data = [f for f in vec if f.image_data]
        assert len(with_data) >= 1, "Vector figures should have rendered image_data"
        # Check it's valid PNG (starts with PNG signature)
        assert with_data[0].image_data[:4] == b'\x89PNG'


def test_vector_figures_no_table_rules():
    """Table rules should not be detected as vector figures."""
    r = _convert("attention")
    # Tables should still be detected normally
    assert len(r.tables) >= 3


# === OCR / scanned PDF detection ===

def test_normal_pdfs_not_detected_as_scanned():
    """Normal text PDFs should NOT trigger the OCR fallback."""
    from paper2md.converters.pdf import PdfConverter
    import fitz
    doc = fitz.open(str(PAPERS["attention"]))
    assert not PdfConverter._is_scanned_pdf(doc)
    doc.close()


# === Attention Is All You Need ===

@pytest.mark.skipif(not (TEST_PAPERS / "attention.pdf").exists(), reason="Paper not downloaded")
class TestAttention:
    def test_title(self):
        r = _convert("attention")
        assert "Attention Is All You Need" in r.title

    def test_starts_with_h1(self):
        r = _convert("attention")
        assert r.markdown.strip().startswith("# ")

    def test_tables(self):
        r = _convert("attention")
        assert len(r.tables) >= 3

    def test_references(self):
        r = _convert("attention")
        assert len(r.references) >= 20

    def test_dehyphenation(self):
        r = _convert("attention")
        # Should not have broken hyphens like "transduc- tion"
        import re
        broken = re.findall(r"[a-z]- [a-z]", r.markdown)
        assert len(broken) < 5, f"Too many broken hyphens: {broken[:5]}"


# === Deep Residual Learning (two-column) ===

@pytest.mark.skipif(not (TEST_PAPERS / "resnet.pdf").exists(), reason="Paper not downloaded")
class TestResNet:
    def test_title(self):
        r = _convert("resnet")
        assert "Deep Residual Learning" in r.title

    def test_starts_with_h1(self):
        r = _convert("resnet")
        assert r.markdown.strip().startswith("# ")

    def test_tables(self):
        r = _convert("resnet")
        assert len(r.tables) >= 5

    def test_references(self):
        r = _convert("resnet")
        assert len(r.references) >= 20


# === BERT (two-column, bold-at-body-size headings) ===

@pytest.mark.skipif(not (TEST_PAPERS / "bert.pdf").exists(), reason="Paper not downloaded")
class TestBERT:
    def test_title(self):
        r = _convert("bert")
        assert "BERT" in r.title

    def test_starts_with_h1(self):
        r = _convert("bert")
        assert r.markdown.strip().startswith("# ")

    def test_references_found(self):
        r = _convert("bert")
        assert len(r.references) >= 10


# === GPT-3 (alphanumeric citation keys) ===

@pytest.mark.skipif(not (TEST_PAPERS / "gpt3.pdf").exists(), reason="Paper not downloaded")
class TestGPT3:
    def test_title(self):
        r = _convert("gpt3")
        assert "Few-Shot Learners" in r.title

    def test_starts_with_h1(self):
        r = _convert("gpt3")
        assert r.markdown.strip().startswith("# ")

    def test_references(self):
        r = _convert("gpt3")
        assert len(r.references) >= 50

    def test_tables(self):
        r = _convert("gpt3")
        assert len(r.tables) >= 10


# === DDPM (two-column, math-heavy) ===

@pytest.mark.skipif(not (TEST_PAPERS / "ddpm.pdf").exists(), reason="Paper not downloaded")
class TestDDPM:
    def test_title(self):
        r = _convert("ddpm")
        assert "Denoising Diffusion" in r.title

    def test_starts_with_h1(self):
        r = _convert("ddpm")
        assert r.markdown.strip().startswith("# ")

    def test_has_math(self):
        r = _convert("ddpm")
        assert "$" in r.markdown

    def test_display_equations(self):
        """DDPM should have display equations ($$...$$)."""
        r = _convert("ddpm")
        assert "$$" in r.markdown

    def test_inline_math_unaffected(self):
        """Inline math should still work with complexity improvements."""
        r = _convert("attention")
        # Attention paper has some math content
        # Just verify no crash and reasonable output
        assert r.markdown
        assert len(r.markdown) > 1000

    def test_references(self):
        r = _convert("ddpm")
        assert len(r.references) >= 30


# === ViT (ICLR format, all-caps headings, drop-cap title) ===

@pytest.mark.skipif(not (TEST_PAPERS / "vit.pdf").exists(), reason="Paper not downloaded")
class TestViT:
    def test_title(self):
        r = _convert("vit")
        assert "16X16 Words" in r.title or "16x16 Words" in r.title
        assert "Image Recognition" in r.title

    def test_starts_with_h1(self):
        r = _convert("vit")
        assert r.markdown.strip().startswith("# ")

    def test_sections(self):
        r = _convert("vit")
        h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
        assert "introduction" in h2s
        assert "related work" in h2s

    def test_references(self):
        r = _convert("vit")
        assert len(r.references) >= 40

    def test_tables(self):
        r = _convert("vit")
        assert len(r.tables) >= 5

    def test_dehyphenation(self):
        r = _convert("vit")
        import re
        broken = re.findall(r"[a-z]- [a-z]", r.markdown)
        assert len(broken) < 5


# === LLaMA (Meta AI format, two-column) ===

@pytest.mark.skipif(not (TEST_PAPERS / "llama.pdf").exists(), reason="Paper not downloaded")
class TestLLaMA:
    def test_title(self):
        r = _convert("llama")
        assert "LLaMA" in r.title

    def test_starts_with_h1(self):
        r = _convert("llama")
        assert r.markdown.strip().startswith("# ")

    def test_sections(self):
        r = _convert("llama")
        h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
        assert "introduction" in h2s
        assert "approach" in h2s

    def test_references(self):
        r = _convert("llama")
        assert len(r.references) >= 50

    def test_tables(self):
        r = _convert("llama")
        assert len(r.tables) >= 10

    def test_year_extraction(self):
        r = _convert("llama")
        with_year = [ref for ref in r.references if ref.year]
        assert len(with_year) == len(r.references)


# === LDM / Stable Diffusion (CVPR format, many sample images) ===

@pytest.mark.skipif(not (TEST_PAPERS / "ldm.pdf").exists(), reason="Paper not downloaded")
class TestLDM:
    def test_title(self):
        r = _convert("ldm")
        assert "Latent Diffusion" in r.title

    def test_starts_with_h1(self):
        r = _convert("ldm")
        assert r.markdown.strip().startswith("# ")

    def test_sections(self):
        r = _convert("ldm")
        h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
        assert "introduction" in h2s
        assert "related work" in h2s

    def test_references(self):
        r = _convert("ldm")
        assert len(r.references) >= 80

    def test_tables(self):
        r = _convert("ldm")
        assert len(r.tables) >= 10

    def test_figures_clustered(self):
        """Figures should be spatially clustered, not hundreds of grid tiles."""
        r = _convert("ldm")
        assert len(r.figures) < 100  # was 338 before clustering

    def test_year_extraction(self):
        r = _convert("ldm")
        with_year = [ref for ref in r.references if ref.year]
        assert len(with_year) == len(r.references)


# === KAN Survey (ACM format, all-caps authors) ===

@pytest.mark.skipif(not (TEST_PAPERS / "kan_survey.pdf").exists(), reason="Paper not downloaded")
class TestKANSurvey:
    def test_title(self):
        r = _convert("kan_survey")
        assert "Kolmogorov-Arnold Network" in r.title

    def test_starts_with_h1(self):
        r = _convert("kan_survey")
        assert r.markdown.strip().startswith("# ")

    def test_authors(self):
        """ACM format with all-caps author names should be detected."""
        r = _convert("kan_survey")
        assert len(r.authors) >= 4, f"Expected >=4 authors, got {r.authors}"
        # Names should be title-cased, not ALL CAPS
        for name in r.authors:
            assert not name.isupper(), f"Author name should not be all caps: {name}"
        assert any("Somvanshi" in a for a in r.authors)

    def test_sections(self):
        r = _convert("kan_survey")
        h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
        assert "introduction" in h2s

    def test_references(self):
        r = _convert("kan_survey")
        assert len(r.references) >= 50

    def test_tables(self):
        r = _convert("kan_survey")
        assert len(r.tables) >= 5


@pytest.mark.skipif(not (TEST_PAPERS / "mamba.pdf").exists(), reason="Paper not downloaded")
class TestMamba:
    def test_title(self):
        r = _convert("mamba")
        assert "Mamba" in r.title

    def test_abstract(self):
        """Mamba's 'Abstract' heading is at 9pt bold (smaller than 10pt body).
        It should still be detected as a heading."""
        r = _convert("mamba")
        assert r.abstract, "Mamba paper should have an abstract"
        assert "foundation models" in r.abstract.lower()

    def test_authors(self):
        r = _convert("mamba")
        assert len(r.authors) >= 2, f"Expected >=2 authors, got {r.authors}"

    def test_sections(self):
        r = _convert("mamba")
        md = r.markdown
        assert "## Abstract" in md or "## abstract" in md.lower()
        assert "Introduction" in md


# === Content quality assertions ===


class TestAbstractContent:
    """Verify abstract text contains expected domain-specific terms."""

    def test_attention_abstract_content(self):
        r = _convert("attention")
        assert r.abstract, "Attention paper should have an abstract"
        abstract_lower = r.abstract.lower()
        assert "sequence" in abstract_lower or "transduction" in abstract_lower, (
            f"Attention abstract should mention 'sequence' or 'transduction': {r.abstract[:200]}"
        )
        assert "attention" in abstract_lower, (
            f"Attention abstract should mention 'attention': {r.abstract[:200]}"
        )

    def test_ddpm_abstract_content(self):
        r = _convert("ddpm")
        assert r.abstract, "DDPM paper should have an abstract"
        abstract_lower = r.abstract.lower()
        assert "diffusion" in abstract_lower, (
            f"DDPM abstract should mention 'diffusion': {r.abstract[:200]}"
        )

    def test_bert_abstract_content(self):
        r = _convert("bert")
        assert r.abstract, "BERT paper should have an abstract"
        abstract_lower = r.abstract.lower()
        assert "language" in abstract_lower, (
            f"BERT abstract should mention 'language': {r.abstract[:200]}"
        )
        assert "bidirectional" in abstract_lower or "pre-training" in abstract_lower, (
            f"BERT abstract should mention 'bidirectional' or 'pre-training': {r.abstract[:200]}"
        )

    def test_mamba_abstract_content(self):
        """Mamba's abstract heading is smaller than body — still detects abstract."""
        r = _convert("mamba")
        assert r.abstract, "Mamba paper should have an abstract"
        assert "foundation models" in r.abstract.lower()

    def test_kan_survey_abstract_content(self):
        """ACM format: abstract detected without heading, between authors and CCS."""
        r = _convert("kan_survey")
        assert r.abstract, "KAN Survey paper should have an abstract"
        assert "kolmogorov" in r.abstract.lower()
        assert len(r.abstract) > 100

    def test_resnet_abstract_content(self):
        r = _convert("resnet")
        assert r.abstract, "ResNet paper should have an abstract"
        abstract_lower = r.abstract.lower()
        assert "residual" in abstract_lower, (
            f"ResNet abstract should mention 'residual': {r.abstract[:200]}"
        )


class TestTableContent:
    """Verify actual table cell values, not just counts."""

    def test_attention_table2_bleu_scores(self):
        """Table 2 in Attention paper has BLEU scores for translation models."""
        r = _convert("attention")
        # Table at index 1 is paper's "Table 2" with BLEU scores
        t = r.tables[1]
        assert "BLEU" in t.caption or "Table 2" in t.caption, (
            f"Expected Table 2 (BLEU), got caption: {t.caption[:100]}"
        )
        # Flatten all cell text
        all_text = " ".join(cell for row in t.cells for cell in row)
        # Should contain the Transformer (big) BLEU score 28.4
        assert "28.4" in all_text, (
            f"Table 2 should contain BLEU score 28.4: {all_text[:300]}"
        )
        # Should contain model names
        assert "Transformer" in all_text, (
            f"Table 2 should mention 'Transformer': {all_text[:300]}"
        )

    def test_attention_table3_variations(self):
        """Table 3 in Attention paper shows Transformer architecture variations."""
        r = _convert("attention")
        # Table at index 2 is paper's "Table 3"
        t = r.tables[2]
        assert "Table 3" in t.caption or "Variations" in t.caption, (
            f"Expected Table 3, got caption: {t.caption[:100]}"
        )
        # Header row should mention PPL or BLEU
        header_text = " ".join(t.cells[0])
        assert "PPL" in header_text or "BLEU" in header_text, (
            f"Table 3 header should contain 'PPL' or 'BLEU': {header_text}"
        )
        # Base row should have d_model=512 (may be in row 1, 2, or 3
        # depending on whether sub-headers are split as separate rows)
        found_512 = any(
            "512" in " ".join(t.cells[i])
            for i in range(1, min(4, len(t.cells)))
        )
        assert found_512, (
            f"Table 3 base rows should contain '512': "
            + str([t.cells[i] for i in range(1, min(4, len(t.cells)))])
        )

    def test_table_cells_not_empty(self):
        """Tables should have actual content, not all empty cells."""
        r = _convert("attention")
        for i, t in enumerate(r.tables):
            all_text = "".join(cell for row in t.cells for cell in row).strip()
            assert len(all_text) > 10, (
                f"Table {i} has mostly empty cells: {all_text[:100]}"
            )

    def test_resnet_table_has_error_rates(self):
        """ResNet Table 2 should contain error rate percentages."""
        r = _convert("resnet")
        # Table at index 1 is paper's "Table 2" with top-1/top-5 error
        t = r.tables[1]
        all_text = " ".join(cell for row in t.cells for cell in row)
        assert "err" in all_text.lower() or "error" in t.caption.lower(), (
            f"ResNet table should mention error rates: {t.caption[:100]}"
        )
        # Should contain numeric values (error percentages)
        import re
        numbers = re.findall(r"\d+\.\d+", all_text)
        assert len(numbers) >= 3, (
            f"ResNet table should have numeric error rates, found {len(numbers)}: {all_text[:200]}"
        )


class TestReferenceContent:
    """Verify actual reference content, not just counts."""

    def test_attention_has_bahdanau(self):
        """Attention paper should reference Bahdanau (attention pioneer)."""
        r = _convert("attention")
        bahdanau_refs = [
            ref for ref in r.references
            if "Bahdanau" in ref.raw or "Bahdanau" in ref.authors
        ]
        assert len(bahdanau_refs) >= 1, (
            "Attention paper should reference Bahdanau"
        )

    def test_attention_refs_have_authors_and_years(self):
        """References should have actual author text and year, not empty fields."""
        r = _convert("attention")
        refs_with_authors = [ref for ref in r.references if ref.authors.strip()]
        assert len(refs_with_authors) >= len(r.references) * 0.8, (
            f"At least 80% of refs should have authors, got "
            f"{len(refs_with_authors)}/{len(r.references)}"
        )
        refs_with_year = [ref for ref in r.references if ref.year.strip()]
        assert len(refs_with_year) >= len(r.references) * 0.8, (
            f"At least 80% of refs should have years, got "
            f"{len(refs_with_year)}/{len(r.references)}"
        )

    def test_bert_refs_have_real_content(self):
        """BERT references should contain recognizable author names."""
        r = _convert("bert")
        # First ref should be a real author
        assert r.references[0].authors.strip(), (
            f"First BERT reference should have authors: {r.references[0].raw[:100]}"
        )
        assert r.references[0].year.strip(), (
            f"First BERT reference should have a year: {r.references[0].raw[:100]}"
        )
        # Year should be a plausible 4-digit year
        year_str = r.references[0].year.strip()
        assert year_str.isdigit() and 1990 <= int(year_str) <= 2025, (
            f"Year should be a plausible year, got: {year_str}"
        )

    def test_attention_ref_raw_not_garbled(self):
        """Reference raw text should be readable, not garbled."""
        r = _convert("attention")
        for ref in r.references[:10]:
            # Raw text should have at least 30 chars of readable content
            assert len(ref.raw) >= 30, (
                f"Reference raw too short: {ref.raw}"
            )
            # Should not be all special characters
            alpha_count = sum(1 for c in ref.raw if c.isalpha())
            assert alpha_count >= 10, (
                f"Reference raw should have readable text: {ref.raw[:100]}"
            )


class TestMathContentQuality:
    """Verify math extraction produces actual LaTeX, not garbled text."""

    def test_ddpm_has_latex_commands(self):
        """DDPM markdown should contain recognizable LaTeX commands."""
        r = _convert("ddpm")
        md = r.markdown
        # Should contain LaTeX backslash commands (alpha, beta, theta are core to DDPM)
        latex_commands_found = []
        for cmd in ["\\alpha", "\\beta", "\\theta", "\\sigma", "\\sqrt"]:
            if cmd in md:
                latex_commands_found.append(cmd)
        assert len(latex_commands_found) >= 3, (
            f"DDPM should have >= 3 LaTeX commands, found: {latex_commands_found}"
        )

    def test_ddpm_has_subscripts(self):
        """DDPM math should have subscript notation like x_{t}."""
        r = _convert("ddpm")
        assert "_{" in r.markdown, (
            "DDPM math should contain subscript notation (_{...})"
        )

    def test_ddpm_no_excessive_garbled_math(self):
        """Should not have excessive single-character math fragments."""
        r = _convert("ddpm")
        import re
        # Count occurrences of $X$ where X is a single non-alphanumeric char
        # These indicate garbled math extraction
        garbled = re.findall(r"\$[^$]\$", r.markdown)
        # Filter to truly garbled ones (non-alphanumeric single chars)
        truly_garbled = [g for g in garbled if not g[1].isalnum()]
        # Current baseline has ~82 garbled fragments (integral signs,
        # replacement chars from PDF math extraction). Use 120 as regression
        # ceiling -- any increase suggests a new extraction bug.
        assert len(truly_garbled) < 120, (
            f"Too many garbled single-char math fragments ({len(truly_garbled)}): "
            f"{truly_garbled[:10]}"
        )

    def test_ddpm_display_math_has_content(self):
        """Display math blocks ($$...$$) should contain actual expressions."""
        r = _convert("ddpm")
        import re
        display_blocks = re.findall(r"\$\$(.+?)\$\$", r.markdown, re.DOTALL)
        # DDPM should have at least one display equation
        # (The count from actual conversion was 1, but inline math may use $...$)
        assert "$" in r.markdown, "DDPM should contain math notation"
        # Check that math content exists with LaTeX-like content
        math_with_backslash = [
            b for b in display_blocks if "\\" in b and len(b.strip()) > 3
        ]
        # The inline math is more prevalent in DDPM; just verify math isn't empty
        all_math = re.findall(r"\$([^$]+)\$", r.markdown)
        substantive = [m for m in all_math if len(m.strip()) > 3]
        assert len(substantive) >= 20, (
            f"DDPM should have >= 20 substantive math expressions, got {len(substantive)}"
        )

    def test_attention_dmodel_rendering(self):
        """d_model should render as $d_{\\text{model}}$ not $d$ model."""
        r = _convert("attention")
        md = r.markdown
        assert "d_{\\text{model}}" in md, (
            "Attention paper should have d_{\\text{model}} subscript notation"
        )
        # Should NOT have bare "$d$ model" pattern
        import re
        bare_d_model = re.findall(r'\$d\$\s?model', md)
        assert len(bare_d_model) == 0, (
            f"Should not have bare '$d$ model' pattern: found {len(bare_d_model)}"
        )

    def test_attention_equation_complete(self):
        """The Attention equation should be a single complete display equation."""
        r = _convert("attention")
        md = r.markdown
        import re
        # Find display equations containing "Attention" and "softmax"
        display_eqs = re.findall(r'\$\$(.+?)\$\$', md)
        attn_eqs = [eq for eq in display_eqs if 'Attention' in eq and 'softmax' in eq]
        assert len(attn_eqs) >= 1, (
            "Should have at least one complete Attention(Q,K,V)=softmax(...) display equation"
        )
        # The equation should contain both softmax and V (complete, not truncated)
        eq = attn_eqs[0]
        assert 'V' in eq, f"Attention equation should contain V: {eq}"


class TestSectionOrdering:
    """Verify that major sections appear in logical order."""

    def test_attention_section_order(self):
        """Introduction should appear before Background and Model Architecture."""
        r = _convert("attention")
        lines = r.markdown.split("\n")
        heading_order = []
        for i, line in enumerate(lines):
            if line.startswith("## "):
                heading_order.append((i, line))

        # Extract positions of key sections
        positions = {}
        for pos, heading in heading_order:
            lower = heading.lower()
            if "introduction" in lower:
                positions["introduction"] = pos
            elif "background" in lower:
                positions["background"] = pos
            elif "model architecture" in lower:
                positions["model_architecture"] = pos
            elif "result" in lower:
                positions["results"] = pos
            elif "conclusion" in lower:
                positions["conclusion"] = pos

        assert "introduction" in positions, "Should have Introduction section"
        if "background" in positions:
            assert positions["introduction"] < positions["background"], (
                "Introduction should come before Background"
            )
        if "model_architecture" in positions:
            assert positions["introduction"] < positions["model_architecture"], (
                "Introduction should come before Model Architecture"
            )
        if "results" in positions:
            assert positions["introduction"] < positions["results"], (
                "Introduction should come before Results"
            )
        if "conclusion" in positions:
            assert positions["results"] < positions["conclusion"], (
                "Results should come before Conclusion"
            )

    def test_bert_section_order(self):
        """BERT: Introduction -> Related Work -> BERT -> Experiments -> Conclusion."""
        r = _convert("bert")
        lines = r.markdown.split("\n")
        heading_positions = []
        for i, line in enumerate(lines):
            if line.startswith("## "):
                heading_positions.append((i, line.lower()))

        # Use first occurrence only (appendix re-uses terms like "experimental")
        positions = {}
        for pos, heading in heading_positions:
            if "introduction" in heading and "introduction" not in positions:
                positions["introduction"] = pos
            elif "related work" in heading and "related_work" not in positions:
                positions["related_work"] = pos
            elif "experiment" in heading and "experiments" not in positions:
                positions["experiments"] = pos
            elif "conclusion" in heading and "conclusion" not in positions:
                positions["conclusion"] = pos

        assert "introduction" in positions, "BERT should have Introduction"
        if "related_work" in positions:
            assert positions["introduction"] < positions["related_work"], (
                "Introduction should come before Related Work"
            )
        if "experiments" in positions:
            assert positions["introduction"] < positions["experiments"], (
                "Introduction should come before Experiments"
            )
        if "conclusion" in positions:
            assert positions["experiments"] < positions["conclusion"], (
                "Experiments should come before Conclusion"
            )

    def test_ddpm_section_order(self):
        """DDPM: Introduction -> Background -> Experiments -> Conclusion."""
        r = _convert("ddpm")
        lines = r.markdown.split("\n")
        heading_positions = []
        for i, line in enumerate(lines):
            if line.startswith("## "):
                heading_positions.append((i, line.lower()))

        # Use first occurrence only (appendix re-uses terms like "experimental")
        positions = {}
        for pos, heading in heading_positions:
            if "introduction" in heading and "introduction" not in positions:
                positions["introduction"] = pos
            elif "background" in heading and "background" not in positions:
                positions["background"] = pos
            elif "experiment" in heading and "experiments" not in positions:
                positions["experiments"] = pos
            elif "conclusion" in heading and "conclusion" not in positions:
                positions["conclusion"] = pos

        assert "introduction" in positions, "DDPM should have Introduction"
        if "background" in positions:
            assert positions["introduction"] < positions["background"], (
                "Introduction should come before Background"
            )
        if "experiments" in positions:
            assert positions["introduction"] < positions["experiments"], (
                "Introduction should come before Experiments"
            )
        if "conclusion" in positions:
            assert positions["experiments"] < positions["conclusion"], (
                "Experiments should come before Conclusion"
            )


# === Diverse paper format tests ===

@pytest.mark.skipif(not (TEST_PAPERS / "batch_norm.pdf").exists(), reason="Paper not downloaded")
class TestBatchNorm:
    def test_title(self):
        r = _convert("batch_norm")
        assert "Batch Normalization" in r.title

    def test_abstract(self):
        r = _convert("batch_norm")
        assert r.abstract and len(r.abstract) > 50
        assert "covariate" in r.abstract.lower()

    def test_authors(self):
        r = _convert("batch_norm")
        assert len(r.authors) >= 2

    def test_sections(self):
        r = _convert("batch_norm")
        h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
        assert "introduction" in h2s
        assert "experiment" in h2s

    def test_section_3_heading_merge(self):
        """Section 3 heading spans 2 lines: 'Normalization via Mini-Batch Statistics'."""
        r = _convert("batch_norm")
        assert "Mini-Batch Statistics" in r.markdown

    def test_subheading_dehyphenation(self):
        """Subsection heading should be dehyphenated: 'Batch-Normalized Networks'."""
        r = _convert("batch_norm")
        assert "Batch-Normalized Networks" in r.markdown

    def test_references(self):
        r = _convert("batch_norm")
        assert len(r.references) >= 30

    def test_math_superscript_parens(self):
        """Parenthesized superscripts like x^{(k)} should render correctly."""
        r = _convert("batch_norm")
        import re
        # Should have ^{(k)} not ^{(}_{k)}
        bad_pattern = re.findall(r'\^\{\(\}_\{', r.markdown)
        assert len(bad_pattern) == 0, f"Found broken ^{{(}}_(k) patterns: {len(bad_pattern)}"
        good_pattern = re.findall(r'\^\{\([a-zA-Z0-9]+\)\}', r.markdown)
        assert len(good_pattern) >= 5, f"Should have ^{{(k)}} patterns, found {len(good_pattern)}"

    def test_no_garbled_chars(self):
        r = _convert("batch_norm")
        garbled = sum(1 for ch in r.markdown if ord(ch) < 0x20 and ch not in '\n\t\r')
        assert garbled == 0


@pytest.mark.skipif(not (TEST_PAPERS / "dropout.pdf").exists(), reason="Paper not downloaded")
class TestDropout:
    def test_title(self):
        r = _convert("dropout")
        assert "neural networks" in r.title.lower()

    def test_abstract(self):
        r = _convert("dropout")
        assert r.abstract and len(r.abstract) > 50

    def test_authors(self):
        r = _convert("dropout")
        assert len(r.authors) >= 4
        assert any("Hinton" in a for a in r.authors)

    def test_sections(self):
        r = _convert("dropout")
        h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
        assert "references" in h2s

    def test_references(self):
        r = _convert("dropout")
        assert len(r.references) >= 15

    def test_no_garbled_chars(self):
        r = _convert("dropout")
        garbled = sum(1 for ch in r.markdown if ord(ch) < 0x20 and ch not in '\n\t\r')
        assert garbled == 0


@pytest.mark.skipif(not (TEST_PAPERS / "yolo_ieee.pdf").exists(), reason="Paper not downloaded")
class TestYoloIEEE:
    def test_title(self):
        r = _convert("yolo_ieee")
        assert "You Only Look Once" in r.title

    def test_abstract(self):
        r = _convert("yolo_ieee")
        assert r.abstract and len(r.abstract) > 50
        assert "detection" in r.abstract.lower()

    def test_authors(self):
        r = _convert("yolo_ieee")
        assert len(r.authors) >= 3

    def test_sections(self):
        r = _convert("yolo_ieee")
        h2s = " ".join(l for l in r.markdown.split("\n") if l.startswith("## ")).lower()
        assert "introduction" in h2s
        assert "experiment" in h2s
        assert "conclusion" in h2s

    def test_no_spurious_headings(self):
        """Model names in tables should not become headings."""
        r = _convert("yolo_ieee")
        headings = [l for l in r.markdown.split("\n") if l.startswith("## ")]
        for h in headings:
            assert "Vgg" not in h, f"Table content should not be heading: {h}"
            assert "Fast R-CNN YOLO" not in h or "Combining" in h, (
                f"Table content should not be heading: {h}"
            )

    def test_references(self):
        r = _convert("yolo_ieee")
        assert len(r.references) >= 30

    def test_no_garbled_chars(self):
        r = _convert("yolo_ieee")
        garbled = sum(1 for ch in r.markdown if ord(ch) < 0x20 and ch not in '\n\t\r')
        assert garbled == 0


# === Diverse format edge cases ===

class TestEdgeCases:
    """Edge case tests for diverse paper formats."""

    def test_allcaps_heading_cjk_no_false_positive(self):
        """CJK text should not trigger all-caps heading detection."""
        from paper2md.extractors.layout import _is_allcaps_heading
        assert not _is_allcaps_heading("深度学习在自然语言处理中的应用")
        assert not _is_allcaps_heading("注意力机制 Attention Mechanism")

    def test_allcaps_heading_english_works(self):
        """Standard all-caps headings should still be detected."""
        from paper2md.extractors.layout import _is_allcaps_heading
        assert _is_allcaps_heading("INTRODUCTION")
        assert _is_allcaps_heading("1 INTRODUCTION")
        assert _is_allcaps_heading("RELATED WORK")

    def test_equation_complexity_scoring(self):
        """Complexity scorer should work on basic inputs."""
        from paper2md.extractors.equations import _equation_complexity
        from paper2md.types import Span, BBox
        # Empty spans = 0 complexity
        assert _equation_complexity([], 10.0) == 0
        # Single span = low complexity
        span = Span("x", "CMMI10", 10.0, 0, 0, BBox(0, 0, 10, 10), (0, 10))
        assert _equation_complexity([span], 10.0) == 0

    def test_multicolumn_detection_single_col(self):
        """Single-column pages should return 1 column."""
        from paper2md.extractors.layout import detect_page_columns
        from paper2md.types import PipelineConfig
        assert len(detect_page_columns([], 612.0, PipelineConfig())) == 1

    def test_split_capitalized_names(self):
        """Name splitter should split consecutive capitalized words into 2-word names."""
        from paper2md.converters.pdf import _split_capitalized_names
        names = _split_capitalized_names("Kaiming He Xiangyu Zhang Shaoqing Ren Jian Sun")
        assert names == ["Kaiming He", "Xiangyu Zhang", "Shaoqing Ren", "Jian Sun"]

    def test_split_names_with_middle_initial(self):
        """Name splitter should handle middle initials."""
        from paper2md.converters.pdf import _split_capitalized_names
        names = _split_capitalized_names("Tom B. Brown Benjamin Mann Nick Ryder Melanie Subbiah")
        assert names[0] == "Tom B. Brown"
        assert "Benjamin Mann" in names

    def test_author_extraction_no_false_positives(self):
        """Known non-name patterns should be filtered."""
        from paper2md.converters.pdf import _extract_names_from_text
        # Single words, institutions, locations should not appear
        names = _extract_names_from_text("Recent Google Brain research at Johns Hopkins University")
        assert "Recent" not in names
        assert "Johns Hopkins" not in names


# === OCR / Scanned PDF pipeline ===

MOCK_SCANNED_PDF = TEST_PAPERS / "mock_scanned.pdf"


@pytest.mark.skipif(not MOCK_SCANNED_PDF.exists(), reason="Mock scanned PDF not available")
class TestOCR:
    """Tests for the OCR pipeline using a mock scanned PDF."""

    def test_detected_as_scanned(self):
        """The mock PDF (image-only pages) should be detected as scanned."""
        from paper2md.converters.pdf import PdfConverter
        import fitz
        doc = fitz.open(str(MOCK_SCANNED_PDF))
        assert PdfConverter._is_scanned_pdf(doc)
        doc.close()

    def test_no_extractable_text(self):
        """Each page should have <50 chars of extractable text."""
        import fitz
        doc = fitz.open(str(MOCK_SCANNED_PDF))
        for i in range(len(doc)):
            text = doc[i].get_text("text").strip()
            assert len(text) < 50, f"Page {i} has {len(text)} extractable chars"
        doc.close()

    def test_conversion_produces_output_or_warning(self):
        """Conversion should produce OCR text (if tesseract is available) or a warning."""
        from paper2md import convert
        r = convert(MOCK_SCANNED_PDF)
        assert r.page_count == 3
        assert "Scanned PDF detected" in r.warnings[0]
        if r.markdown.strip():
            # Tesseract is available and produced output
            assert len(r.markdown) > 50
        else:
            # Tesseract not available - should have warning
            assert any("Tesseract" in w for w in r.warnings)

    def test_ocr_extracts_title(self):
        """If OCR works, it should extract a reasonable title."""
        from paper2md import convert
        r = convert(MOCK_SCANNED_PDF)
        if r.markdown.strip():
            # OCR produced text - title should be non-empty
            assert r.title, "Expected a title from OCR output"

    def test_ocr_warning_present(self):
        """Conversion of scanned PDF should always include OCR quality warning."""
        from paper2md import convert
        r = convert(MOCK_SCANNED_PDF)
        assert any("OCR" in w for w in r.warnings)


# === New diverse publisher format tests ===


class TestCodex:
    """Codex paper — OpenAI format with running headers."""

    def test_title(self):
        r = _convert("codex")
        assert "Evaluating Large Language" in r.title

    def test_authors(self):
        r = _convert("codex")
        assert len(r.authors) >= 10

    def test_abstract(self):
        r = _convert("codex")
        assert len(r.abstract) > 100
        assert "Codex" in r.abstract

    def test_references(self):
        r = _convert("codex")
        assert len(r.references) >= 50


class TestSKLearn:
    """scikit-learn paper — JMLR format."""

    def test_title(self):
        r = _convert("sklearn")
        assert "Scikit-learn" in r.title

    def test_authors(self):
        r = _convert("sklearn")
        assert len(r.authors) >= 10

    def test_abstract(self):
        r = _convert("sklearn")
        assert len(r.abstract) > 30

    def test_references(self):
        r = _convert("sklearn")
        assert len(r.references) >= 10


class TestAdam:
    """Adam optimizer — ICLR format."""

    def test_title(self):
        r = _convert("adam")
        assert "Adam" in r.title

    def test_authors(self):
        r = _convert("adam")
        assert len(r.authors) >= 2
        assert any("Kingma" in a for a in r.authors)

    def test_abstract(self):
        r = _convert("adam")
        assert len(r.abstract) > 100

    def test_sections(self):
        r = _convert("adam")
        h2s = [l for l in r.markdown.split("\n") if l.startswith("## ")]
        assert len(h2s) >= 3


class TestUNet:
    """U-Net — Springer LNCS format."""

    def test_title(self):
        r = _convert("unet")
        assert "U-Net" in r.title

    def test_authors(self):
        r = _convert("unet")
        assert any("Ronneberger" in a for a in r.authors)

    def test_abstract(self):
        r = _convert("unet")
        assert len(r.abstract) > 50

    def test_references(self):
        r = _convert("unet")
        assert len(r.references) >= 10


class TestGAN:
    """GAN — original NeurIPS format."""

    def test_title(self):
        r = _convert("gan")
        assert "Generative Adversarial" in r.title

    def test_authors(self):
        r = _convert("gan")
        assert len(r.authors) >= 5
        assert any("Goodfellow" in a for a in r.authors)

    def test_abstract(self):
        r = _convert("gan")
        assert len(r.abstract) > 100

    def test_references(self):
        r = _convert("gan")
        assert len(r.references) >= 20
