"""Tests for HTMLDownloader pure-logic methods.

Covers: can_handle, _clean_text, _extract_content, _format_extracted_content.
Network/session methods are NOT tested here — only HTML parsing logic.
"""

import pytest

from local_deep_research.research_library.downloaders.html import HTMLDownloader


@pytest.fixture
def dl():
    """Create an HTMLDownloader instance for testing pure-logic methods."""
    downloader = HTMLDownloader(timeout=30)
    yield downloader
    downloader.close()


def make_html(body, head=""):
    """Build a minimal HTML document from head and body fragments."""
    return f"<html><head>{head}</head><body>{body}</body></html>"


# Long paragraph that exceeds both the 10-char fragment threshold
# and contributes enough to pass the 50-char minimum content threshold.
LONG_PARA = "This is a sufficiently long paragraph for content extraction testing purposes."
ANOTHER_PARA = "Another paragraph with enough characters to pass all the length thresholds."


# ===================================================================
# can_handle
# ===================================================================


class TestHTMLDownloaderCanHandle:
    """Tests for HTMLDownloader.can_handle()."""

    def test_http_url(self, dl):
        """Accepts http:// URLs."""
        assert dl.can_handle("http://example.com") is True

    def test_https_url(self, dl):
        """Accepts https:// URLs."""
        assert dl.can_handle("https://example.com/page") is True

    def test_ftp_url(self, dl):
        """Rejects ftp:// URLs."""
        assert dl.can_handle("ftp://files.example.com/data") is False

    def test_empty_string(self, dl):
        """Rejects empty string."""
        assert dl.can_handle("") is False

    def test_plain_text(self, dl):
        """Rejects plain text that is not a URL."""
        assert dl.can_handle("not-a-url") is False


# ===================================================================
# _clean_text
# ===================================================================


class TestCleanText:
    """Tests for HTMLDownloader._clean_text()."""

    def test_removes_blank_lines_from_multiple_newlines(self, dl):
        """Verifies that the combined pipeline strips blank lines from input with multiple newlines."""
        result = dl._clean_text("a\n\n\n\nb")
        assert result == "a\nb"

    def test_collapses_multiple_spaces(self, dl):
        """Collapses runs of spaces into a single space."""
        result = dl._clean_text("hello    world")
        assert result == "hello world"

    def test_converts_tabs_to_spaces(self, dl):
        """Converts tabs to spaces and collapses them."""
        result = dl._clean_text("hello\t\tworld")
        assert result == "hello world"

    def test_strips_whitespace_only_lines(self, dl):
        """Removes lines that contain only whitespace."""
        result = dl._clean_text("line1\n   \nline2")
        assert result == "line1\nline2"

    def test_strips_leading_trailing_whitespace_per_line(self, dl):
        """Strips leading and trailing whitespace from each line."""
        result = dl._clean_text("  hello  \n  world  ")
        assert result == "hello\nworld"

    def test_empty_string(self, dl):
        """Returns empty string for empty input."""
        result = dl._clean_text("")
        assert result == ""


# ===================================================================
# _extract_content
# ===================================================================


class TestExtractContent:
    """Tests for HTMLDownloader._extract_content()."""

    # --- Title extraction ---

    def test_extracts_title_from_title_tag(self, dl):
        """Extracts title from <title> tag."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head="<title>My Page Title</title>",
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["title"] == "My Page Title"

    def test_og_title_overrides_title_tag(self, dl):
        """Prefers og:title over <title> tag."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head='<title>HTML Title</title><meta property="og:title" content="OG Title">',
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["title"] == "OG Title"

    def test_no_title_returns_none_title(self, dl):
        """Returns None title when no title tag present."""
        html = make_html(f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>")
        result = dl._extract_content(html, "http://example.com")
        assert result["title"] is None

    # --- Description extraction ---

    def test_extracts_meta_description(self, dl):
        """Extracts description from meta tag."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head='<meta name="description" content="A page description">',
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["description"] == "A page description"

    def test_og_description_overrides_meta(self, dl):
        """Prefers og:description over meta description."""
        html = make_html(
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>",
            head=(
                '<meta name="description" content="Meta desc">'
                '<meta property="og:description" content="OG desc">'
            ),
        )
        result = dl._extract_content(html, "http://example.com")
        assert result["description"] == "OG desc"

    # --- Element removal ---

    def test_removes_script_and_style(self, dl):
        """Strips <script> and <style> elements."""
        html = make_html(
            f"<script>alert('x')</script><style>.x{{}}</style>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "alert" not in result["content"]

    def test_removes_nav_footer_aside(self, dl):
        """Strips <nav>, <footer>, and <aside> elements."""
        html = make_html(
            f"<nav>Navigation</nav><footer>Footer</footer><aside>Sidebar</aside>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "Navigation" not in result["content"]
        assert "Footer" not in result["content"]
        assert "Sidebar" not in result["content"]

    def test_removes_elements_matching_class_patterns(self, dl):
        """Removes elements whose class matches REMOVE_PATTERNS."""
        html = make_html(
            f'<div class="sidebar-widget">Sidebar stuff</div>'
            f'<div class="cookie-banner">Accept cookies</div>'
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "Sidebar stuff" not in result["content"]
        assert "Accept cookies" not in result["content"]

    def test_removes_elements_matching_id_patterns(self, dl):
        """Verifies that REMOVE_PATTERNS also matches element IDs."""
        html = make_html(
            f'<div id="sidebar-widget">Sidebar by ID</div>'
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "Sidebar by ID" not in result["content"]

    # --- Content area detection ---

    def test_prefers_article_tag(self, dl):
        """Prefers <article> as content area."""
        html = make_html(
            f"<article><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></article>"
            f"<p>Outside article content that should not appear here ideally</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    def test_falls_back_to_main_when_no_article(self, dl):
        """Falls back to <main> when no <article>."""
        html = make_html(
            f"<main><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></main>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    def test_falls_back_to_content_div(self, dl):
        """Falls back to div.content when no semantic containers."""
        html = make_html(
            f'<div class="content"><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p></div>'
        )
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    def test_falls_back_to_body(self, dl):
        """Falls back to <body> when no content containers found."""
        html = make_html(f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>")
        result = dl._extract_content(html, "http://example.com")
        assert LONG_PARA in result["content"]

    # --- Text structure ---

    def test_headings_formatted_as_markdown(self, dl):
        """Converts HTML headings to markdown format."""
        html = make_html(
            f"<h2>Important Section Heading</h2>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "## Important Section Heading" in result["content"]

    def test_list_items_formatted_with_bullet(self, dl):
        """Formats list items with bullet character."""
        html = make_html(
            f"<ul><li>First list item text here</li></ul>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "• First list item text here" in result["content"]

    def test_blockquotes_formatted_with_angle(self, dl):
        """Formats blockquotes with angle bracket prefix."""
        html = make_html(
            f"<blockquote>A notable quoted passage here</blockquote>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "> A notable quoted passage here" in result["content"]

    def test_pre_blocks_wrapped_in_backticks(self, dl):
        """Wraps <pre> blocks in triple backticks."""
        html = make_html(
            f"<pre>def hello_world_function(): pass</pre>"
            f"<p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>"
        )
        result = dl._extract_content(html, "http://example.com")
        assert "```" in result["content"]
        assert "def hello_world_function(): pass" in result["content"]

    # --- Edge cases ---

    def test_skips_short_paragraph_fragments(self, dl):
        """Skips paragraphs shorter than the fragment threshold."""
        html = make_html(f"<p>Short</p><p>{LONG_PARA}</p><p>{ANOTHER_PARA}</p>")
        result = dl._extract_content(html, "http://example.com")
        # "Short" is <=10 chars so should be skipped
        assert "Short" not in result["content"]

    def test_returns_none_when_content_too_short(self, dl):
        """Returns None when extracted content is below minimum threshold."""
        html = make_html("<p>Tiny</p>")
        result = dl._extract_content(html, "http://example.com")
        assert result is None

    def test_returns_none_when_no_body(self, dl):
        """Returns None when HTML has no <body>."""
        result = dl._extract_content(
            "<html><head></head></html>", "http://example.com"
        )
        assert result is None

    def test_falls_back_to_get_text_when_structured_too_short(self, dl):
        """Falls back to get_text() when structured extraction is too short."""
        # Content that has lots of text but not in p/h/li/blockquote/pre tags
        # so structured extraction yields < 100 chars, triggering get_text fallback
        long_text = "A" * 200
        html = make_html(f"<div><span>{long_text}</span></div>")
        result = dl._extract_content(html, "http://example.com")
        assert result is not None
        assert long_text in result["content"]


# ===================================================================
# _format_extracted_content
# ===================================================================


class TestFormatExtractedContent:
    """Tests for HTMLDownloader._format_extracted_content()."""

    def test_title_as_heading(self, dl):
        """Renders title as a markdown heading."""
        result = dl._format_extracted_content(
            {
                "title": "My Title",
                "description": None,
                "url": None,
                "content": "body",
            }
        )
        assert result.startswith("# My Title")

    def test_description_in_italics(self, dl):
        """Renders description in italics."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": "A desc",
                "url": None,
                "content": "body",
            }
        )
        assert "*A desc*" in result

    def test_source_url_line(self, dl):
        """Includes source URL line when URL is present."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": None,
                "url": "http://example.com",
                "content": "body",
            }
        )
        assert "Source: http://example.com" in result

    def test_content_included(self, dl):
        """Includes body content in output."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": None,
                "url": None,
                "content": "Main body text",
            }
        )
        assert "Main body text" in result

    def test_missing_fields_omitted(self, dl):
        """Omits title, description, and source when they are None."""
        result = dl._format_extracted_content(
            {
                "title": None,
                "description": None,
                "url": None,
                "content": "Only content",
            }
        )
        assert "#" not in result
        assert "*" not in result
        assert "Source:" not in result
        assert "Only content" in result
