"""Unit tests for HTMLDownloader._clean_text, _extract_content, _format_extracted_content.

Production code: src/local_deep_research/research_library/downloaders/html.py
"""

from local_deep_research.research_library.downloaders.html import HTMLDownloader


# ---------------------------------------------------------------------------
# _clean_text
# ---------------------------------------------------------------------------


class TestCleanText:
    """Tests for HTMLDownloader._clean_text whitespace normalization."""

    def setup_method(self):
        self.downloader = HTMLDownloader()

    def test_multiple_newlines_collapsed(self):
        # 3+ newlines → 2 newlines, then blank line stripped → single newline
        assert self.downloader._clean_text("a\n\n\n\nb") == "a\nb"

    def test_multiple_spaces_collapsed(self):
        assert self.downloader._clean_text("a   b") == "a b"

    def test_tabs_replaced(self):
        assert self.downloader._clean_text("a\t\tb") == "a b"

    def test_blank_lines_removed(self):
        result = self.downloader._clean_text("a\n   \nb")
        assert result == "a\nb"

    def test_empty_string(self):
        assert self.downloader._clean_text("") == ""

    def test_already_clean(self):
        text = "hello world\nfoo bar"
        assert self.downloader._clean_text(text) == text

    def test_mixed_whitespace(self):
        result = self.downloader._clean_text("a\t\t\n\n\n\n  b  c")
        # tabs → single space, multiple newlines → two, multiple spaces → one
        assert "a" in result
        assert "b c" in result

    def test_lines_stripped(self):
        result = self.downloader._clean_text("  hello  \n  world  ")
        assert result == "hello\nworld"


# ---------------------------------------------------------------------------
# _extract_content
# ---------------------------------------------------------------------------


def _long_text(n=120):
    """Generate text longer than the 100-char fallback threshold."""
    return "A" * n


class TestExtractContent:
    """Tests for HTMLDownloader._extract_content HTML parsing."""

    def setup_method(self):
        self.downloader = HTMLDownloader()

    def test_minimal_html_with_body(self):
        long = _long_text()
        html = f"<html><body><p>{long}</p></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_article_tag_preferred(self):
        long = _long_text()
        html = f"""<html><body>
            <div>sidebar stuff that is long enough to pass {_long_text()}</div>
            <article><p>{long}</p></article>
        </body></html>"""
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_main_tag_fallback(self):
        long = _long_text()
        html = f"<html><body><main><p>{long}</p></main></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_content_div_fallback(self):
        long = _long_text()
        html = f'<html><body><div class="content"><p>{long}</p></div></body></html>'
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_script_style_nav_removed(self):
        long = _long_text()
        html = f"""<html><body>
            <script>var x = 1;</script>
            <style>.foo {{ color: red; }}</style>
            <nav>navigation links</nav>
            <p>{long}</p>
        </body></html>"""
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert "var x" not in result["content"]
        assert "color: red" not in result["content"]

    def test_sidebar_pattern_removed(self):
        long = _long_text()
        html = f"""<html><body>
            <div class="sidebar">sidebar content</div>
            <p>{long}</p>
        </body></html>"""
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert "sidebar content" not in result["content"]

    def test_short_content_falls_back_to_get_text(self):
        """Content < 100 chars from structured extraction → falls back to get_text."""
        html = "<html><body><p>Short text here but enough to be above fifty characters for the final check.</p></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        # Should still extract something via fallback
        assert result is not None
        assert "Short text" in result["content"]

    def test_no_body_returns_none(self):
        # No body/article/main tag
        html = "<html><head><title>Title</title></head></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is None

    def test_title_extraction(self):
        long = _long_text()
        html = f"<html><head><title>My Title</title></head><body><p>{long}</p></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert result["title"] == "My Title"

    def test_url_passed_through(self):
        long = _long_text()
        html = f"<html><body><p>{long}</p></body></html>"
        result = self.downloader._extract_content(
            html, "http://example.com/page"
        )
        assert result["url"] == "http://example.com/page"


# ---------------------------------------------------------------------------
# _format_extracted_content
# ---------------------------------------------------------------------------


class TestFormatExtractedContent:
    """Tests for HTMLDownloader._format_extracted_content."""

    def setup_method(self):
        self.downloader = HTMLDownloader()

    def test_full_dict(self):
        extracted = {
            "title": "My Title",
            "description": "A description",
            "url": "http://example.com",
            "content": "Main content here",
        }
        result = self.downloader._format_extracted_content(extracted)
        assert "# My Title" in result
        assert "*A description*" in result
        assert "Source: http://example.com" in result
        assert "Main content here" in result

    def test_missing_optional_fields(self):
        extracted = {
            "title": None,
            "description": None,
            "url": "http://example.com",
            "content": "Main content",
        }
        result = self.downloader._format_extracted_content(extracted)
        assert "#" not in result  # no title heading
        assert "*" not in result  # no description italic
        assert "Source: http://example.com" in result
        assert "Main content" in result

    def test_empty_content(self):
        extracted = {
            "title": None,
            "description": None,
            "url": None,
            "content": None,
        }
        result = self.downloader._format_extracted_content(extracted)
        # Should not crash, just return minimal/empty output
        assert isinstance(result, str)
