"""Tests for XML-only filings with text_only filtering."""

import lzma
import os
from unittest.mock import MagicMock, patch

from secbulkdownload.secfiling import filingArchive


class TestXMLFilingFiltering:
    """Test handling of XML-only filings with text_only=True."""

    def _create_xbrl_archive(self) -> bytes:
        """Create a typical XBRL filing with XML documents."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
ACCESSION NUMBER:		0000320193-23-000077
TYPE:		10-K
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>apple10k.htm
<DESCRIPTION>Form 10-K Annual Report
<TEXT>
<html><body><h1>Apple Inc. Form 10-K</h1><p>Financial statements...</p></body></html>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.INS
<SEQUENCE>2
<FILENAME>apple-20230930.xml
<DESCRIPTION>XBRL INSTANCE DOCUMENT
<TEXT>
<XML>
<?xml version="1.0"?>
<xbrl xmlns="http://www.xbrl.org/2003/instance">
  <context id="c1">
    <entity><identifier scheme="http://www.sec.gov/CIK">0000320193</identifier></entity>
    <period><instant>2023-09-30</instant></period>
  </context>
  <us-gaap:Assets contextRef="c1" unitRef="USD">352583000000</us-gaap:Assets>
</xbrl>
</XML>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.SCH
<SEQUENCE>3
<FILENAME>apple-20230930.xsd
<DESCRIPTION>XBRL TAXONOMY EXTENSION SCHEMA
<TEXT>
<XML>
<?xml version="1.0"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
  <xs:element name="Assets" type="xs:decimal"/>
</xs:schema>
</XML>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.CAL
<SEQUENCE>4
<FILENAME>apple-20230930_cal.xml
<DESCRIPTION>XBRL TAXONOMY EXTENSION CALCULATION
<TEXT>
<XML>
<?xml version="1.0"?>
<calculation xmlns="http://www.xbrl.org/2003/instance">
  <calc:calcArc xlink:arcrole="summation-item"/>
</calculation>
</XML>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.DEF
<SEQUENCE>5
<FILENAME>apple-20230930_def.xml
<DESCRIPTION>XBRL TAXONOMY EXTENSION DEFINITION
<TEXT>
<XML>
<?xml version="1.0"?>
<definition xmlns="http://www.xbrl.org/2003/instance"/>
</XML>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.LAB
<SEQUENCE>6
<FILENAME>apple-20230930_lab.xml
<DESCRIPTION>XBRL TAXONOMY EXTENSION LABEL
<TEXT>
<XML>
<?xml version="1.0"?>
<labels xmlns="http://www.xbrl.org/2003/instance">
  <label>Assets</label>
</labels>
</XML>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.PRE
<SEQUENCE>7
<FILENAME>apple-20230930_pre.xml
<DESCRIPTION>XBRL TAXONOMY EXTENSION PRESENTATION
<TEXT>
<XML>
<?xml version="1.0"?>
<presentation xmlns="http://www.xbrl.org/2003/instance"/>
</XML>
</TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""
        return content.encode("latin-1")

    def test_xbrl_documents_with_text_only(self, datadir: str) -> None:
        """Test that XML documents are included when text_only=True."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(self._create_xbrl_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(text_only=True)

        # All documents should be included (htm + .xml files)
        # htm, xml, xsd, xml, xml, xml, xml are all text types
        assert len(archive.files) == 7

        # Verify we have the XBRL instance document
        xml_files = [f for f in archive.files if f.filename.endswith(".xml")]
        assert len(xml_files) == 5  # INS, CAL, DEF, LAB, PRE

    def test_xml_content_preserved(self, datadir: str) -> None:
        """Test that XML content is correctly extracted."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(self._create_xbrl_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(text_only=True)

        # Find the XBRL instance document
        xbrl_doc = next(
            (f for f in archive.files if f.filename == "apple-20230930.xml"), None
        )
        assert xbrl_doc is not None
        # body contains text content (stripped of tags), html contains raw markup
        # Check that financial data values are present in the body
        assert "352583000000" in xbrl_doc.body  # Assets value
        assert "0000320193" in xbrl_doc.body  # CIK

    def test_xsd_file_included_with_text_only(self, datadir: str) -> None:
        """Test that XSD files are included with text_only=True."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(self._create_xbrl_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(text_only=True)

        # XSD file should be included (xsd files are not excluded by text_only)
        xsd_doc = next(
            (f for f in archive.files if f.filename.endswith(".xsd")), None
        )
        assert xsd_doc is not None


class TestMixedXMLAndBinary:
    """Test handling of filings with XML documents and binary files."""

    def _create_mixed_archive(self) -> bytes:
        """Create archive with XML documents and binary files."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>report.htm
<DESCRIPTION>Annual Report
<TEXT><html><body>Report content</body></html></TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.INS
<SEQUENCE>2
<FILENAME>instance.xml
<DESCRIPTION>XBRL Instance
<TEXT><XML><?xml version="1.0"?><xbrl/></XML></TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>GRAPHIC
<SEQUENCE>3
<FILENAME>chart.png
<DESCRIPTION>Chart Image
<TEXT>[binary PNG data would be here]</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-23
<SEQUENCE>4
<FILENAME>consent.pdf
<DESCRIPTION>Auditor Consent
<TEXT>[binary PDF data would be here]</TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""
        return content.encode("latin-1")

    def test_binary_excluded_with_text_only(self, datadir: str) -> None:
        """Test that binary files (png, pdf) are excluded with text_only=True."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(self._create_mixed_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(text_only=True)

        # Should only include htm and xml files
        assert len(archive.files) == 2
        extensions = [f.filename.split(".")[-1].lower() for f in archive.files]
        assert "htm" in extensions
        assert "xml" in extensions
        assert "png" not in extensions
        assert "pdf" not in extensions

    def test_binary_included_with_text_only_false(self, datadir: str) -> None:
        """Test that binary files are included when text_only=False."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(self._create_mixed_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(text_only=False)

        # Should include all 4 documents
        assert len(archive.files) == 4


class TestXMLDocumentTypes:
    """Test filtering by document type for XML documents."""

    def _create_xbrl_only_archive(self) -> bytes:
        """Create archive with only XBRL documents."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>EX-101.INS
<SEQUENCE>1
<FILENAME>instance.xml
<DESCRIPTION>XBRL Instance
<TEXT><XML><?xml version="1.0"?><xbrl><data>test</data></xbrl></XML></TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.SCH
<SEQUENCE>2
<FILENAME>schema.xsd
<DESCRIPTION>XBRL Schema
<TEXT><XML><?xml version="1.0"?><xs:schema/></XML></TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.CAL
<SEQUENCE>3
<FILENAME>calculation.xml
<DESCRIPTION>XBRL Calculation
<TEXT><XML><?xml version="1.0"?><calc/></XML></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""
        return content.encode("latin-1")

    def test_filter_xbrl_instance_type(self, datadir: str) -> None:
        """Test filtering for EX-101.INS document type."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(self._create_xbrl_only_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(document_types="EX-101.INS", text_only=True)

        assert len(archive.files) == 1
        assert archive.files[0].type == "EX-101.INS"
        assert archive.files[0].filename == "instance.xml"

    def test_filter_multiple_xbrl_types(self, datadir: str) -> None:
        """Test filtering for multiple XBRL document types."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(self._create_xbrl_only_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(
                document_types=["EX-101.INS", "EX-101.SCH"], text_only=True
            )

        assert len(archive.files) == 2
        types = [f.type for f in archive.files]
        assert "EX-101.INS" in types
        assert "EX-101.SCH" in types


class TestXMLWithUpperCaseExtension:
    """Test handling of XML files with uppercase extension."""

    def _create_uppercase_xml_archive(self) -> bytes:
        """Create archive with uppercase .XML extension."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>EX-101.INS
<SEQUENCE>1
<FILENAME>instance.XML
<DESCRIPTION>XBRL Instance
<TEXT><XML><?xml version="1.0"?><xbrl/></XML></TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.CAL
<SEQUENCE>2
<FILENAME>calc.XML
<DESCRIPTION>Calculation
<TEXT><XML><?xml version="1.0"?><calc/></XML></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""
        return content.encode("latin-1")

    def test_uppercase_xml_included(self, datadir: str) -> None:
        """Test that .XML (uppercase) files are included with text_only=True."""
        sec_filepath = "edgar/data/0000320193/0000320193-23-000077.txt"

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as fp:
            fp.write(lzma.compress(self._create_uppercase_xml_archive()))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive(text_only=True)

        # Both XML files should be included
        assert len(archive.files) == 2
        for doc in archive.files:
            assert doc.filename.upper().endswith(".XML")
