"""End-to-end tests for the full SEC filing pipeline with mocked HTTP."""

import lzma
import os
from unittest.mock import MagicMock, patch

from secbulkdownload.secfiling import filingArchive


class TestEndToEndFlow:
    """Test the complete flow from index to document extraction."""

    def _create_mock_filing_content(self) -> bytes:
        """Create realistic SEC filing content."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
ACCESSION NUMBER:		0000320193-23-000077
TYPE:		10-K
CIK:		0000320193
CONFORMED NAME:		Apple Inc.
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>aapl-20220930.htm
<DESCRIPTION>Form 10-K Annual Report
<TEXT>
<html>
<head><title>Form 10-K</title></head>
<body>
<h1>Apple Inc.</h1>
<h2>Form 10-K for Fiscal Year Ended September 30, 2022</h2>
<p>Total Assets: $352,583 million</p>
<p>Total Liabilities: $283,263 million</p>
<p>Stockholders' Equity: $69,320 million</p>
</body>
</html>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-21
<SEQUENCE>2
<FILENAME>aapl-subsidiaries.htm
<DESCRIPTION>List of Subsidiaries
<TEXT>
<html>
<body>
<h2>List of Subsidiaries</h2>
<ul>
<li>Apple Operations Inc.</li>
<li>Apple Operations Europe Ltd.</li>
<li>Apple Asia LLC</li>
</ul>
</body>
</html>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-31.1
<SEQUENCE>3
<FILENAME>aapl-ex311.htm
<DESCRIPTION>Certification pursuant to Section 302
<TEXT>
<html>
<body>
<p>CERTIFICATION</p>
<p>I, Timothy D. Cook, certify that...</p>
</body>
</html>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-32.1
<SEQUENCE>4
<FILENAME>aapl-ex321.txt
<DESCRIPTION>Certification pursuant to Section 906
<TEXT>
CERTIFICATION

I, Timothy D. Cook, certify that the Form 10-K fully complies...
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-101.INS
<SEQUENCE>5
<FILENAME>aapl-20220930.xml
<DESCRIPTION>XBRL Instance Document
<TEXT>
<XML>
<?xml version="1.0"?>
<xbrl xmlns="http://www.xbrl.org/2003/instance">
  <context id="c1">
    <entity>
      <identifier scheme="http://www.sec.gov/CIK">0000320193</identifier>
    </entity>
    <period><instant>2022-09-30</instant></period>
  </context>
  <us-gaap:Assets contextRef="c1" unitRef="USD">352583000000</us-gaap:Assets>
  <us-gaap:Liabilities contextRef="c1" unitRef="USD">283263000000</us-gaap:Liabilities>
  <us-gaap:StockholdersEquity contextRef="c1" unitRef="USD">69320000000</us-gaap:StockholdersEquity>
</xbrl>
</XML>
</TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""
        return content.encode("latin-1")

    def test_full_pipeline_from_index_to_documents(self, datadir: str) -> None:
        """Test the complete flow: index -> fetch filing -> extract documents."""
        # Create a mock working_idx with a filing record
        mock_content = self._create_mock_filing_content()

        # Pre-create the cached filing file (simulating that fetch_filings already ran)
        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_content))

        # Now create a filingArchive and extract documents
        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {"TYPE": "10-K", "CIK": "0000320193"}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        # Verify document extraction
        assert len(archive.files) == 5
        filenames = archive.get_filenames()
        assert "aapl-20220930.htm" in filenames
        assert "aapl-subsidiaries.htm" in filenames
        assert "aapl-ex311.htm" in filenames
        assert "aapl-ex321.txt" in filenames
        assert "aapl-20220930.xml" in filenames

        # Verify document types
        doc_types = [f.type for f in archive.files]
        assert "10-K" in doc_types
        assert "EX-21" in doc_types
        assert "EX-31.1" in doc_types
        assert "EX-32.1" in doc_types
        assert "EX-101.INS" in doc_types

    def test_pipeline_filter_documents_by_type(self, datadir: str) -> None:
        """Test filtering documents during extraction."""
        mock_content = self._create_mock_filing_content()

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_content))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            # Filter for only 10-K
            archive.get_filingArchive(document_types="10-K", text_only=True)

        # Should only include the 10-K document
        assert len(archive.files) == 1
        assert archive.files[0].type == "10-K"

    def test_pipeline_get_financial_data_from_xbrl(self, datadir: str) -> None:
        """Test extracting financial data from XBRL document."""
        mock_content = self._create_mock_filing_content()

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_content))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(document_types="EX-101.INS", text_only=True)

        # Should only have the XBRL instance
        assert len(archive.files) == 1
        xbrl_doc = archive.files[0]

        # Verify XBRL content is present (body has stripped tags, so check values)
        assert "352583000000" in xbrl_doc.body  # Assets value
        assert "0000320193" in xbrl_doc.body  # CIK


class TestMultipleFilingsFlow:
    """Test handling multiple filings in a single flow."""

    def _create_mock_index_content(self) -> bytes:
        """Create mock SEC index file content."""
        content = b"""CIK|Company Name|Form Type|Date Filed|Filename
0000320193|Apple Inc.|10-K|2023-01-01|edgar/data/0000320193/0000320193-23-000077.txt
0000789019|Microsoft Corp|10-K|2023-01-02|edgar/data/0000789019/0000789019-23-000123.txt
"""
        return content

    def test_multiple_filings_fetch_and_extract(self, datadir: str) -> None:
        """Test fetching and extracting multiple filings."""
        # Create mock filing contents
        apple_content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>apple10k.htm
<DESCRIPTION>Apple 10-K
<TEXT><html><body>Apple 10-K Content</body></html></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""

        msft_content = """<SEC-DOCUMENT>0000789019-23-000123.txt : 20230102
<SEC-HEADER>0000789019-23-000123.hdr.sgml : 20230102
<ACCEPTANCE-DATETIME>20230102120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>msft10k.htm
<DESCRIPTION>Microsoft 10-K
<TEXT><html><body>Microsoft 10-K Content</body></html></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""

        # Pre-create cached files
        for cik, accession, content in [
            ("0000320193", "0000320193-23-000077", apple_content),
            ("0000789019", "0000789019-23-000123", msft_content),
        ]:
            filings_dir = os.path.join(datadir, "filings", cik)
            os.makedirs(filings_dir, exist_ok=True)
            cached_file = os.path.join(filings_dir, f"{accession}.txt.xz")
            with open(cached_file, "wb") as f:
                f.write(lzma.compress(content.encode("latin-1")))

        # Create filingArchives and extract
        archives = []
        for sec_filepath in [
            "edgar/data/0000320193/0000320193-23-000077.txt",
            "edgar/data/0000789019/0000789019-23-000123.txt",
        ]:
            with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
                mock_hdr = MagicMock()
                mock_hdr.get_headerDict.return_value = {}
                mock_headerfile.return_value = mock_hdr

                archive = filingArchive(
                    sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
                )
                archive.get_filingArchive(text_only=True)
                archives.append(archive)

        # Verify both archives extracted correctly
        assert len(archives) == 2
        assert archives[0].files[0].filename == "apple10k.htm"
        assert archives[1].files[0].filename == "msft10k.htm"


class TestCacheRecoveryFlow:
    """Test cache recovery scenarios."""

    def test_recover_from_cached_filing(self, datadir: str) -> None:
        """Test that filing can be recovered from local cache without network."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>report.htm
<DESCRIPTION>Report
<TEXT><html><body>Content</body></html></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""

        # Create cached filing
        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(content.encode("latin-1")))

        # Access without any network calls
        with (
            patch("secbulkdownload.secfiling.headerfile") as mock_headerfile,
            patch("secbulkdownload.secfiling.fetch_sec_file") as mock_fetch,
        ):
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

            # Should NOT call fetch_sec_file since file exists
            mock_fetch.assert_not_called()

        # Verify extraction worked
        assert len(archive.files) == 1
        assert archive.files[0].filename == "report.htm"

    def test_fetch_missing_filing(self, datadir: str) -> None:
        """Test that missing filing is fetched from SEC."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>report.htm
<DESCRIPTION>Report
<TEXT><html><body>Content</body></html></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""

        with (
            patch("secbulkdownload.secfiling.headerfile") as mock_headerfile,
            patch("secbulkdownload.secfiling.fetch_sec_file") as mock_fetch,
        ):
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            # Return tuple (localpath, data) as fetch_sec_file does
            mock_fetch.return_value = (
                "/path/to/file.xz",
                content.encode("latin-1"),
            )

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

            # Should have called fetch_sec_file
            mock_fetch.assert_called_once()


class TestErrorHandling:
    """Test error handling in the pipeline."""

    def test_malformed_archive_content(self, datadir: str) -> None:
        """Test handling of malformed archive content."""
        # Create a malformed cached file
        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(b"This is not valid SEC content"))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            # Should not raise, but may produce empty results
            archive.get_filingArchive(text_only=True)

        # Files list should be empty (no documents found)
        assert len(archive.files) == 0

    def test_missing_document_tags(self, datadir: str) -> None:
        """Test handling of documents with missing required tags."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<SEQUENCE>1
<FILENAME>report.htm
<TEXT><html><body>Content</body></html></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(content.encode("latin-1")))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        # Should still extract document with default values
        assert len(archive.files) == 1
        assert archive.files[0].type == "_"  # Default for missing type
        assert archive.files[0].description == "_"  # Default for missing description


class TestMetadataPreservation:
    """Test that metadata is preserved throughout the pipeline."""

    def test_acceptance_datetime_preserved(self, datadir: str) -> None:
        """Test that acceptance-datetime is correctly extracted."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230115
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230115
<ACCEPTANCE-DATETIME>20230115143022
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>report.htm
<DESCRIPTION>Report
<TEXT><html><body>Content</body></html></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(content.encode("latin-1")))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        # Acceptance datetime should be first 14 chars
        assert archive.acceptance_datetime == "20230115143022"

    def test_header_metadata_available(self, datadir: str) -> None:
        """Test that header metadata is available from headerfile."""
        content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>report.htm
<DESCRIPTION>Report
<TEXT><html><body>Content</body></html></TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""

        filings_dir = os.path.join(datadir, "filings", "0000320193")
        os.makedirs(filings_dir, exist_ok=True)
        cached_file = os.path.join(filings_dir, "0000320193-23-000077.txt.xz")
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(content.encode("latin-1")))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {
                "TYPE": "10-K",
                "CIK": "0000320193",
                "CONFORMED-NAME": "Apple Inc.",
            }
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        # Header should be available
        assert archive.header["TYPE"] == "10-K"
        assert archive.header["CIK"] == "0000320193"
