"""Tests for fresh bootstrap from empty datadir with mocked SEC.

These tests verify the full pipeline works from scratch:
1. Empty datadir -> idx.updateidx, filter_index, fetch_filings(headers=False)
2. filingArchive construction, document reading
3. No pre-seeded caches needed
"""

import gzip
import lzma
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, patch

from secbulkdownload import filingArchive, idx

# Canonical columns for SEC index files
CANONICAL_COLUMNS = ["CIK", "Company Name", "Form Type", "Date Filed", "Filename"]


def create_mock_master_gz_content(records: list[dict[str, Any]]) -> bytes:
    """Create mock master.gz SEC index file content.

    Args:
        records: List of dicts with keys matching canonical schema

    Returns:
        Gzip-compressed content bytes
    """
    header_lines = [
        "Description:           Master Index of EDGAR Dissemination Feed",
        "Last Updated:          2024-01-01",
        "Comments:             comments@sec.gov",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/",
        "Cloud HTTP:           https://www.sec.gov/Archives/",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/full-index/",
        "SFTP:                 sftp://ftp.sec.gov/edgar/full-index/",
        "",
        "",
    ]
    column_header = "CIK|Company Name|Form Type|Date Filed|Filename"
    separator_line = "-" * 80

    data_lines = []
    for rec in records:
        cik = str(rec.get("CIK", ""))
        company = rec.get("Company Name", "")
        form = rec.get("Form Type", "")
        date_filed = rec.get("Date Filed", "2024-01-01")
        filename = rec.get("Filename", "")
        data_lines.append(f"{cik}|{company}|{form}|{date_filed}|{filename}")

    content = (
        "\n".join(header_lines)
        + "\n"
        + column_header
        + "\n"
        + separator_line
        + "\n"
        + "\n".join(data_lines)
        + "\n"
    )

    # Create gzip content
    compressed = gzip.compress(content.encode("latin-1"))
    return compressed


def create_mock_master_gz(tmp_path: Path, records: list[dict[str, Any]]) -> Path:
    """Create a mock master.gz SEC index file with given records.

    Args:
        tmp_path: Temporary directory path
        records: List of dicts with keys matching canonical schema

    Returns:
        Path to created gzip file
    """
    # SEC master.idx format has 9 metadata lines, then a column header row,
    # a dashed separator row, then data rows.
    header_lines = [
        "Description:           Master Index of EDGAR Dissemination Feed",
        "Last Updated:          2024-01-01",
        "Comments:             comments@sec.gov",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/",
        "Cloud HTTP:           https://www.sec.gov/Archives/",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/full-index/",
        "SFTP:                 sftp://ftp.sec.gov/edgar/full-index/",
        "",
        "",
    ]

    column_header = "CIK|Company Name|Form Type|Date Filed|Filename"
    separator_line = "-" * 80

    data_lines = []
    for rec in records:
        cik = str(rec.get("CIK", ""))
        company = rec.get("Company Name", "")
        form = rec.get("Form Type", "")
        date_filed = rec.get("Date Filed", "2024-01-01")
        filename = rec.get("Filename", "")
        data_lines.append(f"{cik}|{company}|{form}|{date_filed}|{filename}")

    content = (
        "\n".join(header_lines)
        + "\n"
        + column_header
        + "\n"
        + separator_line
        + "\n"
        + "\n".join(data_lines)
        + "\n"
    )

    # Create gzip file
    gz_path = tmp_path / "master.gz"
    with gzip.open(gz_path, "wt", encoding="latin-1") as f:
        f.write(content)

    return gz_path


def create_mock_filing_content() -> bytes:
    """Create realistic SEC filing content with multiple documents."""
    content = """<SEC-DOCUMENT>0000320193-23-000077.txt : 20230101
<SEC-HEADER>0000320193-23-000077.hdr.sgml : 20230101
<ACCEPTANCE-DATETIME>20230101120000
ACCESSION NUMBER:		0000320193-23-000077
TYPE:		10-K
CIK:		0000320193
CONFORMED NAME:		Apple Inc.
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>aapl-20220930.htm
<DESCRIPTION>Form 10-K Annual Report
<TEXT>
<html>
<head><title>Form 10-K</title></head>
<body>
<h1>Apple Inc.</h1>
<h2>Form 10-K for Fiscal Year Ended September 30, 2022</h2>
<p>Total Assets: $352,583 million</p>
</body>
</html>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-21
<SEQUENCE>2
<FILENAME>aapl-subsidiaries.htm
<DESCRIPTION>List of Subsidiaries
<TEXT>
<html>
<body>
<h2>List of Subsidiaries</h2>
<ul>
<li>Apple Operations Inc.</li>
</ul>
</body>
</html>
</TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""
    return content.encode("latin-1")


class TestFreshBootstrapPipeline:
    """Test full pipeline from empty datadir."""

    def test_fresh_idx_updateidx_creates_idxdir(self, datadir: str) -> None:
        """From empty datadir, updateidx should create idxfiles directory."""
        idxdir = Path(datadir) / "idxfiles"

        # Verify idxdir doesn't exist initially
        assert not idxdir.exists()

        # Mock fetch_sec_file to provide content
        mock_records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-15",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        mock_content = create_mock_master_gz_content(mock_records)

        with patch("secbulkdownload.secmeta.fetch_sec_file") as mock_fetch:
            mock_fetch.return_value = ("dummy.gz", mock_content)

            _ = idx(
                datadir=datadir,
                start_year=2024,
                start_quarter=1,
                end_year=2024,
                end_quarter=1,
                user_agent="Test Co test@test.com",
            )

        # idxdir should now exist
        assert idxdir.exists()

    def test_fresh_idx_filter_index(self, datadir: str) -> None:
        """From empty datadir, filter_index should work with mocked index files."""
        # Create idx instance first (with mocked network)
        with patch("secbulkdownload.secmeta.fetch_sec_file"):
            instance = idx(
                datadir=datadir,
                start_year=2024,
                start_quarter=1,
                end_year=2024,
                end_quarter=1,
                user_agent="Test Co test@test.com",
            )

        # Create mock index file AFTER instance creation (so it doesn't get deleted by updateidx)
        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        mock_records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-15",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            },
            {
                "CIK": "0000789019",
                "Company Name": "MICROSOFT CORP",
                "Form Type": "10-Q",
                "Date Filed": "2024-02-15",
                "Filename": "edgar/data/0000789019/0000789019-24-000002.txt",
            },
        ]
        gz_path = create_mock_master_gz(qtr_dir, mock_records)

        # Set idxlist to point to our mock file
        instance.idxlist = [str(gz_path)]

        # Filter for 10-K only
        result = instance.filter_index({"Form Type": ["10-K"]})

        assert isinstance(result, list)
        assert len(result) == 1
        assert result[0]["Form Type"] == "10-K"
        assert result[0]["CIK"] == "0000320193"

    def test_fresh_fetch_filings_headers_false(self, datadir: str) -> None:
        """From empty datadir, fetch_filings(headers=False) should work with pre-cached files."""
        # Create idx instance first (with mocked network)
        with patch("secbulkdownload.secmeta.fetch_sec_file"):
            instance = idx(
                datadir=datadir,
                start_year=2024,
                start_quarter=1,
                end_year=2024,
                end_quarter=1,
                user_agent="Test Co test@test.com",
            )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        # Create mock index file AFTER instance creation
        mock_records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-15",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        gz_path = create_mock_master_gz(qtr_dir, mock_records)

        # Pre-create cached filing file (simulating previous fetch)
        filings_dir = Path(datadir) / "filings" / "0000320193"
        filings_dir.mkdir(parents=True, exist_ok=True)
        mock_filing_content = create_mock_filing_content()
        cached_file = filings_dir / "0000320193-24-000001.txt.xz"
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_filing_content))

        # Set idxlist to point to our mock file
        instance.idxlist = [str(gz_path)]

        # Filter to get working_idx
        instance.filter_index({"Form Type": ["10-K"]})

        # Fetch filings without headers - should use cached file
        instance.fetch_filings(headers=False)

        # Should have found the cached filing
        assert len(instance.filinglist) >= 1


class TestFilingArchiveFreshBootstrap:
    """Test filingArchive construction from fresh datadir."""

    def test_filingArchive_from_cached_filing(self, datadir: str) -> None:
        """Test filingArchive can read a cached filing."""
        # Create filings directory and cache a filing
        filings_dir = Path(datadir) / "filings" / "0000320193"
        filings_dir.mkdir(parents=True, exist_ok=True)

        mock_content = create_mock_filing_content()
        cached_file = filings_dir / "0000320193-23-000077.txt.xz"
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_content))

        # Create filingArchive and extract documents
        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {"TYPE": "10-K", "CIK": "0000320193"}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        # Verify document extraction
        assert len(archive.files) == 2
        filenames = archive.get_filenames()
        assert "aapl-20220930.htm" in filenames
        assert "aapl-subsidiaries.htm" in filenames

    def test_filingArchive_document_types(self, datadir: str) -> None:
        """Test that document types are correctly extracted."""
        filings_dir = Path(datadir) / "filings" / "0000320193"
        filings_dir.mkdir(parents=True, exist_ok=True)

        mock_content = create_mock_filing_content()
        cached_file = filings_dir / "0000320193-23-000077.txt.xz"
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_content))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        doc_types = [f.type for f in archive.files]
        assert "10-K" in doc_types
        assert "EX-21" in doc_types

    def test_filingArchive_filter_documents(self, datadir: str) -> None:
        """Test filtering documents during extraction."""
        filings_dir = Path(datadir) / "filings" / "0000320193"
        filings_dir.mkdir(parents=True, exist_ok=True)

        mock_content = create_mock_filing_content()
        cached_file = filings_dir / "0000320193-23-000077.txt.xz"
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_content))

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            # Filter for only 10-K
            archive.get_filingArchive(document_types="10-K", text_only=True)

        # Should only include the 10-K document
        assert len(archive.files) == 1
        assert archive.files[0].type == "10-K"


class TestEndToEndFreshDatadir:
    """End-to-end tests starting from empty datadir."""

    def test_full_pipeline_no_preseeded_caches(self, datadir: str) -> None:
        """Full pipeline from empty datadir with pre-cached filing."""
        # Create idx instance first (with mocked network)
        with patch("secbulkdownload.secmeta.fetch_sec_file"):
            instance = idx(
                datadir=datadir,
                start_year=2024,
                start_quarter=1,
                end_year=2024,
                end_quarter=1,
                user_agent="Test Co test@test.com",
            )

        # Setup directory structure
        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        # Create mock index content AFTER instance creation
        mock_records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-15",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        gz_path = create_mock_master_gz(qtr_dir, mock_records)

        # Pre-create cached filing file
        filings_dir = Path(datadir) / "filings" / "0000320193"
        filings_dir.mkdir(parents=True, exist_ok=True)
        mock_filing_content = create_mock_filing_content()
        cached_file = filings_dir / "0000320193-24-000001.txt.xz"
        with open(cached_file, "wb") as f:
            f.write(lzma.compress(mock_filing_content))

        # Set idxlist to point to our mock file
        instance.idxlist = [str(gz_path)]

        # Filter index
        working_idx = instance.filter_index({"Form Type": ["10-K"]})
        assert len(working_idx) == 1

        # Fetch filings (no headers) - should use cache
        instance.fetch_filings(headers=False)

        # Now create filingArchive
        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {"TYPE": "10-K", "CIK": "0000320193"}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-24-000001.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        # Verify end-to-end worked
        assert len(archive.files) == 2
        assert archive.files[0].type == "10-K"


class TestEmptyDatadirEdgeCases:
    """Test edge cases with empty datadir."""

    def test_idx_empty_filter_returns_empty_with_schema(self, datadir: str) -> None:
        """Empty filter on empty index should return DataFrame with canonical schema."""
        # Create idx instance first (with mocked network)
        with patch("secbulkdownload.secmeta.fetch_sec_file"):
            instance = idx(
                datadir=datadir,
                start_year=2024,
                start_quarter=1,
                end_year=2024,
                end_quarter=1,
                user_agent="Test Co test@test.com",
            )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        # Create empty index file AFTER instance creation
        gz_path = create_mock_master_gz(qtr_dir, [])

        instance.idxlist = [str(gz_path)]
        result = instance.filter_index({})

        # Should be empty list
        assert isinstance(result, list)
        assert len(result) == 0

    def test_filingArchive_handles_missing_file(self, datadir: str) -> None:
        """filingArchive should handle missing cached file gracefully."""
        filings_dir = Path(datadir) / "filings" / "0000320193"
        filings_dir.mkdir(parents=True, exist_ok=True)

        # No cached file created
        with (
            patch("secbulkdownload.secfiling.headerfile") as mock_headerfile,
            patch("secbulkdownload.secfiling.fetch_sec_file") as mock_fetch,
        ):
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {}
            mock_headerfile.return_value = mock_hdr

            mock_content = create_mock_filing_content()
            mock_fetch.return_value = (
                str(filings_dir / "0000320193-23-000077.txt.xz"),
                mock_content,
            )

            archive = filingArchive(
                "edgar/data/0000320193/0000320193-23-000077.txt",
                datadir=datadir,
                user_agent="Test Co test@test.com",
            )
            archive.get_filingArchive(text_only=True)

        # Should have fetched and extracted
        assert len(archive.files) == 2
