"""Comprehensive cache recovery tests for SEC bulk download.

Tests verify that missing cache files are automatically restored when
re-running the index/filing flow. Uses temp directories with mocked HTTP.

Test scenarios:
- VAL-RECOVERY-001: Single index recovery
- VAL-RECOVERY-002: Warm-cache rerun
- VAL-RECOVERY-003: Multi-quarter index recovery
- VAL-RECOVERY-004: Single filing recovery
- VAL-RECOVERY-005: Multi-filing recovery
- VAL-RECOVERY-006: Mixed recovery
- VAL-RECOVERY-007: Index parent-dir recovery
- VAL-RECOVERY-008: Filing parent-dir recovery
- VAL-CROSS-003: End-to-end after recovery
- Control artifact preservation
"""

import gzip
import hashlib
import lzma
import os
import shutil
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, patch

from secbulkdownload import idx
from secbulkdownload.secfiling import filingArchive

# ============================================================================
# Helper functions
# ============================================================================


def compute_file_hash(filepath: str) -> str:
    """Compute SHA256 hash of a file."""
    sha256 = hashlib.sha256()
    with open(filepath, "rb") as f:
        for chunk in iter(lambda: f.read(8192), b""):
            sha256.update(chunk)
    return sha256.hexdigest()


def compute_content_hash(filepath: str) -> str:
    """Compute SHA256 hash of decompressed file content (handles gzip and lzma)."""
    sha256 = hashlib.sha256()

    if filepath.endswith(".gz"):
        with gzip.open(filepath, "rb") as f:
            data = f.read()
    elif filepath.endswith(".xz"):
        with open(filepath, "rb") as f:
            data = lzma.decompress(f.read())
    else:
        with open(filepath, "rb") as f:
            data = f.read()

    sha256.update(data)
    return sha256.hexdigest()


def compute_data_hash(data: bytes) -> str:
    """Compute SHA256 hash of data."""
    return hashlib.sha256(data).hexdigest()


def create_mock_index_content(year: int, qtr: int) -> str:
    """Create consistent mock index content for a given year/quarter."""
    header_lines = [
        "Description:           Master Index of EDGAR Dissemination Feed",
        "Last Updated:          2024-01-01",
        "Comments:             comments@sec.gov",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/",
        "Cloud HTTP:           https://www.sec.gov/Archives/",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/full-index/",
        "SFTP:                 sftp://ftp.sec.gov/edgar/full-index/",
        "",
        "",
    ]
    column_header = "CIK|Company Name|Form Type|Date Filed|Filename"
    separator_line = "-" * 80

    # Deterministic records based on year/quarter
    records = [
        f"0000320193|APPLE INC|10-K|{year}-{qtr * 3:02d}-01|edgar/data/0000320193/0000320193-{year % 100:02d}-{qtr:02d}0001.txt",
        f"0000789019|MICROSOFT CORP|10-Q|{year}-{qtr * 3:02d}-15|edgar/data/0000789019/0000789019-{year % 100:02d}-{qtr:02d}0002.txt",
        f"0001067983|BERKSHIRE HATHAWAY INC|13F-HR|{year}-{qtr * 3:02d}-20|edgar/data/0001067983/0001067983-{year % 100:02d}-{qtr:02d}0003.txt",
    ]

    return (
        "\n".join(header_lines)
        + "\n"
        + column_header
        + "\n"
        + separator_line
        + "\n"
        + "\n".join(records)
        + "\n"
    )


def create_mock_filing_content(cik: str, accession: str) -> bytes:
    """Create consistent mock filing content for a given CIK/accession."""
    content = f"""<SEC-DOCUMENT>{accession}.txt : 20240101
<SEC-HEADER>{accession}.hdr.sgml : 20240101
<ACCEPTANCE-DATETIME>20240101120000
ACCESSION NUMBER:		{accession}
</SEC-HEADER>
<DOCUMENT>
<TYPE>10-K
<SEQUENCE>1
<FILENAME>report.htm
<DESCRIPTION>Annual Report
<TEXT>
<html><body><h1>Company {cik} Annual Report</h1><p>Accession: {accession}</p></body></html>
</TEXT>
</DOCUMENT>
<DOCUMENT>
<TYPE>EX-21
<SEQUENCE>2
<FILENAME>exhibit21.htm
<DESCRIPTION>Subsidiaries
<TEXT>
<html><body><p>List of Subsidiaries for {cik}</p></body></html>
</TEXT>
</DOCUMENT>
</SEC-DOCUMENT>"""
    return content.encode("latin-1")


def create_mock_master_gz(dir_path: Path, year: int, qtr: int) -> str:
    """Create a mock master.gz SEC index file with deterministic content.

    Uses gzip.compress() for reproducible output without timestamps.
    """
    content = create_mock_index_content(year, qtr)
    gz_path = dir_path / "master.gz"

    # Use gzip.compress() for deterministic output (no timestamp)
    compressed = gzip.compress(content.encode("latin-1"), mtime=0)
    with open(gz_path, "wb") as f:
        f.write(compressed)
    return str(gz_path)


def setup_mock_idx_environment(datadir: str, quarters: list[tuple[int, int]]) -> dict[str, Any]:
    """Set up mock index files with consistent, deterministic content."""
    idxdir = Path(datadir) / "idxfiles"
    idxdir.mkdir(parents=True, exist_ok=True)

    idxlist = []
    index_hashes = {}

    for year, qtr in quarters:
        qtr_dir = idxdir / str(year) / f"QTR{qtr}"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        idx_path = create_mock_master_gz(qtr_dir, year, qtr)
        idxlist.append(idx_path)
        index_hashes[idx_path] = compute_file_hash(idx_path)

    return {
        "idxlist": idxlist,
        "idxdir": str(idxdir),
        "hashes": index_hashes,
    }


def setup_mock_filings_environment(datadir: str, filings: list[dict[str, str]]) -> dict[str, Any]:
    """Set up mock filing archives with consistent content."""
    filingsdir = Path(datadir) / "filings"
    filingsdir.mkdir(parents=True, exist_ok=True)

    filing_paths = []
    filing_hashes = {}

    for filing in filings:
        cik = filing["cik"]
        accession = filing["accession"]
        content = create_mock_filing_content(cik, accession)

        cik_dir = filingsdir / cik
        cik_dir.mkdir(parents=True, exist_ok=True)

        filepath = cik_dir / f"{accession}.txt.xz"
        compressed = lzma.compress(content)
        with open(filepath, "wb") as f:
            f.write(compressed)

        filing_paths.append(str(filepath))
        filing_hashes[str(filepath)] = compute_data_hash(compressed)

    return {
        "filing_paths": filing_paths,
        "filingsdir": str(filingsdir),
        "hashes": filing_hashes,
    }


def create_idx_with_mocks(
    datadir: str,
    start_year: int = 2024,
    start_quarter: int = 1,
    end_year: int = 2024,
    end_quarter: int = 1,
    user_agent: str = "Test Co test@test.com",
) -> tuple[idx, Any]:
    """Create idx instance with mocked fetch_sec_file and return context manager.

    Returns tuple of (idx_instance, mock_context) where mock_context must be
    used with 'with' statement to keep mocks active for subsequent calls.
    """
    from contextlib import contextmanager

    @contextmanager
    def mock_context() -> Any:
        def mock_fetch(
            url: str, localpath: str, *args: Any, **kwargs: Any
        ) -> str | tuple[str, bytes]:
            """Mock fetch that writes consistent content to localpath."""
            path = Path(localpath)
            path.parent.mkdir(parents=True, exist_ok=True)

            if url.endswith("master.gz"):
                # Extract year/quarter from URL
                parts = url.split("/")
                year = int(parts[-3])
                qtr = int(parts[-2].replace("QTR", ""))

                content_str = create_mock_index_content(year, qtr)
                # Use gzip.compress() for deterministic output (no timestamp)
                compressed = gzip.compress(content_str.encode("latin-1"), mtime=0)
                with open(path, "wb") as f:
                    f.write(compressed)

                if kwargs.get("content"):
                    return localpath, content_str.encode("latin-1")
                return localpath
            else:
                # Filing URL: extract CIK and accession from URL or localpath
                # URL format: .../data/CIK/ACCESSION.txt
                url_parts = url.split("/")
                cik = url_parts[-2]
                accession = url_parts[-1].replace(".txt", "")

                content_bytes = create_mock_filing_content(cik, accession)
                compressed = lzma.compress(content_bytes)
                with open(path, "wb") as f:
                    f.write(compressed)

                if kwargs.get("content"):
                    return localpath, content_bytes
                return localpath

        with (
            patch("secbulkdownload.secmeta.fetch_sec_file", side_effect=mock_fetch),
            patch("secbulkdownload.utils.fetch_sec_file", side_effect=mock_fetch),
        ):
            yield

    # Create instance within mock context
    with mock_context():
        instance = idx(
            datadir=datadir,
            start_year=start_year,
            start_quarter=start_quarter,
            end_year=end_year,
            end_quarter=end_quarter,
            user_agent=user_agent,
        )

    return instance, mock_context


# ============================================================================
# Test Classes
# ============================================================================


class TestSingleIndexRecovery:
    """VAL-RECOVERY-001: Single index recovery."""

    def test_single_master_gz_recovery(self, datadir: str) -> None:
        """Delete one master.gz, re-run, verify recreated with matching content."""
        # Set up environment with 2 quarters
        setup_mock_idx_environment(datadir, [(2024, 1), (2024, 2)])

        # Create idx instance with mocked network (returns context too)
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=2,
        )

        idxdir = Path(datadir) / "idxfiles"
        qtr1_path = idxdir / "2024" / "QTR1" / "master.gz"
        qtr2_path = idxdir / "2024" / "QTR2" / "master.gz"

        # Capture baseline CONTENT hash (decompressed) after idx initialization
        baseline_content_hash_qtr1 = compute_content_hash(str(qtr1_path))

        # Delete one master.gz (QTR1)
        os.remove(qtr1_path)
        assert not qtr1_path.exists()

        # Re-run updateidx (simulated recovery) WITH mock context active
        with mock_ctx():
            idx_instance.updateidx()

        # Verify QTR1 recreated with matching CONTENT
        assert qtr1_path.exists()
        recovery_content_hash_qtr1 = compute_content_hash(str(qtr1_path))
        assert recovery_content_hash_qtr1 == baseline_content_hash_qtr1

        # QTR2 should still exist (though may be refreshed as "most recent")
        assert qtr2_path.exists()


class TestWarmCacheRerun:
    """VAL-RECOVERY-002: Warm-cache rerun."""

    def test_warm_cache_rerun_preserves_files(self, datadir: str) -> None:
        """Run flow twice without deletions, verify files still present."""
        setup_mock_idx_environment(datadir, [(2024, 1), (2024, 2)])

        idxdir = Path(datadir) / "idxfiles"
        qtr1_path = idxdir / "2024" / "QTR1" / "master.gz"
        qtr2_path = idxdir / "2024" / "QTR2" / "master.gz"

        # First run
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=2,
        )

        # Capture content hash of QTR1 (not most recent, so content should be stable)
        first_run_content_hash_qtr1 = compute_content_hash(str(qtr1_path))

        # Second run (warm cache) WITH mock context active
        with mock_ctx():
            idx_instance.updateidx()

        # QTR1 content should remain unchanged (not the most recent quarter)
        second_run_content_hash_qtr1 = compute_content_hash(str(qtr1_path))
        assert second_run_content_hash_qtr1 == first_run_content_hash_qtr1

        # Both files should exist
        assert qtr1_path.exists()
        assert qtr2_path.exists()


class TestMultiQuarterIndexRecovery:
    """VAL-RECOVERY-003: Multi-quarter index recovery."""

    def test_multiple_master_gz_recovery(self, datadir: str) -> None:
        """Delete multiple master.gz files, re-run, verify all recreated."""
        setup_mock_idx_environment(datadir, [(2024, 1), (2024, 2), (2024, 3), (2024, 4)])

        idxdir = Path(datadir) / "idxfiles"

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=4,
        )

        # Capture baseline CONTENT hashes for quarters that won't be "most recent"
        baseline_hashes = {}
        for qtr in [1, 2, 3]:
            path = idxdir / "2024" / f"QTR{qtr}" / "master.gz"
            baseline_hashes[qtr] = compute_content_hash(str(path))

        # Delete QTR1, QTR2, QTR3 (keep QTR4 as most recent)
        deleted_quarters = [1, 2, 3]
        for qtr in deleted_quarters:
            path = idxdir / "2024" / f"QTR{qtr}" / "master.gz"
            os.remove(path)
            assert not path.exists()

        # Re-run updateidx WITH mock context
        with mock_ctx():
            idx_instance.updateidx()

        # Verify all deleted quarters recreated with matching CONTENT hashes
        for qtr in deleted_quarters:
            path = idxdir / "2024" / f"QTR{qtr}" / "master.gz"
            assert path.exists(), f"QTR{qtr} master.gz not recreated"
            recovery_hash = compute_content_hash(str(path))
            assert recovery_hash == baseline_hashes[qtr], f"QTR{qtr} content hash mismatch"


class TestSingleFilingRecovery:
    """VAL-RECOVERY-004: Single filing recovery."""

    def test_single_filing_txt_xz_recovery(self, datadir: str) -> None:
        """Delete one .txt.xz, re-run, verify recreated with matching content."""
        # Set up index environment
        setup_mock_idx_environment(datadir, [(2024, 1)])

        # Set up filing with CIK/accession matching the mock index content
        # The mock index has: edgar/data/0000320193/0000320193-24-010001.txt
        filings_info = [{"cik": "0000320193", "accession": "0000320193-24-010001"}]
        filing_setup = setup_mock_filings_environment(datadir, filings_info)

        filing_path = Path(filing_setup["filing_paths"][0])
        baseline_content_hash = compute_content_hash(str(filing_path))

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Filter and fetch filings WITH mock context
        with mock_ctx():
            idx_instance.filter_index({"CIK": ["0000320193"]})
            idx_instance.fetch_filings()

        # Verify filing still exists after fetch (wasn't re-fetched)
        assert filing_path.exists()

        # Delete the filing
        os.remove(filing_path)
        assert not filing_path.exists()

        # Verify no raw .txt sibling exists
        raw_txt = str(filing_path).replace(".txt.xz", ".txt")
        assert not os.path.exists(raw_txt), "Raw .txt should not exist"

        # Re-run fetch_filings WITH mock context
        with mock_ctx():
            idx_instance.fetch_filings()

        # Verify filing recreated with matching content
        assert filing_path.exists()
        recovery_content_hash = compute_content_hash(str(filing_path))
        assert recovery_content_hash == baseline_content_hash


class TestMultiFilingRecovery:
    """VAL-RECOVERY-005: Multi-filing recovery."""

    def test_multiple_filings_recovery(self, datadir: str) -> None:
        """Delete multiple .txt.xz files, re-run, verify all recreated."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        # Set up multiple filings with CIKs/accessions matching mock index content
        # Mock index filenames use format: {cik}-{year%02d}-{qtr:02d}000{seq}.txt
        # For year=2024, qtr=1: 0000320193-24-010001.txt, 0000789019-24-010002.txt, etc.
        filings_info = [
            {"cik": "0000320193", "accession": "0000320193-24-010001"},
            {"cik": "0000789019", "accession": "0000789019-24-010002"},
            {"cik": "0001067983", "accession": "0001067983-24-010003"},
        ]
        filing_setup = setup_mock_filings_environment(datadir, filings_info)

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Fetch all filings WITH mock context
        with mock_ctx():
            idx_instance.filter_index({})
            idx_instance.fetch_filings()

        # Capture baseline CONTENT hashes
        baseline_hashes = {}
        for path_str in filing_setup["filing_paths"]:
            baseline_hashes[path_str] = compute_content_hash(path_str)

        # Delete first two filings
        deleted_paths = filing_setup["filing_paths"][:2]
        for path_str in deleted_paths:
            os.remove(path_str)
            assert not Path(path_str).exists()

        # Re-run fetch_filings WITH mock context
        with mock_ctx():
            idx_instance.fetch_filings()

        # Verify all deleted filings recreated with matching content
        for path_str in deleted_paths:
            assert Path(path_str).exists(), f"Filing not recreated: {path_str}"
            recovery_hash = compute_content_hash(path_str)
            assert recovery_hash == baseline_hashes[path_str], f"Content hash mismatch: {path_str}"


class TestMixedRecovery:
    """VAL-RECOVERY-006: Mixed recovery."""

    def test_mixed_index_and_filing_recovery(self, datadir: str) -> None:
        """Delete both index and filing files, single re-run restores all."""
        setup_mock_idx_environment(datadir, [(2024, 1), (2024, 2)])

        filings_info = [{"cik": "0000320193", "accession": "0000320193-24-010001"}]
        filing_setup = setup_mock_filings_environment(datadir, filings_info)

        idxdir = Path(datadir) / "idxfiles"
        idx_path = idxdir / "2024" / "QTR1" / "master.gz"
        filing_path = Path(filing_setup["filing_paths"][0])

        # Capture baseline CONTENT hashes
        baseline_idx_hash = compute_content_hash(str(idx_path))
        baseline_filing_hash = compute_content_hash(str(filing_path))

        # Create idx instance and complete initial flow WITH mock context
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=2,
        )

        with mock_ctx():
            idx_instance.filter_index({"CIK": ["0000320193"]})
            idx_instance.fetch_filings()

        # Delete both index and filing
        os.remove(idx_path)
        os.remove(filing_path)
        assert not idx_path.exists()
        assert not filing_path.exists()

        # Single re-run (updateidx + fetch_filings) WITH mock context
        with mock_ctx():
            idx_instance.updateidx()
            idx_instance.filter_index({"CIK": ["0000320193"]})
            idx_instance.fetch_filings()

        # Verify both restored
        assert idx_path.exists(), "Index not recreated"
        assert filing_path.exists(), "Filing not recreated"

        # Verify content hashes match
        assert compute_content_hash(str(idx_path)) == baseline_idx_hash
        assert compute_content_hash(str(filing_path)) == baseline_filing_hash


class TestIndexParentDirRecovery:
    """VAL-RECOVERY-007: Index parent-dir recovery."""

    def test_index_parent_directory_recovery(self, datadir: str) -> None:
        """Delete entire idxfiles/YYYY/QTRn/ directory, re-run, verify recreated."""
        setup_mock_idx_environment(datadir, [(2024, 1), (2024, 2)])

        idxdir = Path(datadir) / "idxfiles"
        qtr1_dir = idxdir / "2024" / "QTR1"
        qtr1_path = qtr1_dir / "master.gz"

        # Capture baseline CONTENT hash
        baseline_hash = compute_content_hash(str(qtr1_path))

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=2,
        )

        # Delete entire QTR1 directory
        shutil.rmtree(qtr1_dir)
        assert not qtr1_dir.exists()

        # Re-run updateidx WITH mock context
        with mock_ctx():
            idx_instance.updateidx()

        # Verify directory and file recreated
        assert qtr1_dir.exists(), "QTR1 directory not recreated"
        assert qtr1_path.exists(), "master.gz not recreated"
        assert compute_content_hash(str(qtr1_path)) == baseline_hash


class TestFilingParentDirRecovery:
    """VAL-RECOVERY-008: Filing parent-dir recovery."""

    def test_filing_parent_directory_recovery(self, datadir: str) -> None:
        """Delete entire filings/<CIK>/ directory, re-run, verify recreated."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        filings_info = [{"cik": "0000320193", "accession": "0000320193-24-010001"}]
        filing_setup = setup_mock_filings_environment(datadir, filings_info)

        filingsdir = Path(datadir) / "filings"
        cik_dir = filingsdir / "0000320193"
        filing_path = Path(filing_setup["filing_paths"][0])

        # Capture baseline CONTENT hash
        baseline_hash = compute_content_hash(str(filing_path))

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Filter and fetch filings WITH mock context
        with mock_ctx():
            idx_instance.filter_index({"CIK": ["0000320193"]})
            idx_instance.fetch_filings()

        # Delete entire CIK directory
        shutil.rmtree(cik_dir)
        assert not cik_dir.exists()

        # Re-run fetch_filings WITH mock context
        with mock_ctx():
            idx_instance.fetch_filings()

        # Verify directory and files recreated
        assert cik_dir.exists(), "CIK directory not recreated"
        assert filing_path.exists(), "Filing not recreated"
        assert compute_content_hash(str(filing_path)) == baseline_hash


class TestEndToEndAfterRecovery:
    """VAL-CROSS-003: End-to-end after recovery."""

    def test_end_to_end_after_recovery(self, datadir: str) -> None:
        """After recovery, run full flow, verify documents still readable."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        filings_info = [{"cik": "0000320193", "accession": "0000320193-24-010001"}]
        filing_setup = setup_mock_filings_environment(datadir, filings_info)

        filing_path = Path(filing_setup["filing_paths"][0])

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Complete initial flow WITH mock context
        with mock_ctx():
            idx_instance.filter_index({"CIK": ["0000320193"]})
            idx_instance.fetch_filings()

        # Delete filing
        os.remove(filing_path)
        assert not filing_path.exists()

        # Recovery WITH mock context
        with mock_ctx():
            idx_instance.fetch_filings()

        # Verify filing exists again
        assert filing_path.exists()

        # End-to-end: Read recovered filing with filingArchive
        sec_filepath = "edgar/data/0000320193/0000320193-24-010001.txt"

        with patch("secbulkdownload.secfiling.headerfile") as mock_headerfile:
            mock_hdr = MagicMock()
            mock_hdr.get_headerDict.return_value = {"TYPE": "10-K"}
            mock_headerfile.return_value = mock_hdr

            archive = filingArchive(
                sec_filepath, datadir=datadir, user_agent="Test Co test@test.com"
            )
            archive.get_filingArchive()

        # Verify documents readable
        assert len(archive.files) >= 1, "No documents extracted"
        assert archive.files[0].filename == "report.htm"
        assert "Company 0000320193 Annual Report" in archive.files[0].body


class TestControlArtifactPreservation:
    """Control artifact preservation tests."""

    def test_ds_store_preserved_during_recovery(self, datadir: str) -> None:
        """Place .DS_Store or control files, verify they survive recovery."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        idxdir = Path(datadir) / "idxfiles"

        # Place control file in index directory
        qtr1_dir = idxdir / "2024" / "QTR1"
        control_file = qtr1_dir / ".DS_Store"
        with open(control_file, "wb") as f:
            f.write(b"DS_Store marker content")

        control_hash = compute_file_hash(str(control_file))

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Re-run updateidx WITH mock context
        with mock_ctx():
            idx_instance.updateidx()

        # Verify control file still exists
        assert control_file.exists(), "Control file was deleted"
        assert compute_file_hash(str(control_file)) == control_hash, "Control file was modified"

    def test_control_file_in_filings_dir(self, datadir: str) -> None:
        """Control files in filings directory should survive recovery."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        filings_info = [{"cik": "0000320193", "accession": "0000320193-24-000001"}]
        filing_setup = setup_mock_filings_environment(datadir, filings_info)

        # Place control file in filings CIK directory
        filingsdir = Path(datadir) / "filings"
        cik_dir = filingsdir / "0000320193"
        control_file = cik_dir / ".gitkeep"
        with open(control_file, "w") as f:
            f.write("preserve this file")

        control_hash = compute_file_hash(str(control_file))

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Delete and recover filing WITH mock context
        filing_path = Path(filing_setup["filing_paths"][0])
        os.remove(filing_path)

        with mock_ctx():
            idx_instance.filter_index({"CIK": ["0000320193"]})
            idx_instance.fetch_filings()

        # Verify control file still exists
        assert control_file.exists(), "Control file was deleted"
        assert compute_file_hash(str(control_file)) == control_hash, "Control file was modified"

    def test_custom_metadata_file_preserved(self, datadir: str) -> None:
        """Custom metadata files should survive recovery."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        # Create custom metadata file in data root
        metadata_file = Path(datadir) / "fetch_log.txt"
        with open(metadata_file, "w") as f:
            f.write("Custom fetch log entry\n")

        metadata_hash = compute_file_hash(str(metadata_file))

        # Create idx instance and run update WITH mock context
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        with mock_ctx():
            idx_instance.updateidx()

        # Verify metadata file still exists
        assert metadata_file.exists(), "Metadata file was deleted"
        assert compute_file_hash(str(metadata_file)) == metadata_hash, "Metadata file was modified"


class TestRecoveryEdgeCases:
    """Additional edge case tests for recovery scenarios."""

    def test_empty_cache_initial_fetch(self, datadir: str) -> None:
        """Starting with empty cache should create all required files."""
        # No setup - start with empty datadir

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Verify index files created
        idxdir = Path(datadir) / "idxfiles"
        qtr1_path = idxdir / "2024" / "QTR1" / "master.gz"
        assert qtr1_path.exists(), "Index file not created on empty cache"

    def test_partial_filing_recovery(self, datadir: str) -> None:
        """Recovery of filing when only compressed version exists."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        filings_info = [{"cik": "0000320193", "accession": "0000320193-24-010001"}]
        filing_setup = setup_mock_filings_environment(datadir, filings_info)

        filing_path = Path(filing_setup["filing_paths"][0])

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Delete filing
        os.remove(filing_path)

        # Verify .txt.xz is what gets recovered (not raw .txt) WITH mock context
        with mock_ctx():
            idx_instance.filter_index({"CIK": ["0000320193"]})
            idx_instance.fetch_filings()

        # Only .txt.xz should exist
        assert filing_path.exists(), ".txt.xz not recreated"
        raw_txt = str(filing_path).replace(".txt.xz", ".txt")
        assert not os.path.exists(raw_txt), "Raw .txt should not exist"

    def test_multiple_recovery_cycles(self, datadir: str) -> None:
        """Multiple delete-recover cycles should maintain content consistency."""
        setup_mock_idx_environment(datadir, [(2024, 1)])

        idxdir = Path(datadir) / "idxfiles"
        idx_path = idxdir / "2024" / "QTR1" / "master.gz"

        # Capture original CONTENT hash
        original_hash = compute_content_hash(str(idx_path))

        # Create idx instance
        idx_instance, mock_ctx = create_idx_with_mocks(
            datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
        )

        # Multiple recovery cycles WITH mock context
        for cycle in range(3):
            # Delete
            os.remove(idx_path)
            assert not idx_path.exists(), f"Cycle {cycle}: file not deleted"

            # Recover WITH mock context
            with mock_ctx():
                idx_instance.updateidx()
            assert idx_path.exists(), f"Cycle {cycle}: file not recovered"

            # Verify content hash matches
            cycle_hash = compute_content_hash(str(idx_path))
            assert cycle_hash == original_hash, f"Cycle {cycle}: content hash mismatch"
