"""SEC index and header file metadata handling using DuckDB."""

import datetime
import gzip
import os
import pathlib
import re
from functools import lru_cache
from typing import Any

import duckdb
import xmltodict  # type: ignore[import-untyped]

from secbulkdownload.config import binary_file_types as default_binary_file_types
from secbulkdownload.config import default_datadir, sec_base_url
from secbulkdownload.limiter import rate_limiter
from secbulkdownload.utils import fetch_sec_file, seclimiter, timer

# Valid column names in SEC index files
VALID_INDEX_COLUMNS = {"CIK", "Company Name", "Form Type", "Date Filed", "Filename"}

INDEX_CSV_COLUMNS = {
    "CIK": "VARCHAR",
    "Company Name": "VARCHAR",
    "Form Type": "VARCHAR",
    "Date Filed": "VARCHAR",
    "Filename": "VARCHAR",
}

SEC_INDEX_PREFERRED_ENCODING = "utf-8"
SEC_INDEX_FALLBACK_ENCODING = "latin-1"


@lru_cache(maxsize=512)
def _detect_index_encoding(filepath: str) -> str:
    """Return the first lossless text encoding that can decode a gzipped SEC index file."""
    try:
        with gzip.open(filepath, "rt", encoding=SEC_INDEX_PREFERRED_ENCODING) as handle:
            for _ in handle:
                pass
    except UnicodeDecodeError:
        return SEC_INDEX_FALLBACK_ENCODING

    return SEC_INDEX_PREFERRED_ENCODING


class idx:
    """
    Interface to SEC quarterly index files using DuckDB for data processing.

    Required arguments:
        user_agent: SEC User-Agent string "<Company>, <email>"

    Optional arguments:
        datadir: Directory for cached SEC files (default: ./secdata)
        start_year: Start year for index (default: 1993)
        start_quarter: Start quarter (default: 1)
        end_year: End year (default: 0 = current year)
        end_quarter: End quarter (default: 4)
        rate_limiter: Rate limiter instance
        binary_file_types: Binary file extensions to not compress
    """

    def __init__(
        self,
        datadir: str = default_datadir,
        start_year: int = 1993,
        start_quarter: int = 1,
        end_year: int = 0,
        end_quarter: int = 4,
        rate_limiter: rate_limiter = seclimiter,
        user_agent: str | None = None,
        binary_file_types: list[str] | None = None,
    ) -> None:
        self.user_agent = user_agent
        self.binary_types = binary_file_types or default_binary_file_types
        self.datadir = datadir
        self.idxdir = os.path.join(self.datadir, "idxfiles")
        self.hdrdir = os.path.join(self.datadir, "headerfiles")
        self.filingsdir = os.path.join(self.datadir, "filings")
        self.sec_base = sec_base_url
        self.limiter = rate_limiter

        # Validate quarter and year parameters
        self._validate_quarter_params(start_year, start_quarter, end_year, end_quarter)

        # Resolve end_year = 0 to current year
        if end_year == 0:
            end_year = datetime.date.today().year

        # Clamp end_quarter to current quarter if in the future
        current_qtr = (datetime.date.today().month - 1) // 3 + 1
        if end_year == datetime.date.today().year and end_quarter > current_qtr:
            end_quarter = current_qtr

        self.beg_yr = start_year
        self.beg_qtr = start_quarter
        self.end_yr = end_year
        self.end_qtr = end_quarter

        # Initialize DuckDB connection for this instance
        self._conn = duckdb.connect()

        # Update idxlist during initialization
        self.idxlist: list[str] = self.updateidx()
        self.hdrlist: list[Any] = []
        self.filinglist: list[str] = []
        self.working_idx: list[dict[str, Any]] = []
        self.working_paths: list[tuple[str, str]] = []

    def _validate_quarter_params(
        self,
        start_year: int,
        start_quarter: int,
        end_year: int,
        end_quarter: int,
    ) -> None:
        """Validate quarter and year parameters before any network calls."""
        # Validate quarter numbers
        if not 1 <= start_quarter <= 4:
            raise ValueError(f"Invalid start_quarter: {start_quarter}. Must be between 1 and 4.")
        if not 1 <= end_quarter <= 4:
            raise ValueError(f"Invalid end_quarter: {end_quarter}. Must be between 1 and 4.")

        # Validate years
        if start_year < 0:
            raise ValueError(f"Invalid start_year: {start_year}. Must be non-negative.")
        if end_year < 0 and end_year != 0:  # 0 is special case for current year
            raise ValueError(f"Invalid end_year: {end_year}. Must be non-negative or 0.")

        # Validate range (if end_year != 0, which means "current year")
        resolved_end_year = end_year if end_year != 0 else datetime.date.today().year
        if start_year > resolved_end_year:
            raise ValueError(
                f"Invalid date range: start_year ({start_year}) is after "
                f"end_year ({resolved_end_year})."
            )
        if start_year == resolved_end_year and start_quarter > end_quarter:
            raise ValueError(
                f"Invalid date range: start_quarter ({start_quarter}) is after "
                f"end_quarter ({end_quarter}) in the same year."
            )

    def _idxurls(self) -> list[str]:
        """
        Creates a list of URLs for the SEC quarterly index files included in the date range
        specified by the instance beg_yr, beg_qtr, end_yr and end_qtr variables.
        """
        yq = []
        for y in range(self.beg_yr, self.end_yr + 1):
            bq, eq = 1, 4
            if y == self.beg_yr:
                bq = self.beg_qtr
            if y == self.end_yr:
                eq = self.end_qtr
            for i in range(bq, eq + 1):
                yq.append([y, i])
        return [
            f"https://www.sec.gov/Archives/edgar/full-index/{x[0]}/QTR{x[1]}/master.gz" for x in yq
        ]

    def _expected_cache_paths(self, urls: list[str]) -> list[str]:
        """Convert SEC URLs to expected local cache paths."""
        paths = []
        for url in urls:
            # Extract YYYY/QTRn from URL
            parts = url.split("/")
            year = parts[-3]
            qtr = parts[-2]
            path = os.path.join(self.idxdir, year, qtr, "master.gz")
            paths.append(path)
        return paths

    def _catalog_local_files(self, localdir: str, suffix: str | None = None) -> list[str]:
        """Catalog all locally cached files in a directory."""
        localfiles = []
        for root, _dirs, files in os.walk(localdir, topdown=False):
            for fname in files:
                fpath = os.sep.join([root, fname])
                if suffix:
                    if fname.endswith(suffix):
                        localfiles.append(fpath)
                else:
                    localfiles.append(fpath)
        return localfiles

    def _clear_local_cache(self, localdir: str, suffix: str | None = None) -> None:
        """Deletes all locally cached SEC files."""
        for root, dirs, files in os.walk(localdir, topdown=False):
            for fname in files:
                fpath = os.sep.join([root, fname])
                if suffix:
                    if fname.endswith(suffix):
                        os.remove(fpath)
                else:
                    os.remove(fpath)
            for dname in dirs:
                dpath = os.sep.join([root, dname])
                if not os.listdir(dpath):
                    os.rmdir(dpath)

    @timer
    def updateidx(self, verbose: bool = False) -> list[str]:
        """
        Updates the locally cached quarterly index files for the date range specified.
        By default the most recent quarterly index file included in the local cache is deleted
        and replaced by the most current version available from the SEC website.
        """
        if verbose:
            print("Updating locally cached SEC index files...")

        # Catalog locally saved index files
        idxlocal = self._catalog_local_files(self.idxdir, suffix="gz")

        # Drop the most recently saved idx file since it may be incomplete.
        if len(idxlocal) > 0:
            idxlocal.sort()
            last = idxlocal.pop()
            os.remove(last)
        else:
            if verbose:
                print("You don't seem to have any SEC index files cached in your data directory.")
                print("Downloading these files for the first time may take a few minutes.")

        # Get URLs for the date range
        urls = self._idxurls()
        expected_paths = self._expected_cache_paths(urls)

        # Fetch any missing idxfiles
        idxlist = []
        for url, expected_path in zip(urls, expected_paths, strict=True):
            if expected_path not in idxlocal:
                path = pathlib.Path(expected_path)
                path.parent.mkdir(parents=True, exist_ok=True)
                try:
                    fetch_sec_file(url, expected_path, user_agent=self.user_agent, verbose=verbose)
                except Exception as e:
                    if verbose:
                        print(f"Failed to fetch {url}: {e}")
            idxlist.append(expected_path)

        return idxlist

    def _read_index_file_with_duckdb(self, filepath: str) -> duckdb.DuckDBPyRelation:
        """
        Read a single SEC index file using DuckDB.

        Args:
            filepath: Path to the master.gz file

        Returns:
            DuckDB relation with the index data
        """
        query = self._index_select_sql(filepath, include_not_null_guards=True)
        return self._conn.sql(query)

    def _index_select_sql(self, filepath: str, include_not_null_guards: bool = False) -> str:
        """Build a DuckDB SELECT statement for an SEC master index file."""
        encoding = _detect_index_encoding(filepath)
        not_null_guards = (
            """
            WHERE "CIK" IS NOT NULL
              AND "Company Name" IS NOT NULL
              AND "Form Type" IS NOT NULL
              AND "Date Filed" IS NOT NULL
              AND "Filename" IS NOT NULL
        """
            if include_not_null_guards
            else """
            WHERE "CIK" IS NOT NULL
        """
        )

        return f"""
            SELECT
                LPAD("CIK", 10, '0') as "CIK",
                "Company Name",
                "Form Type",
                strptime("Date Filed", '%Y-%m-%d')::DATE as "Date Filed",
                "Filename"
            FROM read_csv(
                '{filepath}',
                sep='|',
                skip=11,
                compression='gzip',
                encoding='{encoding}',
                header=false,
                auto_detect=false,
                columns={INDEX_CSV_COLUMNS!r}
            )
            {not_null_guards}
        """

    @timer
    def filter_index(
        self, filters: dict[str, list[Any]] | None = None, verbose: bool = False
    ) -> list[dict[str, Any]]:
        """
        Apply filters to the full set of index files for the date range specified.

        Args:
            filters: Dictionary mapping column names to lists of values to filter on
            verbose: Print progress information

        Returns:
            List of dicts with canonical columns (CIK, Company Name, Form Type, Date Filed, Filename)
        """
        # Validate filter keys before processing
        if filters:
            invalid_keys = set(filters.keys()) - VALID_INDEX_COLUMNS
            if invalid_keys:
                raise KeyError(
                    f"Invalid filter key(s): {', '.join(sorted(invalid_keys))}. "
                    f"Valid keys are: {', '.join(sorted(VALID_INDEX_COLUMNS))}"
                )

        if verbose:
            print(f"Filtering {len(self.idxlist)} quarterly index files...")

        # Build SQL for all index files with UNION ALL
        if not self.idxlist:
            # Return empty list
            self.working_idx = []
            return []

        # Read all index files into a DuckDB table
        all_selects = []
        for idxpath in self.idxlist:
            if os.path.exists(idxpath):
                all_selects.append(self._index_select_sql(idxpath))

        if not all_selects:
            self.working_idx = []
            return []

        union_query = " UNION ALL ".join(all_selects)

        # Build WHERE clause from filters
        where_clauses = []
        if filters:
            for key, values in filters.items():
                if values:  # Only add if non-empty
                    # Escape single quotes in values
                    escaped_values = [str(v).replace("'", "''") for v in values]
                    values_str = ", ".join(f"'{v}'" for v in escaped_values)
                    where_clauses.append(f'"{key}" IN ({values_str})')

        if where_clauses:
            final_query = f'SELECT * FROM ({union_query}) WHERE {" AND ".join(where_clauses)} ORDER BY "Date Filed"'
        else:
            final_query = f'SELECT * FROM ({union_query}) ORDER BY "Date Filed"'

        # Execute query and convert to list of dicts
        result = self._conn.execute(final_query).fetchall()
        columns = ["CIK", "Company Name", "Form Type", "Date Filed", "Filename"]
        self.working_idx = [dict(zip(columns, row, strict=True)) for row in result]

        if verbose:
            print(f"Filtered index contains {len(self.working_idx)} records.")

        return self.working_idx

    def show_index_fields(self) -> list[str]:
        """Return index file column names and data types."""
        if not self.idxlist:
            return []

        # Return the canonical column names
        return ["CIK", "Company Name", "Form Type", "Date Filed", "Filename"]

    def index_peek(self, rows: int = 5) -> list[dict[str, Any]]:
        """
        Read the specified number of rows from the index file for the most recent quarter.
        """
        if not self.idxlist:
            return []

        # Get the last (most recent) index file
        latest_idx = self.idxlist[-1]
        if not os.path.exists(latest_idx):
            return []

        # Read the file and get last N rows
        query = (
            self._index_select_sql(latest_idx)
            + f"""
            ORDER BY "Date Filed" DESC
            LIMIT {rows}
        """
        )
        result = self._conn.execute(query).fetchall()
        columns = ["CIK", "Company Name", "Form Type", "Date Filed", "Filename"]
        return [dict(zip(columns, row, strict=True)) for row in result]

    def _convert_sec_filename(self, sec_filename: str) -> tuple[str, str]:
        """
        Convert SEC filepath to header URL and local path.

        Args:
            sec_filename: SEC filepath from index

        Returns:
            Tuple of (URL, local_path)
        """
        _, _, cik, fname = sec_filename.split("/")
        fname_body = fname.split(".")[0]
        new_fname = fname_body + ".hdr.sgml"
        hdrfilepath = os.sep.join([self.hdrdir, cik, new_fname])
        url = self.sec_base + "/".join([cik, "".join(fname_body.split("-")), new_fname])
        return url, hdrfilepath

    def _get_working_paths(self) -> list[tuple[str, str]]:
        """
        Creates a list of SGML headerfile URL and local file path tuples.
        """
        if len(self.working_idx) == 0:
            print("Run filter_index() first...")
            return []
        working_paths = []
        for row in self.working_idx:
            working_paths.append(self._convert_sec_filename(row["Filename"]))
        return working_paths

    def fetch_headers(self, verbose: bool = False) -> None:
        """
        Fetch any SGML header files for the current working index that are not already cached.
        """
        if len(self.working_idx) == 0:
            print("Run filter_index() before fetch_headers()...")
            return

        # Make sure we have relevant paths for files in self.working_idx
        if len(self.working_paths) == 0:
            self.working_paths = self._get_working_paths()

        # Determine whether additional header files need to be fetched
        fetchnum = 0
        self.hdrlist = []
        for row in self.working_idx:
            file = row["Filename"]
            hdr = headerfile(file, datadir=self.datadir, user_agent=self.user_agent)
            if not hdr.islocal:
                fetchnum += 1
            self.hdrlist.append(hdr)

        if fetchnum != 0:
            if verbose:
                print(f"Fetching {fetchnum} header files...")
            for item in self.hdrlist:
                if not item.islocal:
                    item.fetch_file(verbose=verbose)
            if verbose:
                print("Finished fetching header files.")
        else:
            if verbose:
                print("All header files are present in the local cache.")

    def fetch_filings(self, headers: bool = False, verbose: bool = False) -> None:
        """
        Fetch any filing archive files included in the current working index.
        """
        if len(self.working_idx) == 0:
            print("Run filter_index() before fetch_filings()...")
            return

        # Build list of filing paths
        filing_paths = []
        for row in self.working_idx:
            sec_filepath = row["Filename"]
            filing_url = self.sec_base + sec_filepath
            _, _, cik, fname = sec_filepath.split("/")
            localfilepath = os.sep.join([self.filingsdir, cik, fname])
            filing_paths.append((filing_url, f"{localfilepath}.xz"))

        # Check which need to be fetched
        fetchnum = 0
        fetchlist = []
        self.filinglist = []

        for item in filing_paths:
            if not os.path.exists(item[1]):
                fetchnum += 1
                fetchlist.append(item)
            else:
                self.filinglist.append(item[1])

        if fetchnum != 0:
            if verbose:
                print(f"Fetching {fetchnum} filing archives...")
            for item in fetchlist:
                fetch_sec_file(item[0], item[1], user_agent=self.user_agent, verbose=verbose)
            if verbose:
                print("Finished fetching filing archives.")
        else:
            if verbose:
                print("All filing archives are present in the local cache.")

        if headers:
            self.fetch_headers(verbose=verbose)

    def clear_index_cache(self) -> None:
        """Delete locally cached SEC index files."""
        self._clear_local_cache(self.idxdir)
        self.idxlist = []

    def clear_header_cache(self) -> None:
        """Delete locally cached SGML header files."""
        self._clear_local_cache(self.hdrdir)
        for item in self.hdrlist:
            item.islocal = False

    def clear_filing_cache(self) -> None:
        """Delete locally cached filing archive files."""
        self._clear_local_cache(self.filingsdir)
        self.filinglist = []


class headerfile:
    """
    Interface to SEC SGML header files for filing metadata.

    Required arguments:
        sec_filename: SEC filepath from index record
        user_agent: SEC User-Agent string

    Optional arguments:
        datadir: Directory for cached files
        rate_limiter: Rate limiter instance
    """

    def __init__(
        self,
        sec_filename: str,
        datadir: str = default_datadir,
        rate_limiter: rate_limiter = seclimiter,
        user_agent: str | None = None,
    ) -> None:
        self.user_agent = user_agent
        self.limiter = rate_limiter
        self.datadir = datadir
        self.hdrdir = os.path.join(self.datadir, "headerfiles")
        self.sec_base = "http://www.sec.gov/Archives/edgar/data/"
        self.url, self.localpath = self._convert_sec_filename(sec_filename)
        self.islocal = os.path.exists(self.localpath)

    def _convert_sec_filename(self, sec_filename: str) -> tuple[str, str]:
        """
        Convert SEC filepath to header URL and local path.
        """
        _, _, cik, fname = sec_filename.split("/")
        fname_body = fname.split(".")[0]
        new_fname = fname_body + ".hdr.sgml"
        hdrfilepath = os.sep.join([self.hdrdir, cik, new_fname])
        url = self.sec_base + "/".join([cik, "".join(fname_body.split("-")), new_fname])
        return url, hdrfilepath

    def _SGMLfiletoDict(self) -> dict[str, Any]:
        """
        Reads an SGML header file and converts it to a Python dictionary.
        """
        end_tags = []
        xmlstring = ""
        tag = re.compile(r"<.[^(><.)]+>")
        with open(self.localpath, encoding="latin-1") as f:
            xmlstring += '<?xml version="1.0"?>' + "\n"
            lines = f.read().splitlines()
            # Identify containers
            for line in lines:
                if line.startswith("</"):
                    end_tags.append(line)
            container_tags = [s.replace("/", "") for s in end_tags] + end_tags
            # Add closing tags for non-container fields
            for line in lines:
                a = re.findall(tag, line)
                if len(a) > 0:
                    if a[0] in container_tags:
                        xmlstring += line + "\n"
                    else:
                        xmlstring += line.replace("&", "and") + a[0][:1] + "/" + a[0][1:] + "\n"
        return xmltodict.parse(xmlstring)["SEC-HEADER"]  # type: ignore[no-any-return]

    def get_headerDict(self, verbose: bool = False) -> dict[str, Any]:
        """
        Returns the contents of the SGML header file in dictionary form.
        """
        if not self.islocal:
            self.fetch_file(verbose=verbose)
        return self._SGMLfiletoDict()

    def fetch_file(self, content: bool = False, verbose: bool = False) -> None:
        """Download the SGML header file from SEC."""
        fetch_sec_file(
            self.url,
            self.localpath,
            content=content,
            limiter=self.limiter,
            user_agent=self.user_agent,
            verbose=verbose,
        )
