"""Parquet-based tick storage with ZSTD-3 compression.

This module replaces ClickHouse Tier 1 (raw trades cache) with local
Parquet files for better portability and no server requirement.

Issue #277: Migrated from monthly (YYYY-MM.parquet) to daily (YYYY-MM-DD.parquet)
partitioning. Key wins:
  - ``has_ticks()`` for a single day is O(1) via ``Path.exists()``
  - ``write_ticks()`` is simpler — no merge/append to monthly files
  - Each daily file is an atomic unit (write to .tmp then rename)

Compression choice (based on empirical benchmark 2026-01-07):
- ZSTD-3: 6.50 MB for 761K trades (5.37x compression)
- Write: 0.019s, Read: 0.006s
- Beats Brotli on BOTH size AND speed for tick data
"""

from __future__ import annotations

import logging
import shutil
from collections.abc import Iterator
from datetime import date
from pathlib import Path
from typing import TYPE_CHECKING

import polars as pl

from .parquet_io import (
    COMPRESSION,
    COMPRESSION_LEVEL,
    _atomic_write_parquet,
    _is_valid_parquet,  # noqa: F401 — re-exported for backward compatibility
    _validate_and_recover_parquet,
    get_cache_dir,
)
from .parquet_query import (
    _get_parquet_path,
    _get_symbol_dir,
    _iter_days,
    _timestamp_to_day,
    _timestamp_to_year_month,
)
from .parquet_query import fetch_date_range as _fetch_date_range
from .parquet_query import has_ticks as _has_ticks
from .parquet_query import read_ticks_streaming as _read_ticks_streaming

if TYPE_CHECKING:
    import pandas as pd

    from opendeviationbar.config.backfill import BackfillConfig

logger = logging.getLogger(__name__)

# Microsecond/millisecond detection threshold.
# ms values for modern dates are ~1e12; µs values are ~1e15.
# Threshold 1e13 safely separates the two for any date 2001+.
_US_MS_THRESHOLD = 10_000_000_000_000  # 1e13
_DAY_KEY_LEN = 10   # "YYYY-MM-DD"
_MONTH_KEY_LEN = 7  # "YYYY-MM"
_DAY_KEY_MONTH_END = 7  # Index to extract "YYYY-MM" from "YYYY-MM-DD"


def _backfill_cfg() -> BackfillConfig:
    from opendeviationbar.config.backfill import BackfillConfig
    return BackfillConfig()


class TickStorage:
    """Parquet-based tick data storage with ZSTD-3 compression.

    Stores raw tick data in Parquet files partitioned by symbol and day.
    Uses polars for fast I/O with ZSTD-3 compression.

    Parameters
    ----------
    cache_dir : Path | str | None
        Custom cache directory. If None, uses platformdirs default.

    Examples
    --------
    >>> storage = TickStorage()
    >>> storage.write_ticks("BTCUSDT", trades_df)
    >>> df = storage.read_ticks("BTCUSDT", start_ts, end_ts)

    Directory Structure
    -------------------
    ~/.cache/opendeviationbar/ticks/
    ├── BTCUSDT/
    │   ├── 2024-01-01.parquet
    │   ├── 2024-01-02.parquet
    │   └── ...
    └── EURUSD/
        └── ...
    """

    def __init__(self, cache_dir: Path | str | None = None) -> None:
        """Initialize tick storage.

        Parameters
        ----------
        cache_dir : Path | str | None
            Custom cache directory. If None, uses platformdirs default.
        """
        if cache_dir is None:
            self._cache_dir = get_cache_dir()
        else:
            self._cache_dir = Path(cache_dir)

        self._ticks_dir = self._cache_dir / "ticks"

    @property
    def cache_dir(self) -> Path:
        """Get the cache directory path."""
        return self._cache_dir

    @property
    def ticks_dir(self) -> Path:
        """Get the ticks storage directory path."""
        return self._ticks_dir

    def _get_symbol_dir(self, symbol: str) -> Path:
        """Get directory for a symbol's tick files."""
        return _get_symbol_dir(self._ticks_dir, symbol)

    def _get_parquet_path(self, symbol: str, day_key: str) -> Path:
        """Get path for a specific day's parquet file.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        day_key : str
            Day string (e.g., "2024-01-15")

        Returns
        -------
        Path
            Path to the parquet file
        """
        return _get_parquet_path(self._ticks_dir, symbol, day_key)

    def _timestamp_to_day(self, timestamp_ms: int) -> str:
        """Convert millisecond timestamp to day string (YYYY-MM-DD)."""
        return _timestamp_to_day(timestamp_ms)

    def _timestamp_to_year_month(self, timestamp_ms: int) -> str:
        """Convert millisecond timestamp to year-month string.

        Kept for backward compatibility (used by resource_guard.py).
        """
        return _timestamp_to_year_month(timestamp_ms)

    @staticmethod
    def _normalize_timestamps(
        ticks: pl.DataFrame, timestamp_col: str = "timestamp"
    ) -> pl.DataFrame:
        """Normalize timestamps to milliseconds.

        Handles:
        - Datetime columns → epoch ms
        - Int64 microsecond values (>1e13) → divide by 1000

        Parameters
        ----------
        df : pl.DataFrame
            DataFrame with timestamp column.
        timestamp_col : str
            Name of the timestamp column.

        Returns
        -------
        pl.DataFrame
            DataFrame with timestamps normalized to milliseconds (Int64).
        """
        if ticks.schema[timestamp_col] in (pl.Datetime, pl.Date):
            ticks = ticks.with_columns(
                pl.col(timestamp_col).dt.epoch(time_unit="ms").alias(timestamp_col)
            )
        elif ticks.schema[timestamp_col] == pl.Int64:
            max_val = ticks[timestamp_col].max()
            if max_val is not None and max_val > _US_MS_THRESHOLD:
                ticks = ticks.with_columns(
                    (pl.col(timestamp_col) // 1000).alias(timestamp_col)
                )
        return ticks

    def write_ticks(
        self,
        symbol: str,
        ticks: pl.DataFrame | pd.DataFrame,
        *,
        timestamp_col: str = "timestamp",
    ) -> int:
        """Write tick data to Parquet files, partitioned by day.

        Each UTC day gets its own Parquet file, written atomically.
        Timestamps are normalized to milliseconds. Within each daily file,
        rows are sorted by ``agg_trade_id`` if that column exists.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        ticks : pl.DataFrame | pd.DataFrame
            Tick data with timestamp column
        timestamp_col : str
            Name of the timestamp column (milliseconds since epoch or datetime)

        Returns
        -------
        int
            Number of rows written

        Notes
        -----
        Tick data is partitioned by day and written atomically per day.
        If a daily file already exists, new trades are deduped on
        ``agg_trade_id`` and merged with existing data.
        """
        # Convert pandas to polars if needed
        if not isinstance(ticks, pl.DataFrame):
            ticks = pl.from_pandas(ticks)

        if ticks.is_empty():
            return 0

        # Ensure symbol directory exists
        symbol_dir = self._get_symbol_dir(symbol)
        symbol_dir.mkdir(parents=True, exist_ok=True)

        # Normalize timestamps to milliseconds (handles datetime, µs Int64)
        ticks = self._normalize_timestamps(ticks, timestamp_col)

        # Add day column for partitioning (vectorized, no Python per-row calls)
        # MEM-001: Replaced map_elements() with native Polars dt operations
        # Impact: 13.4 GB → ~100 MB (99% reduction)
        ticks = ticks.with_columns(
            pl.col(timestamp_col)
            .cast(pl.Datetime(time_unit="ms"))
            .dt.strftime("%Y-%m-%d")
            .alias("_day")
        )

        # Group by day and write (Issue #73: atomic writes + corruption recovery)
        total_rows = 0
        for (day_key,), group_df in ticks.group_by("_day"):
            parquet_path = self._get_parquet_path(symbol, day_key)

            # Drop the partition column before writing
            write_df = group_df.drop("_day")

            # Sort by agg_trade_id within each daily file for deterministic ordering
            if "agg_trade_id" in write_df.columns:
                write_df = write_df.sort("agg_trade_id")

            if parquet_path.exists():
                # Issue #96 Task #39: Skip validation for tiny files
                file_size = parquet_path.stat().st_size
                if file_size < _backfill_cfg().min_parquet_cache or not _validate_and_recover_parquet(
                    parquet_path, auto_delete=True
                ):
                    # File was corrupted/tiny and deleted, write fresh
                    _atomic_write_parquet(write_df, parquet_path)
                # Dedup on agg_trade_id to prevent accumulation (Issue #78)
                elif "agg_trade_id" in write_df.columns:
                    write_deduped = write_df.unique(
                        subset=["agg_trade_id"], maintain_order=True
                    )
                    # Single read: load full file once for both dedup and concat
                    existing_df = pl.read_parquet(parquet_path)
                    write_new = write_deduped.join(
                        existing_df.select("agg_trade_id"),
                        on="agg_trade_id",
                        how="anti",
                    )
                    if not write_new.is_empty():
                        # Align columns
                        for col in existing_df.columns:
                            if col not in write_new.columns:
                                write_new = write_new.with_columns(
                                    pl.lit(None)
                                    .cast(existing_df.schema[col])
                                    .alias(col)
                                )
                        write_new = write_new.select(existing_df.columns)
                        combined = pl.concat([existing_df, write_new])
                        if "agg_trade_id" in combined.columns:
                            combined = combined.sort("agg_trade_id")
                        _atomic_write_parquet(combined, parquet_path)
                        del combined
                    del existing_df
                else:
                    # No trade ID column, overwrite
                    _atomic_write_parquet(write_df, parquet_path)
            else:
                # Write new file atomically
                _atomic_write_parquet(write_df, parquet_path)

            total_rows += len(write_df)

        return total_rows

    # Default Parquet compression ratio (compressed -> in-memory expansion)
    # Empirically measured: Binance aggTrades Parquet files expand ~4x
    _COMPRESSION_RATIO: float = 4.0

    def read_ticks(
        self,
        symbol: str,
        start_ts: int | None = None,
        end_ts: int | None = None,
        *,
        timestamp_col: str = "timestamp",
        max_memory_mb: int | None = None,
    ) -> pl.DataFrame:
        """Read tick data from Parquet files.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        start_ts : int | None
            Start timestamp in milliseconds (inclusive)
        end_ts : int | None
            End timestamp in milliseconds (inclusive)
        timestamp_col : str
            Name of the timestamp column
        max_memory_mb : int | None
            Memory budget in MB. If the estimated in-memory size exceeds
            this limit, raises MemoryError with a suggestion to use
            read_ticks_streaming(). None disables the guard.

        Returns
        -------
        pl.DataFrame
            Tick data filtered by time range

        Raises
        ------
        MemoryError
            If estimated memory exceeds max_memory_mb budget.

        Notes
        -----
        Reads all relevant daily files and concatenates them.
        Uses lazy evaluation for efficient memory usage.
        """
        symbol_dir = self._get_symbol_dir(symbol)

        if not symbol_dir.exists():
            return pl.DataFrame()

        # Find relevant parquet files
        parquet_files = sorted(symbol_dir.glob("*.parquet"))

        if not parquet_files:
            return pl.DataFrame()

        # Filter files by day range if time range specified
        if start_ts is not None and end_ts is not None:
            start_day = self._timestamp_to_day(start_ts)
            end_day = self._timestamp_to_day(end_ts)

            parquet_files = [
                f for f in parquet_files if start_day <= f.stem <= end_day
            ]

        if not parquet_files:
            return pl.DataFrame()

        # Issue #73: Validate files before reading, auto-delete corrupted ones
        valid_files = []
        for f in parquet_files:
            if _validate_and_recover_parquet(f, auto_delete=True):
                valid_files.append(f)
            # else: corrupted file deleted, will be re-fetched on next call

        if not valid_files:
            return pl.DataFrame()

        parquet_files = valid_files

        # MEM-004: Estimate size before materializing (Issue #49)
        if max_memory_mb is not None:
            total_bytes = sum(f.stat().st_size for f in parquet_files)
            estimated_mb = int(total_bytes * self._COMPRESSION_RATIO / (1024 * 1024))
            if estimated_mb > max_memory_mb:
                msg = (
                    f"Estimated {estimated_mb} MB for {symbol} "
                    f"({len(parquet_files)} files), exceeds budget "
                    f"{max_memory_mb} MB. Use read_ticks_streaming() "
                    f"for chunked loading."
                )
                raise MemoryError(msg)

        # LAZY LOADING with predicate pushdown
        lazy_dfs = [pl.scan_parquet(f) for f in parquet_files]
        result = pl.concat(lazy_dfs)

        # Apply time range filter (pushed down to Parquet)
        if start_ts is not None:
            result = result.filter(pl.col(timestamp_col) >= start_ts)
        if end_ts is not None:
            result = result.filter(pl.col(timestamp_col) <= end_ts)

        # Sort and materialize
        return result.sort(timestamp_col).collect()

    def read_ticks_streaming(
        self,
        symbol: str,
        start_ts: int | None = None,
        end_ts: int | None = None,
        *,
        chunk_size: int = 100_000,
        timestamp_col: str = "timestamp",
    ) -> Iterator[pl.DataFrame]:
        """Read tick data in streaming chunks to avoid OOM on large ranges.

        This method yields chunks of tick data instead of loading everything
        into memory at once. Essential for high-volume days.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        start_ts : int | None
            Start timestamp in milliseconds (inclusive)
        end_ts : int | None
            End timestamp in milliseconds (inclusive)
        chunk_size : int
            Number of rows per chunk (default: 100,000)
        timestamp_col : str
            Name of the timestamp column

        Yields
        ------
        pl.DataFrame
            Chunks of tick data, sorted by timestamp within each chunk

        Notes
        -----
        Memory usage is O(chunk_size) instead of O(total_ticks).
        Each chunk is sorted independently; overall order is maintained
        because parquet files are read in day order.

        Examples
        --------
        >>> storage = TickStorage()
        >>> for chunk in storage.read_ticks_streaming("BTCUSDT", start_ts, end_ts):
        ...     process_chunk(chunk)
        """
        return _read_ticks_streaming(
            self._ticks_dir,
            symbol,
            start_ts,
            end_ts,
            chunk_size=chunk_size,
            timestamp_col=timestamp_col,
        )

    def has_ticks(
        self,
        symbol: str,
        start_ts: int,
        end_ts: int,
        *,
        min_coverage: float = 0.95,
        timestamp_col: str = "timestamp",
    ) -> bool:
        """Check if tick data exists for the specified time range.

        Issue #277: With daily partitioning, this is a fast Path.exists()
        check per day instead of reading Parquet metadata.

        Parameters
        ----------
        symbol : str
            Trading symbol
        start_ts : int
            Start timestamp in milliseconds
        end_ts : int
            End timestamp in milliseconds
        min_coverage : float
            Minimum coverage ratio (0.0 to 1.0)
        timestamp_col : str
            Name of the timestamp column

        Returns
        -------
        bool
            True if sufficient data exists
        """
        return _has_ticks(
            self._ticks_dir,
            symbol,
            start_ts,
            end_ts,
            min_coverage=min_coverage,
            timestamp_col=timestamp_col,
        )

    def list_symbols(self) -> list[str]:
        """List all symbols with stored tick data.

        Returns
        -------
        list[str]
            List of symbol names
        """
        if not self._ticks_dir.exists():
            return []

        return sorted(
            d.name for d in self._ticks_dir.iterdir() if d.is_dir() and d.name != ""
        )

    def list_days(self, symbol: str) -> list[str]:
        """List all days with stored tick data for a symbol.

        Parameters
        ----------
        symbol : str
            Trading symbol

        Returns
        -------
        list[str]
            List of day strings (e.g., ["2024-01-01", "2024-01-02"])
        """
        symbol_dir = self._get_symbol_dir(symbol)

        if not symbol_dir.exists():
            return []

        return sorted(f.stem for f in symbol_dir.glob("*.parquet"))

    def list_months(self, symbol: str) -> list[str]:
        """List all months with stored tick data for a symbol.

        Backward compatible: derives unique months from daily files.

        Parameters
        ----------
        symbol : str
            Trading symbol

        Returns
        -------
        list[str]
            List of year-month strings (e.g., ["2024-01", "2024-02"])
        """
        days = self.list_days(symbol)
        # Extract YYYY-MM from YYYY-MM-DD, deduplicate, sort
        return sorted({d[:_DAY_KEY_MONTH_END] for d in days if len(d) >= _DAY_KEY_MONTH_END})

    def delete_ticks(self, symbol: str, day_or_month: str | None = None) -> bool:
        """Delete tick data for a symbol, specific day, or month.

        Parameters
        ----------
        symbol : str
            Trading symbol
        day_or_month : str | None
            Specific day ("2024-01-15") or month ("2024-01") to delete,
            or None for all data for the symbol.

        Returns
        -------
        bool
            True if files were deleted
        """
        if day_or_month is not None:
            return self._delete_by_key(symbol, day_or_month)

        # Delete all data for symbol
        symbol_dir = self._get_symbol_dir(symbol)
        if symbol_dir.exists():
            shutil.rmtree(symbol_dir)
            return True
        return False

    def _delete_by_key(self, symbol: str, day_or_month: str) -> bool:
        """Delete tick data by day or month key."""
        if len(day_or_month) == _DAY_KEY_LEN:
            parquet_path = self._get_parquet_path(symbol, day_or_month)
            if parquet_path.exists():
                parquet_path.unlink()
                return True
            return False
        if len(day_or_month) == _MONTH_KEY_LEN:
            symbol_dir = self._get_symbol_dir(symbol)
            if not symbol_dir.exists():
                return False
            deleted = False
            for f in symbol_dir.glob(f"{day_or_month}-*.parquet"):
                f.unlink()
                deleted = True
            return deleted
        # Unknown format — try as day key
        parquet_path = self._get_parquet_path(symbol, day_or_month)
        if parquet_path.exists():
            parquet_path.unlink()
            return True
        return False

    def get_stats(self, symbol: str) -> dict:
        """Get storage statistics for a symbol.

        Parameters
        ----------
        symbol : str
            Trading symbol

        Returns
        -------
        dict
            Statistics including file count, total size, row count, date range
        """
        symbol_dir = self._get_symbol_dir(symbol)

        if not symbol_dir.exists():
            return {
                "symbol": symbol,
                "exists": False,
                "file_count": 0,
                "total_size_bytes": 0,
                "total_rows": 0,
            }

        parquet_files = list(symbol_dir.glob("*.parquet"))
        total_size = sum(f.stat().st_size for f in parquet_files)
        total_rows = 0
        days = []

        for f in parquet_files:
            file_data = pl.read_parquet(f)
            total_rows += len(file_data)
            days.append(f.stem)

        return {
            "symbol": symbol,
            "exists": True,
            "file_count": len(parquet_files),
            "total_size_bytes": total_size,
            "total_size_mb": total_size / 1024 / 1024,
            "total_rows": total_rows,
            "days": sorted(days),
            "compression": f"{COMPRESSION}-{COMPRESSION_LEVEL}",
        }

    def fetch_month(
        self,
        symbol: str,
        year: int,
        month: int,
        *,
        timestamp_col: str = "timestamp",
        force_refresh: bool = False,
    ) -> pl.LazyFrame:
        """Fetch tick data for a specific month (lazy loading).

        Returns a LazyFrame for memory efficiency. Reads all daily files
        for the requested month and concatenates them lazily.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT" or "BINANCE_SPOT_BTCUSDT")
        year : int
            Year (e.g., 2024)
        month : int
            Month (1-12)
        timestamp_col : str
            Name of the timestamp column
        force_refresh : bool
            If True, skip cache and return empty LazyFrame (caller must fetch)

        Returns
        -------
        pl.LazyFrame
            Lazy frame for the month's tick data, or empty LazyFrame if not cached

        Examples
        --------
        >>> storage = TickStorage()
        >>> lf = storage.fetch_month("BTCUSDT", 2024, 1)
        >>> df = lf.collect()  # Materialize when needed
        """
        if force_refresh:
            return pl.LazyFrame()

        # Build start/end dates for the month
        from calendar import monthrange
        _, n_days = monthrange(year, month)
        start_date = date(year, month, 1)
        end_date = date(year, month, n_days)

        # Collect all daily files for this month
        lazy_frames = []
        for day_str in _iter_days(start_date, end_date):
            parquet_path = self._get_parquet_path(symbol, day_str)
            if parquet_path.exists():
                lazy_frames.append(pl.scan_parquet(parquet_path))

        if not lazy_frames:
            return pl.LazyFrame()

        return pl.concat(lazy_frames)

    def fetch_date_range(
        self,
        symbol: str,
        start_date: str,
        end_date: str,
        *,
        timestamp_col: str = "timestamp",
    ) -> Iterator[pl.LazyFrame]:
        """Iterate over tick data for date range, one day at a time.

        Yields LazyFrames for each day in range. This is the recommended
        approach for processing large date ranges without loading all data
        into memory at once.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        start_date : str
            Start date "YYYY-MM-DD"
        end_date : str
            End date "YYYY-MM-DD"
        timestamp_col : str
            Name of the timestamp column

        Yields
        ------
        pl.LazyFrame
            Lazy frame for each day with available data

        Examples
        --------
        >>> storage = TickStorage()
        >>> for lf in storage.fetch_date_range("BTCUSDT", "2024-01-01", "2024-03-31"):
        ...     df = lf.collect()
        ...     print(f"Processing {len(df)} ticks")

        Notes
        -----
        - Only yields LazyFrames for days with cached data
        - Data is NOT automatically downloaded - use get_open_deviation_bars() first
        - Each LazyFrame can be collected independently for O(day) memory
        """
        return _fetch_date_range(
            self._ticks_dir,
            symbol,
            start_date,
            end_date,
        )
