"""File watcher extractor for monitoring directories for new data files.

Uses ``watchfiles`` (optional) for efficient file system watching with a
fallback to polling via ``pathlib.Path.glob()``.
"""

from __future__ import annotations

import asyncio
import json
import logging
import shutil
from pathlib import Path
from typing import Any, AsyncIterator

from pycharter.etl_generator.extractors._streaming import (
    BatchAccumulator,
    StreamingConfig,
)
from pycharter.etl_generator.extractors.base import BaseExtractor

logger = logging.getLogger(__name__)

_FORMAT_READERS: dict[str, Any] = {}


def _read_file_records(path: Path, file_format: str | None) -> list[dict[str, Any]]:
    """Read records from a data file."""
    fmt = file_format or _detect_format(path)

    if fmt == "json":
        with open(path) as f:
            data = json.load(f)
        return data if isinstance(data, list) else [data]

    if fmt == "jsonl":
        records = []
        with open(path) as f:
            for line in f:
                line = line.strip()
                if line:
                    records.append(json.loads(line))
        return records

    if fmt == "csv":
        import csv

        records = []
        with open(path, newline="") as f:
            reader = csv.DictReader(f)
            for row in reader:
                records.append(dict(row))
        return records

    if fmt == "yaml":
        import yaml

        with open(path) as f:
            data = yaml.safe_load(f)
        if isinstance(data, list):
            return data
        if isinstance(data, dict):
            return [data]
        return []

    raise ValueError(f"Unsupported file format: {fmt}")


def _detect_format(path: Path) -> str:
    """Detect file format from extension."""
    suffix = path.suffix.lower()
    mapping = {
        ".json": "json",
        ".jsonl": "jsonl",
        ".ndjson": "jsonl",
        ".csv": "csv",
        ".yaml": "yaml",
        ".yml": "yaml",
    }
    fmt = mapping.get(suffix)
    if not fmt:
        raise ValueError(f"Cannot detect format for extension: {suffix}")
    return fmt


class FileWatcherExtractor(BaseExtractor):
    """Watch a directory for new files and extract their contents.

    Uses ``watchfiles`` for efficient OS-level file watching when available,
    falling back to polling with ``pathlib.Path.glob()``.

    Args:
        watch_dir: Directory to watch for new files.
        patterns: Glob patterns to match (e.g. ``["*.json", "*.csv"]``).
        file_format: Explicit file format (auto-detected from extension if None).
        recursive: Whether to watch subdirectories.
        poll_interval: Seconds between polls when using fallback mode.
        process_existing: Whether to process files already present on startup.
        move_after_process: Directory to move files to after processing.
        batch_size: Records per batch.
        shutdown_event: External event to signal graceful shutdown.
        max_records: Stop after this many total records.
    """

    def __init__(
        self,
        watch_dir: str | Path,
        patterns: list[str] | None = None,
        file_format: str | None = None,
        recursive: bool = False,
        poll_interval: float = 1.0,
        process_existing: bool = False,
        move_after_process: str | Path | None = None,
        batch_size: int = 1000,
        shutdown_event: asyncio.Event | None = None,
        max_records: int | None = None,
    ) -> None:
        self._watch_dir = Path(watch_dir)
        self._patterns = patterns or ["*"]
        self._file_format = file_format
        self._recursive = recursive
        self._poll_interval = poll_interval
        self._process_existing = process_existing
        self._move_after_process = (
            Path(move_after_process) if move_after_process else None
        )
        self._streaming_config = StreamingConfig(
            shutdown_event=shutdown_event,
            max_records=max_records,
            batch_size=batch_size,
        )
        self._processed_files: set[Path] = set()

    @classmethod
    def from_config(cls, config: dict[str, Any]) -> FileWatcherExtractor:
        """Create a FileWatcherExtractor from a configuration dictionary.

        Args:
            config: Extractor configuration with file watcher-specific keys.

        Returns:
            Configured FileWatcherExtractor instance.
        """
        return cls(
            watch_dir=config["watch_dir"],
            patterns=config.get("patterns"),
            file_format=config.get("file_format"),
            recursive=config.get("recursive", False),
            poll_interval=config.get("poll_interval", 1.0),
            process_existing=config.get("process_existing", False),
            move_after_process=config.get("move_after_process"),
            batch_size=config.get("batch_size", 1000),
            max_records=config.get("max_records"),
        )

    def validate_config(self, extract_config: dict[str, Any]) -> None:
        """Validate file watcher-specific configuration."""
        if not extract_config.get("watch_dir"):
            raise ValueError("File watcher extractor requires 'watch_dir'")

    async def extract(self, **params: Any) -> AsyncIterator[list[dict[str, Any]]]:
        """Watch directory and extract data from new files.

        Yields:
            Batches of records from detected files.
        """
        cfg = self._streaming_config
        accumulator = BatchAccumulator(cfg.batch_size, cfg.batch_timeout)
        total_records = 0

        # Process existing files first if configured
        if self._process_existing:
            async for batch in self._process_directory(accumulator, cfg):
                yield batch
                total_records += sum(1 for _ in batch)
                if cfg.max_records and total_records >= cfg.max_records:
                    return
        else:
            # Mark existing files as already processed
            for file_path in self._find_matching_files():
                self._processed_files.add(file_path)

        # Watch for new files
        use_watchfiles = False
        try:
            import watchfiles

            use_watchfiles = True
        except ImportError:
            logger.debug("watchfiles not available, using polling fallback")

        if use_watchfiles:
            async for batch in self._watch_with_watchfiles(
                accumulator, cfg, total_records
            ):
                yield batch
        else:
            async for batch in self._watch_with_polling(
                accumulator, cfg, total_records
            ):
                yield batch

    async def _process_directory(
        self,
        accumulator: BatchAccumulator,
        cfg: StreamingConfig,
    ) -> AsyncIterator[list[dict[str, Any]]]:
        """Process all matching files in the directory."""
        for file_path in self._find_matching_files():
            if file_path in self._processed_files:
                continue
            async for batch in self._process_file(file_path, accumulator, cfg):
                yield batch

    async def _process_file(
        self,
        file_path: Path,
        accumulator: BatchAccumulator,
        cfg: StreamingConfig,
    ) -> AsyncIterator[list[dict[str, Any]]]:
        """Read a file and yield batches of its records."""
        try:
            records = _read_file_records(file_path, self._file_format)
            self._processed_files.add(file_path)

            for record in records:
                batch = await accumulator.add(record)
                if batch:
                    yield batch

            # Move file after processing if configured
            if self._move_after_process:
                self._move_after_process.mkdir(parents=True, exist_ok=True)
                dest = self._move_after_process / file_path.name
                shutil.move(str(file_path), str(dest))
                logger.debug("Moved processed file: %s -> %s", file_path, dest)

        except Exception as e:
            logger.warning("Failed to process file %s: %s", file_path, e)

    async def _watch_with_watchfiles(
        self,
        accumulator: BatchAccumulator,
        cfg: StreamingConfig,
        total_records: int,
    ) -> AsyncIterator[list[dict[str, Any]]]:
        """Watch using watchfiles library."""
        import watchfiles

        async for changes in watchfiles.awatch(
            self._watch_dir,
            recursive=self._recursive,
            step=int(self._poll_interval * 1000),
            stop_event=cfg.shutdown_event,
        ):
            for change_type, path_str in changes:
                if change_type != watchfiles.Change.added:
                    continue
                file_path = Path(path_str)
                if not self._matches_patterns(file_path):
                    continue
                if file_path in self._processed_files:
                    continue

                async for batch in self._process_file(file_path, accumulator, cfg):
                    yield batch
                    total_records += len(batch)

            if cfg.max_records and total_records >= cfg.max_records:
                break
            if cfg.shutdown_event and cfg.shutdown_event.is_set():
                break

        remaining = await accumulator.flush()
        if remaining:
            yield remaining

    async def _watch_with_polling(
        self,
        accumulator: BatchAccumulator,
        cfg: StreamingConfig,
        total_records: int,
    ) -> AsyncIterator[list[dict[str, Any]]]:
        """Watch using polling fallback."""
        while True:
            if cfg.shutdown_event and cfg.shutdown_event.is_set():
                break

            new_files = [
                f for f in self._find_matching_files() if f not in self._processed_files
            ]

            for file_path in new_files:
                async for batch in self._process_file(file_path, accumulator, cfg):
                    yield batch
                    total_records += len(batch)

                if cfg.max_records and total_records >= cfg.max_records:
                    break

            if cfg.max_records and total_records >= cfg.max_records:
                break

            await asyncio.sleep(self._poll_interval)

        remaining = await accumulator.flush()
        if remaining:
            yield remaining

    def _find_matching_files(self) -> list[Path]:
        """Find all files matching patterns in the watch directory."""
        files: list[Path] = []
        for pattern in self._patterns:
            if self._recursive:
                files.extend(self._watch_dir.rglob(pattern))
            else:
                files.extend(self._watch_dir.glob(pattern))
        return sorted(set(f for f in files if f.is_file()))

    def _matches_patterns(self, file_path: Path) -> bool:
        """Check if a file matches any of the configured patterns."""
        for pattern in self._patterns:
            if file_path.match(pattern):
                return True
        return False

    async def extract_streaming(
        self,
        extract_config: dict[str, Any],
        params: dict[str, Any],
        headers: dict[str, Any],
        contract_dir: Any | None = None,
        batch_size: int = 1000,
        max_records: int | None = None,
        config_context: dict[str, Any] | None = None,
    ) -> AsyncIterator[list[dict[str, Any]]]:
        """Extract via config-driven path (delegates to extract)."""
        async for batch in self.extract(**params):
            yield batch
