"""
增量爬虫
支持断点续爬、进度保存和恢复
"""
import json
import hashlib
from dataclasses import dataclass, field
from typing import Optional, List, Set, Dict, Any, Callable, Awaitable
from pathlib import Path
from datetime import datetime
from playwright.async_api import Page
from loguru import logger


@dataclass
class CrawlConfig:
    """爬虫配置"""
    max_concurrent: int = 5
    request_delay: float = 1.0
    max_retries: int = 3
    timeout: float = 30000
    save_interval: int = 10  # 每爬取多少页面保存一次进度
    user_agent: Optional[str] = None


@dataclass
class CrawlProgress:
    """爬取进度"""
    total_urls: int = 0
    visited_count: int = 0
    success_count: int = 0
    failed_count: int = 0
    start_time: Optional[datetime] = None
    end_time: Optional[datetime] = None
    current_url: Optional[str] = None

    @property
    def progress_percent(self) -> float:
        """进度百分比"""
        if self.total_urls == 0:
            return 0.0
        return (self.visited_count / self.total_urls) * 100

    @property
    def elapsed_seconds(self) -> float:
        """已用时间（秒）"""
        if self.start_time is None:
            return 0.0
        end = self.end_time or datetime.now()
        return (end - self.start_time).total_seconds()


@dataclass
class CrawlResult:
    """爬取结果"""
    success: bool
    url: str
    data: Optional[Dict[str, Any]] = None
    error: Optional[str] = None
    timestamp: datetime = field(default_factory=datetime.now)


@dataclass
class CrawlState:
    """爬取状态（用于持久化）"""
    visited_urls: List[str] = field(default_factory=list)
    pending_urls: List[str] = field(default_factory=list)
    failed_urls: Dict[str, str] = field(default_factory=dict)
    created_at: datetime = field(default_factory=datetime.now)
    updated_at: datetime = field(default_factory=datetime.now)


class IncrementalCrawler:
    """
    增量爬虫
    支持断点续爬，自动保存和恢复进度
    """

    def __init__(
        self,
        page: Page,
        config: Optional[CrawlConfig] = None,
    ):
        self.page = page
        self.config = config or CrawlConfig()

        # URL 管理
        self._visited: Set[str] = set()
        self._pending: List[str] = []
        self._failed: Dict[str, str] = {}  # url -> error message

        # 进度追踪
        self._progress = CrawlProgress()
        self._results: List[CrawlResult] = []

        # 状态
        self._is_running = False
        self._should_stop = False

    # ==================== URL 管理 ====================

    def add_url(self, url: str) -> bool:
        """
        添加单个 URL

        Args:
            url: 要添加的 URL

        Returns:
            bool: 是否添加成功（重复返回 False）
        """
        normalized = self._normalize_url(url)
        if normalized in self._visited:
            return False
        if normalized not in self._pending:
            self._pending.append(normalized)
            self._progress.total_urls += 1
            return True
        return False

    def add_urls(self, urls: List[str]) -> int:
        """
        批量添加 URL

        Args:
            urls: URL 列表

        Returns:
            int: 成功添加的数量
        """
        added = 0
        for url in urls:
            if self.add_url(url):
                added += 1
        return added

    def remove_url(self, url: str) -> bool:
        """移除待爬取的 URL"""
        normalized = self._normalize_url(url)
        if normalized in self._pending:
            self._pending.remove(normalized)
            self._progress.total_urls -= 1
            return True
        return False

    def get_visited_urls(self) -> List[str]:
        """获取已访问的 URL 列表"""
        return list(self._visited)

    def get_pending_urls(self) -> List[str]:
        """获取待访问的 URL 列表"""
        return list(self._pending)

    def get_failed_urls(self) -> Dict[str, str]:
        """获取失败的 URL 及错误信息"""
        return dict(self._failed)

    # ==================== 爬取控制 ====================

    async def crawl(
        self,
        processor: Callable[[Page, str], Awaitable[Dict[str, Any]]],
        save_path: Optional[str] = None,
    ) -> CrawlProgress:
        """
        执行爬取

        Args:
            processor: 页面处理函数，接收 Page 和 URL，返回爬取数据
            save_path: 进度保存路径

        Returns:
            CrawlProgress: 爬取进度
        """
        if self._is_running:
            logger.warning("爬虫已在运行中")
            return self._progress

        self._is_running = True
        self._should_stop = False
        self._progress.start_time = datetime.now()
        self._progress.total_urls = len(self._pending) + len(self._visited)

        crawled_count = 0

        try:
            while self._pending and not self._should_stop:
                url = self._pending.pop(0)
                self._progress.current_url = url

                result = await self._crawl_single(url, processor)
                self._results.append(result)

                if result.success:
                    self._visited.add(url)
                    self._progress.success_count += 1
                else:
                    self._failed[url] = result.error or "Unknown error"
                    self._progress.failed_count += 1

                self._progress.visited_count += 1
                crawled_count += 1

                # 定期保存进度
                if save_path and crawled_count % self.config.save_interval == 0:
                    self.save_progress(save_path)

                # 延迟
                if self._pending and self.config.request_delay > 0:
                    import asyncio
                    await asyncio.sleep(self.config.request_delay)

        finally:
            self._is_running = False
            self._progress.end_time = datetime.now()
            self._progress.current_url = None

            if save_path:
                self.save_progress(save_path)

        return self._progress

    async def _crawl_single(
        self,
        url: str,
        processor: Callable[[Page, str], Awaitable[Dict[str, Any]]],
    ) -> CrawlResult:
        """爬取单个页面"""
        for attempt in range(self.config.max_retries):
            try:
                # 导航到页面
                await self.page.goto(url, timeout=self.config.timeout)

                # 执行处理
                data = await processor(self.page, url)

                logger.debug(f"爬取成功: {url}")
                return CrawlResult(
                    success=True,
                    url=url,
                    data=data,
                )

            except Exception as e:
                error_msg = str(e)
                logger.warning(f"爬取失败 (attempt {attempt + 1}): {url}, error={error_msg}")

                if attempt < self.config.max_retries - 1:
                    import asyncio
                    await asyncio.sleep(self.config.request_delay * (attempt + 1))

        return CrawlResult(
            success=False,
            url=url,
            error=error_msg if 'error_msg' in dir() else "Max retries exceeded",
        )

    def stop(self):
        """停止爬取"""
        self._should_stop = True

    def is_running(self) -> bool:
        """是否正在运行"""
        return self._is_running

    # ==================== 进度持久化 ====================

    def save_progress(self, file_path: str) -> bool:
        """
        保存进度到文件

        Args:
            file_path: 保存路径

        Returns:
            bool: 是否保存成功
        """
        try:
            state = CrawlState(
                visited_urls=list(self._visited),
                pending_urls=self._pending.copy(),
                failed_urls=dict(self._failed),
                updated_at=datetime.now(),
            )

            Path(file_path).parent.mkdir(parents=True, exist_ok=True)

            with open(file_path, 'w', encoding='utf-8') as f:
                json.dump({
                    'visited_urls': state.visited_urls,
                    'pending_urls': state.pending_urls,
                    'failed_urls': state.failed_urls,
                    'created_at': state.created_at.isoformat(),
                    'updated_at': state.updated_at.isoformat(),
                }, f, ensure_ascii=False, indent=2)

            logger.debug(f"进度已保存: {file_path}")
            return True

        except Exception as e:
            logger.error(f"保存进度失败: {e}")
            return False

    def load_progress(self, file_path: str) -> bool:
        """
        从文件恢复进度

        Args:
            file_path: 进度文件路径

        Returns:
            bool: 是否恢复成功
        """
        try:
            if not Path(file_path).exists():
                logger.warning(f"进度文件不存在: {file_path}")
                return False

            with open(file_path, 'r', encoding='utf-8') as f:
                data = json.load(f)

            self._visited = set(data.get('visited_urls', []))
            self._pending = data.get('pending_urls', [])
            self._failed = data.get('failed_urls', {})

            # 更新进度
            self._progress.visited_count = len(self._visited)
            self._progress.success_count = len(self._visited)
            self._progress.failed_count = len(self._failed)
            self._progress.total_urls = len(self._visited) + len(self._pending)

            logger.info(f"进度已恢复: 已访问 {len(self._visited)}, 待处理 {len(self._pending)}")
            return True

        except Exception as e:
            logger.error(f"恢复进度失败: {e}")
            return False

    def clear_progress(self):
        """清除所有进度"""
        self._visited.clear()
        self._pending.clear()
        self._failed.clear()
        self._results.clear()
        self._progress = CrawlProgress()

    # ==================== 结果获取 ====================

    def get_results(self) -> List[CrawlResult]:
        """获取所有爬取结果"""
        return self._results.copy()

    def get_successful_results(self) -> List[CrawlResult]:
        """获取成功的结果"""
        return [r for r in self._results if r.success]

    def get_failed_results(self) -> List[CrawlResult]:
        """获取失败的结果"""
        return [r for r in self._results if not r.success]

    def get_progress(self) -> CrawlProgress:
        """获取当前进度"""
        return self._progress

    # ==================== 私有方法 ====================

    def _normalize_url(self, url: str) -> str:
        """标准化 URL（去除 fragment 等）"""
        url = url.strip()
        # 移除 fragment
        if '#' in url:
            url = url.split('#')[0]
        return url

    def _url_hash(self, url: str) -> str:
        """计算 URL 的哈希值"""
        return hashlib.md5(url.encode()).hexdigest()


def create_incremental_crawler(
    page: Page,
    config: Optional[CrawlConfig] = None,
) -> IncrementalCrawler:
    """创建增量爬虫实例"""
    return IncrementalCrawler(page, config)
