"""
智能爬虫 V2
自动识别页面结构，支持列表和详情页抓取
"""
import json
import csv
from dataclasses import dataclass, field
from typing import Optional, List, Dict, Any, Set
from pathlib import Path
from datetime import datetime
from playwright.async_api import Page, Locator
from loguru import logger


@dataclass
class ListStructure:
    """列表结构检测结果"""
    container_selector: str
    item_selector: str
    item_count: int
    detected_fields: List[str]
    confidence: float  # 0.0 - 1.0


@dataclass
class ScrapeResult:
    """爬取结果"""
    success: bool
    data: List[Dict[str, Any]]
    total_count: int
    scraped_count: int
    errors: List[str] = field(default_factory=list)
    timestamp: datetime = field(default_factory=datetime.now)


@dataclass
class SmartScraperConfig:
    """智能爬虫配置"""
    max_items: int = 100
    scroll_to_load: bool = True
    max_scrolls: int = 10
    wait_for_content: float = 1000  # 等待内容加载的毫秒数
    include_links: bool = True
    include_images: bool = True


class SmartScraperV2:
    """
    智能爬虫 V2
    自动识别页面结构，支持列表页和详情页抓取
    """

    def __init__(
        self,
        page: Page,
        config: Optional[SmartScraperConfig] = None,
    ):
        self.page = page
        self.config = config or SmartScraperConfig()
        self._detected_structure: Optional[ListStructure] = None

    # ==================== 结构检测 ====================

    async def detect_list_structure(self) -> Optional[ListStructure]:
        """
        自动检测页面列表结构

        Returns:
            ListStructure: 检测到的列表结构
        """
        # 常见列表容器选择器
        container_patterns = [
            "ul", "ol", ".list", ".items", ".results",
            "[role='list']", ".grid", ".cards", "table tbody",
            ".product-list", ".article-list", ".news-list",
        ]

        # 常见列表项选择器
        item_patterns = [
            "li", ".item", ".card", ".product", ".article",
            "[role='listitem']", "tr", ".entry", ".post",
        ]

        best_match: Optional[ListStructure] = None
        best_confidence = 0.0

        for container in container_patterns:
            container_locator = self.page.locator(container)
            container_count = await container_locator.count()

            if container_count == 0:
                continue

            for item in item_patterns:
                # 构建完整选择器
                if container.startswith('.') or container.startswith('[') or container.startswith('#'):
                    full_selector = f"{container} > {item}"
                else:
                    full_selector = f"{container} {item}"

                item_locator = self.page.locator(full_selector)
                item_count = await item_locator.count()

                if item_count >= 3:  # 至少 3 个项目才可能是列表
                    # 检测字段
                    fields = await self._detect_item_fields(item_locator.first)
                    confidence = self._calculate_confidence(item_count, fields)

                    if confidence > best_confidence:
                        best_confidence = confidence
                        best_match = ListStructure(
                            container_selector=container,
                            item_selector=full_selector,
                            item_count=item_count,
                            detected_fields=fields,
                            confidence=confidence,
                        )

        if best_match:
            self._detected_structure = best_match
            logger.info(f"检测到列表结构: {best_match.item_selector}, {best_match.item_count} 个项目")

        return best_match

    async def _detect_item_fields(self, item_locator: Locator) -> List[str]:
        """检测列表项中的字段"""
        fields = []

        # 检测标题
        title_selectors = ["h1", "h2", "h3", "h4", ".title", ".name", "a"]
        for selector in title_selectors:
            locator = item_locator.locator(selector)
            if await locator.count() > 0:
                text = await locator.first.text_content()
                if text and len(text.strip()) > 0:
                    fields.append("title")
                    break

        # 检测描述
        desc_selectors = ["p", ".description", ".desc", ".summary", ".content"]
        for selector in desc_selectors:
            locator = item_locator.locator(selector)
            if await locator.count() > 0:
                fields.append("description")
                break

        # 检测链接
        link_locator = item_locator.locator("a[href]")
        if await link_locator.count() > 0:
            fields.append("link")

        # 检测图片
        img_locator = item_locator.locator("img")
        if await img_locator.count() > 0:
            fields.append("image")

        # 检测价格
        price_selectors = [".price", ".cost", "[data-price]", ".amount"]
        for selector in price_selectors:
            locator = item_locator.locator(selector)
            if await locator.count() > 0:
                fields.append("price")
                break

        # 检测日期
        date_selectors = ["time", ".date", ".datetime", "[data-date]"]
        for selector in date_selectors:
            locator = item_locator.locator(selector)
            if await locator.count() > 0:
                fields.append("date")
                break

        return fields

    def _calculate_confidence(self, item_count: int, fields: List[str]) -> float:
        """计算结构置信度"""
        confidence = 0.0

        # 项目数量评分
        if item_count >= 10:
            confidence += 0.4
        elif item_count >= 5:
            confidence += 0.3
        elif item_count >= 3:
            confidence += 0.2

        # 字段数量评分
        if len(fields) >= 4:
            confidence += 0.4
        elif len(fields) >= 2:
            confidence += 0.3
        elif len(fields) >= 1:
            confidence += 0.2

        return min(confidence, 1.0)

    # ==================== 列表抓取 ====================

    async def scrape_list(
        self,
        item_selector: Optional[str] = None,
        fields: Optional[Dict[str, str]] = None,
    ) -> ScrapeResult:
        """
        抓取列表数据

        Args:
            item_selector: 列表项选择器（不提供则自动检测）
            fields: 字段映射 {字段名: 选择器}

        Returns:
            ScrapeResult: 抓取结果
        """
        # 确定选择器
        if not item_selector:
            if not self._detected_structure:
                await self.detect_list_structure()
            if self._detected_structure:
                item_selector = self._detected_structure.item_selector
            else:
                return ScrapeResult(
                    success=False,
                    data=[],
                    total_count=0,
                    scraped_count=0,
                    errors=["无法检测列表结构"],
                )

        # 如果需要滚动加载更多
        if self.config.scroll_to_load:
            await self._scroll_to_load_more(item_selector)

        # 获取所有项目
        items_locator = self.page.locator(item_selector)
        total_count = await items_locator.count()

        if total_count == 0:
            return ScrapeResult(
                success=False,
                data=[],
                total_count=0,
                scraped_count=0,
                errors=["未找到列表项"],
            )

        # 限制数量
        limit = min(total_count, self.config.max_items)

        data = []
        errors = []

        for i in range(limit):
            try:
                item_locator = items_locator.nth(i)
                item_data = await self._extract_item_data(item_locator, fields)
                data.append(item_data)
            except Exception as e:
                errors.append(f"第 {i + 1} 项提取失败: {str(e)}")

        logger.info(f"列表抓取完成: {len(data)}/{total_count} 项")

        return ScrapeResult(
            success=len(data) > 0,
            data=data,
            total_count=total_count,
            scraped_count=len(data),
            errors=errors,
        )

    async def _scroll_to_load_more(self, item_selector: str):
        """滚动加载更多内容"""
        prev_count = 0
        scrolls = 0

        while scrolls < self.config.max_scrolls:
            # 滚动到底部
            await self.page.evaluate("window.scrollTo(0, document.body.scrollHeight)")

            # 等待新内容加载
            await self.page.wait_for_timeout(self.config.wait_for_content)

            # 检查是否有新内容
            current_count = await self.page.locator(item_selector).count()

            if current_count <= prev_count:
                break

            prev_count = current_count
            scrolls += 1

            # 检查是否达到上限
            if current_count >= self.config.max_items:
                break

    async def _extract_item_data(
        self,
        item_locator: Locator,
        fields: Optional[Dict[str, str]] = None,
    ) -> Dict[str, Any]:
        """提取单个项目的数据"""
        data = {}

        # 自动提取常见字段
        # 标题
        title = await self._extract_text(item_locator, ["h1", "h2", "h3", "h4", ".title", ".name", "a"])
        if title:
            data["title"] = title

        # 描述
        desc = await self._extract_text(item_locator, ["p", ".description", ".desc", ".summary"])
        if desc:
            data["description"] = desc

        # 链接
        if self.config.include_links:
            link = await self._extract_attribute(item_locator, "a[href]", "href")
            if link:
                data["link"] = link

        # 图片
        if self.config.include_images:
            img = await self._extract_attribute(item_locator, "img", "src")
            if img:
                data["image"] = img

        # 价格
        price = await self._extract_text(item_locator, [".price", ".cost", "[data-price]"])
        if price:
            data["price"] = price

        # 日期
        date = await self._extract_text(item_locator, ["time", ".date", "[data-date]"])
        if date:
            data["date"] = date

        # 使用自定义字段映射覆盖
        if fields:
            for field_name, selector in fields.items():
                value = await self._extract_text(item_locator, [selector])
                if value:
                    data[field_name] = value

        return data

    async def _extract_text(self, parent: Locator, selectors: List[str]) -> Optional[str]:
        """从父元素中提取文本"""
        for selector in selectors:
            locator = parent.locator(selector)
            if await locator.count() > 0:
                text = await locator.first.text_content()
                if text:
                    return text.strip()
        return None

    async def _extract_attribute(
        self,
        parent: Locator,
        selector: str,
        attr: str,
    ) -> Optional[str]:
        """从父元素中提取属性"""
        locator = parent.locator(selector)
        if await locator.count() > 0:
            return await locator.first.get_attribute(attr)
        return None

    # ==================== 详情页抓取 ====================

    async def scrape_detail(self, url: str) -> ScrapeResult:
        """
        抓取详情页

        Args:
            url: 详情页 URL

        Returns:
            ScrapeResult: 抓取结果
        """
        try:
            await self.page.goto(url)

            # 等待页面加载
            await self.page.wait_for_load_state("domcontentloaded")

            # 提取详情数据
            data = {}

            # 标题
            title = await self._extract_text(self.page, ["h1", ".title", ".name"])
            if title:
                data["title"] = title

            # 正文内容
            content_selectors = [
                "article", ".content", ".article-body", ".post-content",
                ".detail-content", ".main-content", "#content",
            ]
            content = await self._extract_text(self.page, content_selectors)
            if content:
                data["content"] = content

            # 图片
            if self.config.include_images:
                images = []
                img_locator = self.page.locator("article img, .content img, .article-body img")
                count = await img_locator.count()
                for i in range(count):
                    src = await img_locator.nth(i).get_attribute("src")
                    if src:
                        images.append(src)
                if images:
                    data["images"] = images

            # 元数据
            # 作者
            author = await self._extract_text(self.page, [".author", "[data-author]", ".byline"])
            if author:
                data["author"] = author

            # 日期
            date = await self._extract_text(self.page, ["time", ".date", "[data-date]", ".publish-date"])
            if date:
                data["date"] = date

            # 标签
            tags_locator = self.page.locator(".tags a, .tag, [data-tag]")
            tag_count = await tags_locator.count()
            if tag_count > 0:
                tags = []
                for i in range(tag_count):
                    tag = await tags_locator.nth(i).text_content()
                    if tag:
                        tags.append(tag.strip())
                if tags:
                    data["tags"] = tags

            data["url"] = url

            return ScrapeResult(
                success=True,
                data=[data],
                total_count=1,
                scraped_count=1,
            )

        except Exception as e:
            logger.error(f"详情页抓取失败: {url}, error={e}")
            return ScrapeResult(
                success=False,
                data=[],
                total_count=1,
                scraped_count=0,
                errors=[str(e)],
            )

    # ==================== 列表+详情抓取 ====================

    async def scrape_list_with_details(
        self,
        list_url: str,
        detail_link_selector: str = "a[href]",
        max_details: Optional[int] = None,
    ) -> ScrapeResult:
        """
        抓取列表页并进入详情页抓取

        Args:
            list_url: 列表页 URL
            detail_link_selector: 详情链接选择器
            max_details: 最大详情页数量

        Returns:
            ScrapeResult: 抓取结果
        """
        # 先抓取列表
        await self.page.goto(list_url)
        list_result = await self.scrape_list()

        if not list_result.success:
            return list_result

        # 收集详情链接
        detail_links: Set[str] = set()
        items_locator = self.page.locator(
            self._detected_structure.item_selector if self._detected_structure else ".item"
        )
        count = await items_locator.count()

        for i in range(count):
            link_locator = items_locator.nth(i).locator(detail_link_selector)
            if await link_locator.count() > 0:
                href = await link_locator.first.get_attribute("href")
                if href:
                    # 转换为绝对 URL
                    if not href.startswith("http"):
                        href = self.page.url.rsplit("/", 1)[0] + "/" + href.lstrip("./")
                    detail_links.add(href)

        # 限制详情页数量
        if max_details:
            detail_links = set(list(detail_links)[:max_details])

        # 抓取详情页
        all_data = list_result.data.copy()
        errors = list(list_result.errors)

        for link in detail_links:
            detail_result = await self.scrape_detail(link)
            if detail_result.success:
                all_data.append(detail_result.data[0])
            else:
                errors.extend(detail_result.errors)

        return ScrapeResult(
            success=True,
            data=all_data,
            total_count=list_result.total_count + len(detail_links),
            scraped_count=len(all_data),
            errors=errors,
        )

    # ==================== 导出功能 ====================

    async def export_to_json(
        self,
        result: ScrapeResult,
        file_path: str,
        indent: int = 2,
    ) -> bool:
        """导出为 JSON 文件"""
        try:
            Path(file_path).parent.mkdir(parents=True, exist_ok=True)

            with open(file_path, 'w', encoding='utf-8') as f:
                json.dump({
                    "success": result.success,
                    "total_count": result.total_count,
                    "scraped_count": result.scraped_count,
                    "timestamp": result.timestamp.isoformat(),
                    "data": result.data,
                    "errors": result.errors,
                }, f, ensure_ascii=False, indent=indent)

            logger.info(f"已导出 JSON: {file_path}")
            return True

        except Exception as e:
            logger.error(f"导出 JSON 失败: {e}")
            return False

    async def export_to_csv(
        self,
        result: ScrapeResult,
        file_path: str,
    ) -> bool:
        """导出为 CSV 文件"""
        if not result.data:
            logger.warning("没有数据可导出")
            return False

        try:
            Path(file_path).parent.mkdir(parents=True, exist_ok=True)

            # 获取所有字段
            fieldnames: Set[str] = set()
            for item in result.data:
                fieldnames.update(item.keys())

            with open(file_path, 'w', encoding='utf-8', newline='') as f:
                writer = csv.DictWriter(f, fieldnames=list(fieldnames))
                writer.writeheader()
                writer.writerows(result.data)

            logger.info(f"已导出 CSV: {file_path}")
            return True

        except Exception as e:
            logger.error(f"导出 CSV 失败: {e}")
            return False


def create_smart_scraper(
    page: Page,
    config: Optional[SmartScraperConfig] = None,
) -> SmartScraperV2:
    """创建智能爬虫实例"""
    return SmartScraperV2(page, config)