"""
网页爬虫工具模块

提供HTML页面解析功能，支持：
- 远程网页获取和解析
- 本地HTML文件解析
- 标签查找（单个/批量）
- 字符编码自动检测
"""
import logging
from typing import Optional, Dict, List, Any

import requests
from bs4 import BeautifulSoup, Tag

logger = logging.getLogger(__name__)


class SpiderUtil:
    """
    网页爬虫工具类

    基于BeautifulSoup的HTML解析工具，提供远程和本地HTML解析功能。

    Example:
        >>> html = SpiderUtil.get_tag_html("https://example.com")
        >>> title = SpiderUtil.find_tag(html, "title")
    """

    @staticmethod
    def get_charset(tag_html: BeautifulSoup) -> str:
        """
        从HTML中提取字符编码

        Args:
            tag_html: BeautifulSoup解析后的HTML对象

        Returns:
            字符编码字符串，默认返回"utf-8"
        """
        try:
            meta_tags = tag_html.find_all("meta")
            for meta_tag in meta_tags:
                try:
                    tmp_str = str(meta_tag)
                    idx = tmp_str.find("charset=")
                    if idx >= 0:
                        charset_str = tmp_str[idx + len("charset="):-1]
                        idx_end = charset_str.find('"')
                        if idx_end < 0:
                            idx_end = charset_str.find(";")
                        charset = charset_str[:idx_end]
                        return charset.lower().strip()
                except Exception as e:
                    logger.warning(f"Failed to parse charset from meta tag: {e}")
        except Exception as e:
            logger.warning(f"Failed to get charset: {e}")

        logger.debug("Using default charset 'utf-8'")
        return "utf-8"

    @staticmethod
    def get_tag_html(
        url: str,
        charset: str = "utf-8",
        headers: Optional[Dict[str, str]] = None,
        timeout: int = 30
    ) -> Optional[BeautifulSoup]:
        """
        获取远程网页并解析为BeautifulSoup对象

        Args:
            url: 网页URL
            charset: 字符编码
            headers: 请求头
            timeout: 请求超时时间（秒）

        Returns:
            BeautifulSoup对象，失败时返回None
        """
        if headers is None:
            headers = {}

        try:
            response = requests.get(url, headers=headers, timeout=timeout)
            response.encoding = charset
            tag_html = BeautifulSoup(response.text, "html.parser")
            return tag_html
        except Exception as e:
            logger.warning(f"Failed to get HTML from {url}: {e}")
            return None

    @staticmethod
    def get_tag_html_local(local_file: str) -> Optional[BeautifulSoup]:
        """
        解析本地HTML文件

        Args:
            local_file: 本地文件路径

        Returns:
            BeautifulSoup对象，失败时返回None
        """
        try:
            with open(local_file, "r", encoding="utf-8") as f:
                html_text = f.read()
            return BeautifulSoup(html_text, "html.parser")
        except Exception as e:
            logger.warning(f"Failed to parse local file {local_file}: {e}")
            return None

    @staticmethod
    def find_tag(
        tag_html: Optional[BeautifulSoup],
        tag_name: str,
        tag_class: Optional[str] = None,
        tag_id: Optional[str] = None
    ) -> Optional[Tag]:
        """
        查找第一个满足条件的标签

        Args:
            tag_html: BeautifulSoup对象
            tag_name: 标签名称
            tag_class: CSS类名（可选）
            tag_id: 元素ID（可选）

        Returns:
            找到的Tag对象，未找到时返回None
        """
        if tag_html is None:
            return None

        if tag_class and tag_id:
            return tag_html.find(tag_name, class_=tag_class, id=tag_id)
        elif tag_class:
            return tag_html.find(tag_name, class_=tag_class)
        elif tag_id:
            return tag_html.find(tag_name, id=tag_id)
        else:
            return tag_html.find(tag_name)

    @staticmethod
    def find_tag_all(
        tag_html: Optional[BeautifulSoup],
        tag_name: str,
        tag_class: Optional[str] = None,
        tag_id: Optional[str] = None
    ) -> Optional[List[Tag]]:
        """
        查找所有满足条件的标签

        Args:
            tag_html: BeautifulSoup对象
            tag_name: 标签名称
            tag_class: CSS类名（可选）
            tag_id: 元素ID（可选）

        Returns:
            Tag对象列表，tag_html为None时返回None
        """
        if tag_html is None:
            return None

        if tag_class and tag_id:
            return tag_html.find_all(tag_name, class_=tag_class, id=tag_id)
        elif tag_class:
            return tag_html.find_all(tag_name, class_=tag_class)
        elif tag_id:
            return tag_html.find_all(tag_name, id=tag_id)
        else:
            return tag_html.find_all(tag_name)


if __name__ == "__main__":
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
    }
    test_url = "https://www.baidu.com"
    html = SpiderUtil.get_tag_html(test_url, headers=headers)
    if html:
        title = SpiderUtil.find_tag(html, "title")
        print(f"Title: {title.text if title else 'Not found'}")
