"""
Collector Agent

Autonomous data collection agent with multi-tier scraping strategy:
- Tier 1: HTTP/API (httpx)
- Tier 2: JS Rendering (Crawl4AI)
- Tier 3: Complex Scraping (Firecrawl)
- Tier 4: Browser Automation (Playwright)
"""

from __future__ import annotations

import asyncio
import logging
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from typing import Any

from ..config import Config, get_config

logger = logging.getLogger(__name__)


class CollectorTier(str, Enum):
    """Data collection tiers."""
    TIER_1_HTTP = "http"      # httpx - REST APIs, static pages
    TIER_2_JS = "js"          # Crawl4AI - JS rendering
    TIER_3_COMPLEX = "complex"  # Firecrawl - complex scraping
    TIER_4_BROWSER = "browser"  # Playwright - browser automation


class CollectionStatus(str, Enum):
    """Collection job status."""
    PENDING = "pending"
    RUNNING = "running"
    COMPLETED = "completed"
    FAILED = "failed"
    RETRYING = "retrying"


@dataclass
class CollectionResult:
    """Result of a collection job."""
    success: bool
    source: str
    tier: CollectorTier
    records_collected: int = 0
    duration_ms: float = 0.0
    error: str | None = None
    data: list[dict[str, Any]] = field(default_factory=list)
    metadata: dict[str, Any] = field(default_factory=dict)
    collected_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))


@dataclass
class CollectionJob:
    """A collection job definition."""
    source: str
    url: str | None = None
    tier: CollectorTier = CollectorTier.TIER_1_HTTP
    params: dict[str, Any] = field(default_factory=dict)
    headers: dict[str, str] = field(default_factory=dict)
    schedule: str | None = None  # Cron expression
    retry_count: int = 3
    timeout: float = 60.0


# Tool definitions for Collector Agent
COLLECTOR_TOOLS = [
    {
        "name": "collect_http",
        "description": "Collect data via HTTP/API (Tier 1). Use for REST APIs and static pages.",
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "URL to collect from"},
                "method": {"type": "string", "enum": ["GET", "POST"], "default": "GET"},
                "headers": {"type": "object", "description": "HTTP headers"},
                "params": {"type": "object", "description": "Query parameters"},
                "body": {"type": "object", "description": "Request body for POST"},
                "timeout": {"type": "number", "description": "Timeout in seconds", "default": 30},
            },
            "required": ["url"],
        },
    },
    {
        "name": "collect_js",
        "description": "Collect data from JS-rendered pages (Tier 2). Use for dynamic content.",
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "URL to collect from"},
                "wait_for": {"type": "string", "description": "CSS selector to wait for"},
                "timeout": {"type": "number", "description": "Timeout in seconds", "default": 60},
                "extract_schema": {"type": "object", "description": "Data extraction schema"},
            },
            "required": ["url"],
        },
    },
    {
        "name": "collect_complex",
        "description": "Collect data using Firecrawl (Tier 3). Use for complex scraping with anti-bot handling.",
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "URL to collect from"},
                "formats": {"type": "array", "items": {"type": "string"}, "default": ["markdown"]},
                "wait_for": {"type": "number", "description": "Wait time in ms"},
                "extract_schema": {"type": "object", "description": "LLM extraction schema"},
            },
            "required": ["url"],
        },
    },
    {
        "name": "collect_browser",
        "description": "Collect data using browser automation (Tier 4). Use for anti-bot sites like Amazon, LinkedIn.",
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "URL to collect from"},
                "actions": {"type": "array", "items": {"type": "object"}, "description": "Browser actions to perform"},
                "headless": {"type": "boolean", "default": True},
                "timeout": {"type": "number", "description": "Timeout in seconds", "default": 120},
            },
            "required": ["url"],
        },
    },
    {
        "name": "detect_tier",
        "description": "Auto-detect the appropriate collection tier for a URL.",
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "URL to analyze"},
            },
            "required": ["url"],
        },
    },
    {
        "name": "smart_collect",
        "description": "Automatically collect data using the best tier for the URL.",
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "URL to collect from"},
                "extract_schema": {"type": "object", "description": "Data extraction schema"},
                "max_tier": {"type": "string", "enum": ["http", "js", "complex", "browser"], "description": "Maximum tier to use"},
            },
            "required": ["url"],
        },
    },
    {
        "name": "batch_collect",
        "description": "Collect data from multiple URLs in parallel.",
        "input_schema": {
            "type": "object",
            "properties": {
                "urls": {"type": "array", "items": {"type": "string"}, "description": "URLs to collect from"},
                "tier": {"type": "string", "enum": ["http", "js", "complex", "browser"], "default": "http"},
                "concurrency": {"type": "integer", "default": 5, "description": "Max concurrent requests"},
            },
            "required": ["urls"],
        },
    },
    {
        "name": "get_collection_stats",
        "description": "Get statistics about recent collection jobs.",
        "input_schema": {
            "type": "object",
            "properties": {
                "hours": {"type": "integer", "default": 24, "description": "Hours to look back"},
            },
        },
    },
]


class CollectorAgent:
    """
    Autonomous Data Collection Agent.

    Implements multi-tier collection strategy with automatic tier selection,
    retry logic, and rate limiting.
    """

    # Domains that require browser automation (Tier 4)
    TIER_4_DOMAINS = [
        "amazon.com", "amazon.co.uk", "amazon.de",
        "linkedin.com",
        "facebook.com", "instagram.com",
        "glassdoor.com", "indeed.com",
    ]

    # Domains that require complex scraping (Tier 3)
    TIER_3_DOMAINS = [
        "reddit.com",
        "twitter.com", "x.com",
    ]

    def __init__(
        self,
        config: Config | None = None,
        tools: list[dict] | None = None,
    ):
        """
        Initialize Collector Agent.

        Args:
            config: Configuration object
            tools: Custom tool definitions (defaults to COLLECTOR_TOOLS)
        """
        self.config = config or get_config()
        self.tools = tools or COLLECTOR_TOOLS
        self._collection_history: list[CollectionResult] = []
        self._rate_limits: dict[str, float] = {}
        logger.info("CollectorAgent initialized")

    def detect_tier(self, url: str) -> CollectorTier:
        """
        Auto-detect the appropriate collection tier for a URL.

        Args:
            url: URL to analyze

        Returns:
            Recommended collection tier
        """
        from urllib.parse import urlparse

        domain = urlparse(url).netloc.lower()

        # Check Tier 4 domains
        for t4_domain in self.TIER_4_DOMAINS:
            if t4_domain in domain:
                return CollectorTier.TIER_4_BROWSER

        # Check Tier 3 domains
        for t3_domain in self.TIER_3_DOMAINS:
            if t3_domain in domain:
                return CollectorTier.TIER_3_COMPLEX

        # Check if likely API endpoint
        if "/api/" in url or url.endswith(".json"):
            return CollectorTier.TIER_1_HTTP

        # Default to Tier 1
        return CollectorTier.TIER_1_HTTP

    async def collect_http(
        self,
        url: str,
        method: str = "GET",
        headers: dict[str, str] | None = None,
        params: dict[str, Any] | None = None,
        body: dict[str, Any] | None = None,
        timeout: float = 30.0,
    ) -> CollectionResult:
        """
        Collect data via HTTP/API (Tier 1).

        Args:
            url: URL to collect from
            method: HTTP method
            headers: Request headers
            params: Query parameters
            body: Request body for POST
            timeout: Request timeout
        """
        import time
        import httpx

        start_time = time.time()

        try:
            async with httpx.AsyncClient() as client:
                if method.upper() == "GET":
                    response = await client.get(
                        url,
                        headers=headers or {},
                        params=params or {},
                        timeout=timeout,
                    )
                else:
                    response = await client.post(
                        url,
                        headers=headers or {},
                        params=params or {},
                        json=body,
                        timeout=timeout,
                    )

                response.raise_for_status()

                # Try to parse as JSON
                try:
                    data = response.json()
                    if isinstance(data, list):
                        records = data
                    elif isinstance(data, dict):
                        records = [data]
                    else:
                        records = [{"value": data}]
                except Exception:
                    records = [{"content": response.text}]

                duration_ms = (time.time() - start_time) * 1000

                result = CollectionResult(
                    success=True,
                    source=url,
                    tier=CollectorTier.TIER_1_HTTP,
                    records_collected=len(records),
                    duration_ms=duration_ms,
                    data=records,
                    metadata={
                        "status_code": response.status_code,
                        "content_type": response.headers.get("content-type"),
                    },
                )
                self._collection_history.append(result)
                return result

        except Exception as e:
            duration_ms = (time.time() - start_time) * 1000
            result = CollectionResult(
                success=False,
                source=url,
                tier=CollectorTier.TIER_1_HTTP,
                duration_ms=duration_ms,
                error=str(e),
            )
            self._collection_history.append(result)
            return result

    async def collect_js(
        self,
        url: str,
        wait_for: str | None = None,
        timeout: float = 60.0,
        extract_schema: dict | None = None,
    ) -> CollectionResult:
        """
        Collect data from JS-rendered pages (Tier 2).

        Note: Requires Crawl4AI to be available.
        """
        import time

        start_time = time.time()

        try:
            # Try to use Crawl4AI
            try:
                from crawl4ai import AsyncWebCrawler

                async with AsyncWebCrawler() as crawler:
                    result = await crawler.arun(url)
                    content = result.markdown

                    records = [{"content": content, "url": url}]
                    duration_ms = (time.time() - start_time) * 1000

                    return CollectionResult(
                        success=True,
                        source=url,
                        tier=CollectorTier.TIER_2_JS,
                        records_collected=1,
                        duration_ms=duration_ms,
                        data=records,
                    )

            except ImportError:
                # Fallback to HTTP if Crawl4AI not available
                logger.warning("Crawl4AI not available, falling back to HTTP")
                return await self.collect_http(url, timeout=timeout)

        except Exception as e:
            duration_ms = (time.time() - start_time) * 1000
            return CollectionResult(
                success=False,
                source=url,
                tier=CollectorTier.TIER_2_JS,
                duration_ms=duration_ms,
                error=str(e),
            )

    async def collect_complex(
        self,
        url: str,
        formats: list[str] | None = None,
        wait_for: int | None = None,
        extract_schema: dict | None = None,
    ) -> CollectionResult:
        """
        Collect data using Firecrawl (Tier 3).

        Note: Requires Firecrawl API key.
        """
        import time
        import os

        start_time = time.time()

        try:
            api_key = os.environ.get("FIRECRAWL_API_KEY")
            if not api_key:
                # Fallback to JS tier
                logger.warning("Firecrawl API key not set, falling back to JS tier")
                return await self.collect_js(url)

            import httpx

            async with httpx.AsyncClient() as client:
                response = await client.post(
                    "https://api.firecrawl.dev/v1/scrape",
                    headers={"Authorization": f"Bearer {api_key}"},
                    json={
                        "url": url,
                        "formats": formats or ["markdown"],
                        "waitFor": wait_for,
                    },
                    timeout=120.0,
                )
                response.raise_for_status()
                data = response.json()

                records = [data.get("data", {})]
                duration_ms = (time.time() - start_time) * 1000

                return CollectionResult(
                    success=True,
                    source=url,
                    tier=CollectorTier.TIER_3_COMPLEX,
                    records_collected=1,
                    duration_ms=duration_ms,
                    data=records,
                )

        except Exception as e:
            duration_ms = (time.time() - start_time) * 1000
            return CollectionResult(
                success=False,
                source=url,
                tier=CollectorTier.TIER_3_COMPLEX,
                duration_ms=duration_ms,
                error=str(e),
            )

    async def collect_browser(
        self,
        url: str,
        actions: list[dict] | None = None,
        headless: bool = True,
        timeout: float = 120.0,
    ) -> CollectionResult:
        """
        Collect data using browser automation (Tier 4).

        Note: Requires Playwright to be available.
        """
        import time

        start_time = time.time()

        try:
            from playwright.async_api import async_playwright

            async with async_playwright() as p:
                browser = await p.chromium.launch(headless=headless)
                page = await browser.new_page()

                await page.goto(url, timeout=int(timeout * 1000))

                # Execute custom actions
                if actions:
                    for action in actions:
                        action_type = action.get("type")
                        if action_type == "click":
                            await page.click(action.get("selector"))
                        elif action_type == "fill":
                            await page.fill(action.get("selector"), action.get("value", ""))
                        elif action_type == "wait":
                            await page.wait_for_selector(action.get("selector"))
                        elif action_type == "scroll":
                            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")

                content = await page.content()
                await browser.close()

                records = [{"html": content, "url": url}]
                duration_ms = (time.time() - start_time) * 1000

                return CollectionResult(
                    success=True,
                    source=url,
                    tier=CollectorTier.TIER_4_BROWSER,
                    records_collected=1,
                    duration_ms=duration_ms,
                    data=records,
                )

        except ImportError:
            logger.warning("Playwright not available, falling back to complex tier")
            return await self.collect_complex(url)

        except Exception as e:
            duration_ms = (time.time() - start_time) * 1000
            return CollectionResult(
                success=False,
                source=url,
                tier=CollectorTier.TIER_4_BROWSER,
                duration_ms=duration_ms,
                error=str(e),
            )

    async def smart_collect(
        self,
        url: str,
        extract_schema: dict | None = None,
        max_tier: CollectorTier | None = None,
    ) -> CollectionResult:
        """
        Automatically collect data using the best tier for the URL.

        Args:
            url: URL to collect from
            extract_schema: Optional extraction schema
            max_tier: Maximum tier to use (for cost control)
        """
        detected_tier = self.detect_tier(url)

        # Respect max_tier if set
        if max_tier:
            tier_order = [
                CollectorTier.TIER_1_HTTP,
                CollectorTier.TIER_2_JS,
                CollectorTier.TIER_3_COMPLEX,
                CollectorTier.TIER_4_BROWSER,
            ]
            detected_idx = tier_order.index(detected_tier)
            max_idx = tier_order.index(max_tier)
            if detected_idx > max_idx:
                detected_tier = max_tier

        # Execute collection based on tier
        if detected_tier == CollectorTier.TIER_1_HTTP:
            return await self.collect_http(url)
        elif detected_tier == CollectorTier.TIER_2_JS:
            return await self.collect_js(url, extract_schema=extract_schema)
        elif detected_tier == CollectorTier.TIER_3_COMPLEX:
            return await self.collect_complex(url, extract_schema=extract_schema)
        else:
            return await self.collect_browser(url)

    async def batch_collect(
        self,
        urls: list[str],
        tier: CollectorTier = CollectorTier.TIER_1_HTTP,
        concurrency: int = 5,
    ) -> list[CollectionResult]:
        """
        Collect data from multiple URLs in parallel.

        Args:
            urls: URLs to collect from
            tier: Collection tier to use
            concurrency: Maximum concurrent requests
        """
        semaphore = asyncio.Semaphore(concurrency)

        async def collect_with_limit(url: str) -> CollectionResult:
            async with semaphore:
                if tier == CollectorTier.TIER_1_HTTP:
                    return await self.collect_http(url)
                elif tier == CollectorTier.TIER_2_JS:
                    return await self.collect_js(url)
                elif tier == CollectorTier.TIER_3_COMPLEX:
                    return await self.collect_complex(url)
                else:
                    return await self.collect_browser(url)

        results = await asyncio.gather(*[collect_with_limit(url) for url in urls])
        return list(results)

    def get_collection_stats(self, hours: int = 24) -> dict[str, Any]:
        """
        Get statistics about recent collection jobs.

        Args:
            hours: Hours to look back
        """
        from datetime import timedelta

        cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
        recent = [r for r in self._collection_history if r.collected_at >= cutoff]

        if not recent:
            return {
                "total_jobs": 0,
                "success_rate": 0.0,
                "by_tier": {},
            }

        successful = sum(1 for r in recent if r.success)

        by_tier: dict[str, dict] = {}
        for tier in CollectorTier:
            tier_results = [r for r in recent if r.tier == tier]
            if tier_results:
                by_tier[tier.value] = {
                    "count": len(tier_results),
                    "success": sum(1 for r in tier_results if r.success),
                    "avg_duration_ms": sum(r.duration_ms for r in tier_results) / len(tier_results),
                    "total_records": sum(r.records_collected for r in tier_results),
                }

        return {
            "total_jobs": len(recent),
            "successful": successful,
            "failed": len(recent) - successful,
            "success_rate": successful / len(recent) if recent else 0.0,
            "total_records": sum(r.records_collected for r in recent),
            "avg_duration_ms": sum(r.duration_ms for r in recent) / len(recent),
            "by_tier": by_tier,
        }

    async def handle_tool_call(self, tool_name: str, tool_input: dict[str, Any]) -> dict[str, Any]:
        """Handle a tool call from the LLM."""
        if tool_name == "collect_http":
            result = await self.collect_http(**tool_input)
        elif tool_name == "collect_js":
            result = await self.collect_js(**tool_input)
        elif tool_name == "collect_complex":
            result = await self.collect_complex(**tool_input)
        elif tool_name == "collect_browser":
            result = await self.collect_browser(**tool_input)
        elif tool_name == "detect_tier":
            tier = self.detect_tier(tool_input["url"])
            return {"tier": tier.value, "url": tool_input["url"]}
        elif tool_name == "smart_collect":
            max_tier = None
            if "max_tier" in tool_input:
                max_tier = CollectorTier(tool_input["max_tier"])
            result = await self.smart_collect(
                tool_input["url"],
                tool_input.get("extract_schema"),
                max_tier,
            )
        elif tool_name == "batch_collect":
            tier = CollectorTier(tool_input.get("tier", "http"))
            results = await self.batch_collect(
                tool_input["urls"],
                tier,
                tool_input.get("concurrency", 5),
            )
            return {
                "total": len(results),
                "successful": sum(1 for r in results if r.success),
                "results": [
                    {
                        "url": r.source,
                        "success": r.success,
                        "records": r.records_collected,
                        "error": r.error,
                    }
                    for r in results
                ],
            }
        elif tool_name == "get_collection_stats":
            return self.get_collection_stats(tool_input.get("hours", 24))
        else:
            return {"error": f"Unknown tool: {tool_name}"}

        # Convert CollectionResult to dict
        return {
            "success": result.success,
            "source": result.source,
            "tier": result.tier.value,
            "records_collected": result.records_collected,
            "duration_ms": result.duration_ms,
            "error": result.error,
            "data": result.data[:10] if result.data else [],  # Limit data in response
        }


__all__ = [
    "CollectorAgent",
    "CollectorTier",
    "CollectionStatus",
    "CollectionResult",
    "CollectionJob",
    "COLLECTOR_TOOLS",
]
