import asyncio
import time
import json
from typing import List, Dict, Any, Optional
from datetime import datetime
from urllib.parse import urlparse
import pandas as pd
from rich.console import Console
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeRemainingColumn
from crawl4ai import AsyncWebCrawler
from ..models.company import CompanyData, CrawlJob, CrawlStatus
from ..extractors.company_extractor import SwedishCompanyExtractor


class BatchCrawler:
    """Batch-crawler med kö-system och rate limiting"""
    
    def __init__(
        self,
        max_concurrent: int = 5,
        rate_limit_delay: float = 2.0,
        max_retries: int = 3,
        retry_backoff_factor: float = 2.0
    ):
        self.max_concurrent = max_concurrent
        self.rate_limit_delay = rate_limit_delay
        self.max_retries = max_retries
        self.retry_backoff_factor = retry_backoff_factor
        
        self.console = Console()
        self.extractor = SwedishCompanyExtractor()
        self.domain_next_allowed_at: Dict[str, float] = {}
        self.domain_locks: Dict[str, asyncio.Lock] = {}
        self.semaphore = asyncio.Semaphore(max_concurrent)
        
        # Statistik
        self.stats = {
            'total_urls': 0,
            'processed_urls': 0,
            'successful_urls': 0,
            'failed_urls': 0,
            'retries': 0
        }
    
    async def crawl_from_file(self, file_path: str, output_format: str = 'json') -> List[CompanyData]:
        """Läs URLs från fil och crawla dem"""
        urls = await self._load_urls_from_file(file_path)
        
        if not urls:
            self.console.print("[red]Inga URLs hittades i filen[/red]")
            return []
        
        self.console.print(f"[green]Laddade {len(urls)} URLs från {file_path}[/green]")
        
        return await self.crawl_urls(urls)
    
    async def crawl_urls(self, urls: List[str]) -> List[CompanyData]:
        """Crawla en lista av URLs med batch-processing"""
        self.stats = {
            'total_urls': 0,
            'processed_urls': 0,
            'successful_urls': 0,
            'failed_urls': 0,
            'retries': 0
        }
        self.stats['total_urls'] = len(urls)
        
        # Skapa throttlers för varje domän
        await self._setup_domain_throttlers(urls)
        
        # Skapa crawl-jobb
        job = CrawlJob(
            job_id=f"job_{int(time.time())}",
            urls=[url for url in urls],  # Konvertera till HttpUrl i modellen
            status=CrawlStatus.IN_PROGRESS,
            started_at=datetime.now(),
            total_urls=len(urls)
        )
        
        self.console.print(f"[blue]Startar crawl-jobb {job.job_id} med {len(urls)} URLs[/blue]")
        
        results = []
        
        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            BarColumn(),
            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
            TimeRemainingColumn(),
            console=self.console
        ) as progress:
            
            task = progress.add_task(f"Crawlar {len(urls)} URLs...", total=len(urls))
            
            # Skapa tasks för varje URL
            tasks = []
            for url in urls:
                task_coroutine = self._crawl_single_url_with_retry(url, progress, task)
                tasks.append(task_coroutine)
            
            # Kör alla tasks med semaphore för att begränsa concurrency
            completed_tasks = await asyncio.gather(*tasks, return_exceptions=True)
            
            # Processera resultat
            for result in completed_tasks:
                if isinstance(result, Exception):
                    self.console.print(f"[red]Task exception: {result}[/red]")
                    self.stats['failed_urls'] += 1
                elif result:
                    if result.success:
                        results.append(result.data)
                        self.stats['successful_urls'] += 1
                    else:
                        self.console.print(f"[red]Failed to crawl: {result.error_message}[/red]")
                        self.stats['failed_urls'] += 1
                
                self.stats['processed_urls'] += 1
                progress.advance(task)
        
        # Uppdatera jobb-status
        job.completed_at = datetime.now()
        job.status = CrawlStatus.COMPLETED
        job.processed_urls = self.stats['processed_urls']
        job.successful_urls = self.stats['successful_urls']
        job.failed_urls = self.stats['failed_urls']
        
        # Visa statistik
        self._print_statistics(job)
        
        return results
    
    async def _crawl_single_url_with_retry(self, url: str, progress: Progress, task_id: int) -> Optional[Any]:
        """Crawla enstaka URL med retry-logik"""
        domain = urlparse(url).netloc
        
        for attempt in range(self.max_retries + 1):
            try:
                # Använd semaphore för att begränsa concurrency
                async with self.semaphore:
                    # Rate limiting per domän
                    await self._acquire_domain_slot(domain)
                    
                    # Uppdatera progress
                    progress.update(task_id, description=f"Crawlar {url[:50]}...")
                    
                    # Skapa crawler för denna URL
                    async with AsyncWebCrawler(verbose=False) as crawler:
                        result = await self.extractor.extract_from_url(url, crawler)
                        
                        if result.success:
                            return result
                        else:
                            if attempt < self.max_retries:
                                wait_time = self.retry_backoff_factor ** attempt
                                self.console.print(f"[yellow]Retry {attempt + 1}/{self.max_retries} för {url} om {wait_time}s[/yellow]")
                                await asyncio.sleep(wait_time)
                                self.stats['retries'] += 1
                            else:
                                return result
            
            except Exception as e:
                if attempt < self.max_retries:
                    wait_time = self.retry_backoff_factor ** attempt
                    self.console.print(f"[yellow]Exception retry {attempt + 1}/{self.max_retries} för {url}: {e}[/yellow]")
                    await asyncio.sleep(wait_time)
                    self.stats['retries'] += 1
                else:
                    return type('Result', (), {'success': False, 'error_message': str(e)})()
        
        return None

    async def _acquire_domain_slot(self, domain: str) -> None:
        """Säkerställ minst `rate_limit_delay` sekunder mellan requests till samma domän."""
        if domain not in self.domain_locks:
            self.domain_locks[domain] = asyncio.Lock()

        async with self.domain_locks[domain]:
            now = time.monotonic()
            next_allowed_at = self.domain_next_allowed_at.get(domain, 0.0)
            wait_time = max(0.0, next_allowed_at - now)

            if wait_time > 0:
                await asyncio.sleep(wait_time)

            self.domain_next_allowed_at[domain] = time.monotonic() + self.rate_limit_delay
    
    async def _setup_domain_throttlers(self, urls: List[str]):
        """Förbered inre rate-limit state för varje unik domän."""
        domains = set()
        for url in urls:
            domain = urlparse(url).netloc
            if domain:
                domains.add(domain)
        
        self.console.print(f"[blue]Setup rate limiting för {len(domains)} domäner[/blue]")
    
    async def _load_urls_from_file(self, file_path: str) -> List[str]:
        """Ladda URLs från CSV eller JSON fil"""
        urls = []
        
        try:
            if file_path.endswith('.csv'):
                df = pd.read_csv(file_path)
                # Försök hitta kolumn med URLs
                url_columns = ['url', 'URL', 'website', 'Website', 'link', 'Link']
                for col in url_columns:
                    if col in df.columns:
                        urls = df[col].dropna().tolist()
                        break
                
                if not urls:
                    self.console.print("[red]Ingen URL-kolumn hittad i CSV-filen[/red]")
                    return []
            
            elif file_path.endswith('.json'):
                with open(file_path, 'r', encoding='utf-8') as f:
                    data = json.load(f)
                    
                    if isinstance(data, list):
                        # Lista med URLs eller objekt med URL-fält
                        for item in data:
                            if isinstance(item, str):
                                urls.append(item)
                            elif isinstance(item, dict):
                                url_fields = ['url', 'URL', 'website', 'Website', 'link', 'Link']
                                for field in url_fields:
                                    if field in item and item[field]:
                                        urls.append(item[field])
                                        break
                    
                    elif isinstance(data, dict):
                        # Dict med URL-lista
                        if 'urls' in data:
                            urls = data['urls']
                        else:
                            urls = list(data.values())
            
            else:
                self.console.print(f"[red]Filformat stöds inte: {file_path}[/red]")
                return []
            
            # Filtrera och validera URLs
            valid_urls = []
            for url in urls:
                if isinstance(url, str) and url.strip():
                    url = url.strip()
                    if not url.startswith(('http://', 'https://')):
                        url = 'https://' + url
                    valid_urls.append(url)
            
            deduplicated_urls = list(dict.fromkeys(valid_urls))
            return deduplicated_urls
        
        except Exception as e:
            self.console.print(f"[red]Fel vid inläsning av fil {file_path}: {e}[/red]")
            return []
    
    def _print_statistics(self, job: CrawlJob):
        """Skriv ut statistik för crawl-jobb"""
        duration = None
        if job.started_at and job.completed_at:
            duration = (job.completed_at - job.started_at).total_seconds()
        
        self.console.print("\n[bold green]Crawl Jobb Slutförd![/bold green]")
        self.console.print(f"Jobb ID: {job.job_id}")
        self.console.print(f"Total URLs: {job.total_urls}")
        self.console.print(f"Lyckade: [green]{job.successful_urls}[/green]")
        self.console.print(f"Misslyckade: [red]{job.failed_urls}[/red]")
        
        if duration:
            self.console.print(f"Tid: {duration:.2f} sekunder")
            if job.successful_urls > 0:
                avg_time = duration / job.successful_urls
                self.console.print(f"Genomsnitt per URL: {avg_time:.2f} sekunder")
        
        self.console.print(f"Retries: {self.stats['retries']}")
        
        success_rate = (job.successful_urls / job.total_urls * 100) if job.total_urls > 0 else 0
        self.console.print(f"Success rate: {success_rate:.1f}%")
    
    async def export_results(self, results: List[CompanyData], output_path: str, format: str = 'json'):
        """Exportera resultat till fil"""
        if not results:
            self.console.print("[yellow]Inga resultat att exportera[/yellow]")
            return
        
        try:
            if format.lower() == 'json':
                data = [result.model_dump(mode='json') for result in results]
                with open(output_path, 'w', encoding='utf-8') as f:
                    json.dump(data, f, ensure_ascii=False, indent=2, default=str)
            
            elif format.lower() == 'csv':
                # Konvertera till DataFrame
                data = []
                for result in results:
                    row = result.model_dump(mode='json')
                    # Platta till nested objekt
                    flat_row = {}
                    for key, value in row.items():
                        if isinstance(value, dict):
                            for sub_key, sub_value in value.items():
                                flat_row[f"{key}_{sub_key}"] = sub_value
                        else:
                            flat_row[key] = value
                    data.append(flat_row)
                
                df = pd.DataFrame(data)
                df.to_csv(output_path, index=False, encoding='utf-8')
            
            else:
                self.console.print(f"[red]Exportformat {format} stöds inte[/red]")
                return
            
            self.console.print(f"[green]Resultat exporterat till {output_path}[/green]")
        
        except Exception as e:
            self.console.print(f"[red]Exportfel: {e}[/red]")
