"""Crawler for llms.txt documentation sources using Crawlee."""

import hashlib
import logging
import shutil
from dataclasses import dataclass, field
from pathlib import Path

from crawlee import Request
from crawlee.configuration import Configuration
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext

from sensei.database.storage import save_document
from sensei.tome.parser import extract_path, is_same_domain, parse_llms_txt_links
from sensei.types import Domain, SaveResult, Success, TransientError

logger = logging.getLogger(__name__)

# Crawler configuration
REQUEST_TIMEOUT_SECS = 30
MAX_REQUESTS_PER_CRAWL = 500
MIN_CONCURRENCY = 1
MAX_CONCURRENCY = 10  # Be respectful to target servers

# Content types we accept (llms.txt standard uses markdown)
# Many servers serve .md files as text/plain, so we accept both
ALLOWED_CONTENT_TYPES = frozenset({"text/markdown", "text/plain", "text/x-markdown"})


@dataclass
class IngestResult:
	"""Result of ingesting a domain's llms.txt documentation."""

	domain: str
	documents_added: int = 0
	documents_updated: int = 0
	documents_skipped: int = 0
	errors: list[str] = field(default_factory=list)


def content_hash(content: str) -> str:
	"""Generate a hash for content change detection."""
	return hashlib.sha256(content.encode()).hexdigest()[:16]


def is_markdown_content(content_type: str | None) -> bool:
	"""Check if content type indicates markdown or plain text.

	Args:
	    content_type: The Content-Type header value (may include charset)

	Returns:
	    True if content type is acceptable for markdown content
	"""
	if not content_type:
		return True  # Accept if no content type (trust the URL)
	# Extract media type (ignore charset and other parameters)
	media_type = content_type.split(";")[0].strip().lower()
	return media_type in ALLOWED_CONTENT_TYPES


async def ingest_domain(domain: str, max_depth: int = 3) -> Success[IngestResult]:
	"""Ingest documentation from a domain's llms.txt file.

	Fetches the llms.txt file from https://{domain}/llms.txt, parses it to
	extract links, then crawls all same-domain linked documents up to max_depth.

	Args:
	    domain: The domain to crawl (e.g., "react.dev"). Can be a full URL -
	            will be normalized automatically.
	    max_depth: Maximum link depth to follow. 0 means only fetch llms.txt
	            itself (no linked documents). 1 means fetch llms.txt and its
	            direct links. Default is 3.

	Returns:
	    Success[IngestResult] with counts of documents processed

	Raises:
	    TransientError: If the crawl fails due to network issues
	"""
	# Normalize domain (handles full URLs, www prefix, ports, etc.)
	normalized_domain = Domain(domain).value
	result = IngestResult(domain=normalized_domain)
	llms_txt_url = f"https://{normalized_domain}/llms.txt"

	crawler = HttpCrawler(
		max_requests_per_crawl=MAX_REQUESTS_PER_CRAWL,
		request_handler_timeout=REQUEST_TIMEOUT_SECS,
		min_concurrency=MIN_CONCURRENCY,
		max_concurrency=MAX_CONCURRENCY,
	)

	@crawler.router.default_handler
	async def handle_document(context: HttpCrawlingContext) -> None:
		"""Handle any markdown document (llms.txt or linked docs)."""
		# Use loaded_url (after redirects) for accurate domain matching
		url = context.request.loaded_url or context.request.url
		current_depth = context.request.user_data.get("depth", 0)
		logger.info(f"Processing document (depth={current_depth}): {url}")

		# Check content type before reading body
		content_type = context.http_response.headers.get("content-type")
		if not is_markdown_content(content_type):
			logger.warning(f"Skipping non-markdown content type '{content_type}': {url}")
			result.errors.append(f"Invalid content type '{content_type}': {url}")
			return

		try:
			content = (await context.http_response.read()).decode("utf-8")
		except UnicodeDecodeError:
			logger.warning(f"Skipping non-text content: {url}")
			result.errors.append(f"Non-text content: {url}")
			return

		# Save the document
		save_result = await save_document(
			domain=normalized_domain,
			url=url,
			path=extract_path(url),
			content=content,
			content_hash=content_hash(content),
			depth=current_depth,
		)
		match save_result:
			case SaveResult.INSERTED:
				result.documents_added += 1
			case SaveResult.UPDATED:
				result.documents_updated += 1
			case SaveResult.SKIPPED:
				result.documents_skipped += 1

		# Parse links and enqueue same-domain ones if within depth limit
		if current_depth < max_depth:
			all_links = parse_llms_txt_links(content, url)
			same_domain_links = [link for link in all_links if is_same_domain(url, link)]
			if same_domain_links:
				logger.info(f"Found {len(all_links)} links, {len(same_domain_links)} same-domain, enqueueing")
				requests = [
					Request.from_url(link, user_data={"depth": current_depth + 1}) for link in same_domain_links
				]
				await context.add_requests(requests)

	# Start crawl with the llms.txt file (depth=0)
	try:
		await crawler.run([Request.from_url(llms_txt_url, user_data={"depth": 0})])
	except Exception as e:
		logger.error(f"Crawl failed for {normalized_domain}: {e}")
		raise TransientError(f"Crawl failed for {normalized_domain}: {e}") from e
	finally:
		# Clean up Crawlee's storage directory to avoid disk accumulation
		_cleanup_crawlee_storage()

	logger.info(
		f"Ingest complete for {normalized_domain}: "
		f"added={result.documents_added}, "
		f"updated={result.documents_updated}, "
		f"skipped={result.documents_skipped}, "
		f"errors={len(result.errors)}"
	)
	return Success(result)


def _cleanup_crawlee_storage() -> None:
	"""Remove Crawlee's storage directory after crawl completes."""
	config = Configuration.get_global_configuration()
	storage_dir = Path(config.storage_dir)
	if storage_dir.exists():
		try:
			shutil.rmtree(storage_dir)
			logger.debug(f"Cleaned up Crawlee storage directory: {storage_dir}")
		except OSError as e:
			logger.warning(f"Failed to clean up Crawlee storage: {e}")
