"""
Data ingestion service for processing webhook payloads and normalizing fields.
"""

import csv
import io
import json
import logging
import re
from typing import Dict, Any, List, Optional, Tuple
from datetime import datetime
from enum import Enum
import uuid

from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
import redis.asyncio as redis

from fmatch.saas.database import get_async_session
from fmatch.saas.model_defs.webhook_models import InboundWebhook

logger = logging.getLogger(__name__)


class DataFormat(Enum):
    """Supported data formats for ingestion."""

    JSON = "json"
    CSV = "csv"
    NDJSON = "ndjson"
    XML = "xml"
    FORM = "form"
    UNKNOWN = "unknown"


class FieldMapping:
    """Common field mappings for normalization."""

    # Company field variations
    COMPANY_FIELDS = [
        "company",
        "company_name",
        "companyname",
        "organization",
        "org",
        "business",
        "business_name",
        "account",
        "account_name",
        "employer",
        "firm",
        "enterprise",
        "corporation",
        "corp",
    ]

    # Contact name fields
    FIRST_NAME_FIELDS = [
        "first_name",
        "firstname",
        "fname",
        "given_name",
        "givenname",
        "first",
        "name_first",
        "forename",
    ]

    LAST_NAME_FIELDS = [
        "last_name",
        "lastname",
        "lname",
        "surname",
        "family_name",
        "familyname",
        "last",
        "name_last",
    ]

    FULL_NAME_FIELDS = [
        "name",
        "full_name",
        "fullname",
        "display_name",
        "displayname",
        "contact_name",
        "contactname",
        "person",
        "person_name",
    ]

    # Contact fields
    EMAIL_FIELDS = [
        "email",
        "email_address",
        "emailaddress",
        "mail",
        "e-mail",
        "contact_email",
        "business_email",
        "work_email",
    ]

    PHONE_FIELDS = [
        "phone",
        "phone_number",
        "phonenumber",
        "telephone",
        "tel",
        "mobile",
        "cell",
        "contact_phone",
        "business_phone",
        "work_phone",
    ]

    TITLE_FIELDS = [
        "title",
        "job_title",
        "jobtitle",
        "position",
        "role",
        "designation",
        "job",
        "job_position",
        "job_role",
    ]

    # Address fields
    STREET_FIELDS = [
        "street",
        "street_address",
        "streetaddress",
        "address",
        "address1",
        "address_1",
        "street1",
        "street_1",
        "line1",
    ]

    CITY_FIELDS = ["city", "town", "locality", "municipality"]

    STATE_FIELDS = ["state", "province", "region", "state_province", "stateprovince"]

    ZIP_FIELDS = [
        "zip",
        "zipcode",
        "zip_code",
        "postal",
        "postal_code",
        "postalcode",
        "postcode",
        "post_code",
    ]

    COUNTRY_FIELDS = ["country", "country_name", "countryname", "nation"]

    # Other common fields
    WEBSITE_FIELDS = [
        "website",
        "web",
        "url",
        "homepage",
        "site",
        "domain",
        "company_website",
        "company_url",
        "web_address",
    ]

    LINKEDIN_FIELDS = [
        "linkedin",
        "linkedin_url",
        "linkedinurl",
        "linkedin_profile",
        "li_url",
        "social_linkedin",
    ]

    INDUSTRY_FIELDS = [
        "industry",
        "sector",
        "vertical",
        "business_type",
        "category",
        "industry_type",
        "business_sector",
    ]

    EMPLOYEE_COUNT_FIELDS = [
        "employees",
        "employee_count",
        "employeecount",
        "size",
        "company_size",
        "companysize",
        "staff_count",
        "headcount",
        "number_of_employees",
    ]

    REVENUE_FIELDS = [
        "revenue",
        "annual_revenue",
        "annualrevenue",
        "sales",
        "turnover",
        "income",
        "revenue_amount",
    ]


class IngestionService:
    """
    Service for ingesting and normalizing data from webhooks.
    """

    def __init__(self, redis_client: Optional[redis.Redis] = None):
        self.redis_client = redis_client
        self.field_mapping = FieldMapping()
        # Note: L2A service would need proper initialization with account_id when used
        self.l2a_service = None  # Will be initialized per-request with account_id

    async def process_inbound_webhook(
        self, webhook_id: str, session: Optional[AsyncSession] = None
    ) -> Dict[str, Any]:
        """
        Process an inbound webhook and extract/normalize data.

        Args:
            webhook_id: ID of the InboundWebhook record
            session: Optional database session

        Returns:
            Processing result with extracted data and statistics
        """
        if not session:
            async with get_async_session() as session:
                return await self.process_inbound_webhook(webhook_id, session)

        # Get webhook record
        query = select(InboundWebhook).where(InboundWebhook.id == webhook_id)
        result = await session.execute(query)
        webhook = result.scalar_one_or_none()

        if not webhook:
            raise ValueError(f"Webhook {webhook_id} not found")

        # Update status
        webhook.status = "processing"
        webhook.processed_at = datetime.utcnow()
        await session.flush()

        try:
            # Detect format if not already determined
            if webhook.data_format == "unknown" or not webhook.data_format:
                webhook.data_format = self._detect_format(
                    webhook.payload, webhook.content_type
                )

            # Parse and normalize data
            records = await self._parse_data(
                webhook.payload, webhook.data_format, webhook.raw_body
            )

            # Normalize fields
            normalized_records = []
            for record in records:
                normalized = self._normalize_record(record)
                normalized_records.append(normalized)

            # Store parsed data
            webhook.parsed_data = {
                "records": normalized_records,
                "count": len(normalized_records),
                "format": webhook.data_format,
            }
            webhook.record_count = len(normalized_records)

            # Auto-detect record type if not specified
            if normalized_records:
                record_type = self._detect_record_type(normalized_records[0])
            else:
                record_type = "unknown"

            # Queue for matching if we have an account
            if webhook.account_id and normalized_records:
                await self._queue_for_matching(
                    webhook.account_id,
                    normalized_records,
                    record_type,
                    webhook.integration_id,
                )

            # Update status
            webhook.status = "processed"
            webhook.processing_time_ms = int(
                (datetime.utcnow() - webhook.processed_at).total_seconds() * 1000
            )

            await session.commit()

            return {
                "status": "success",
                "webhook_id": str(webhook.id),
                "records_processed": len(normalized_records),
                "record_type": record_type,
                "format": webhook.data_format,
            }

        except Exception as e:
            logger.error(f"Failed to process webhook {webhook_id}: {e}")

            webhook.status = "failed"
            webhook.error = str(e)
            webhook.error_code = "PROCESSING_ERROR"

            await session.commit()

            return {"status": "failed", "webhook_id": str(webhook.id), "error": str(e)}

    def _detect_format(self, payload: Any, content_type: Optional[str]) -> str:
        """Detect data format from payload and content type."""
        if content_type:
            content_type = content_type.lower()

            if "json" in content_type:
                return DataFormat.JSON.value
            elif "csv" in content_type:
                return DataFormat.CSV.value
            elif "xml" in content_type:
                return DataFormat.XML.value
            elif "form" in content_type:
                return DataFormat.FORM.value
            elif "ndjson" in content_type or "jsonlines" in content_type:
                return DataFormat.NDJSON.value

        # Try to detect from payload structure
        if isinstance(payload, dict):
            if "raw" in payload:
                # Try to detect CSV or NDJSON
                raw = payload["raw"]
                if self._looks_like_csv(raw):
                    return DataFormat.CSV.value
                elif self._looks_like_ndjson(raw):
                    return DataFormat.NDJSON.value
            return DataFormat.JSON.value
        elif isinstance(payload, list):
            return DataFormat.JSON.value

        return DataFormat.UNKNOWN.value

    def _looks_like_csv(self, text: str) -> bool:
        """Check if text looks like CSV data."""
        lines = text.strip().split("\n")
        if len(lines) < 2:
            return False

        # Check for consistent comma/tab counts
        delimiter_counts = []
        for line in lines[:5]:  # Check first 5 lines
            comma_count = line.count(",")
            tab_count = line.count("\t")
            delimiter_counts.append(max(comma_count, tab_count))

        # If all lines have similar delimiter counts, likely CSV
        if delimiter_counts and all(c > 0 for c in delimiter_counts):
            return len(set(delimiter_counts)) <= 2  # Allow some variation

        return False

    def _looks_like_ndjson(self, text: str) -> bool:
        """Check if text looks like NDJSON (newline-delimited JSON)."""
        lines = text.strip().split("\n")
        if not lines:
            return False

        # Try to parse first few lines as JSON
        try:
            for line in lines[:3]:
                if line.strip():
                    json.loads(line)
            return True
        except:
            return False

    async def _parse_data(
        self, payload: Any, format: str, raw_body: Optional[str] = None
    ) -> List[Dict[str, Any]]:
        """Parse data based on format."""
        if format == DataFormat.JSON.value:
            return self._parse_json(payload)
        elif format == DataFormat.CSV.value:
            return self._parse_csv(raw_body or payload.get("raw", ""))
        elif format == DataFormat.NDJSON.value:
            return self._parse_ndjson(raw_body or payload.get("raw", ""))
        elif format == DataFormat.FORM.value:
            return [payload] if isinstance(payload, dict) else []
        else:
            # Try to extract any dict/list from payload
            if isinstance(payload, dict):
                if "data" in payload:
                    return self._parse_json(payload["data"])
                elif "records" in payload:
                    return self._parse_json(payload["records"])
                else:
                    return [payload]
            elif isinstance(payload, list):
                return payload
            else:
                return []

    def _parse_json(self, payload: Any) -> List[Dict[str, Any]]:
        """Parse JSON payload."""
        if isinstance(payload, list):
            return [r for r in payload if isinstance(r, dict)]
        elif isinstance(payload, dict):
            # Check for common data keys
            for key in ["data", "records", "items", "results", "rows"]:
                if key in payload and isinstance(payload[key], list):
                    return payload[key]
            # Single record
            return [payload]
        else:
            return []

    def _parse_csv(self, csv_text: str) -> List[Dict[str, Any]]:
        """Parse CSV text into records."""
        records = []

        try:
            # Try comma first, then tab
            for delimiter in [",", "\t"]:
                try:
                    reader = csv.DictReader(io.StringIO(csv_text), delimiter=delimiter)
                    temp_records = list(reader)
                    if temp_records:
                        records = temp_records
                        break
                except:
                    continue

            # Clean field names (remove BOM, whitespace)
            cleaned_records = []
            for record in records:
                cleaned = {}
                for key, value in record.items():
                    if key:
                        # Remove BOM and clean key
                        clean_key = key.replace("\ufeff", "").strip()
                        cleaned[clean_key] = value
                cleaned_records.append(cleaned)

            return cleaned_records

        except Exception as e:
            logger.error(f"Failed to parse CSV: {e}")
            return []

    def _parse_ndjson(self, ndjson_text: str) -> List[Dict[str, Any]]:
        """Parse NDJSON (newline-delimited JSON) text."""
        records = []

        for line in ndjson_text.strip().split("\n"):
            if line.strip():
                try:
                    record = json.loads(line)
                    if isinstance(record, dict):
                        records.append(record)
                except Exception as e:
                    logger.warning(f"Failed to parse NDJSON line: {e}")
                    continue

        return records

    def _normalize_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
        """
        Normalize field names in a record to standard format.
        """
        normalized = {}

        # Convert all keys to lowercase for matching
        lower_record = {k.lower().replace(" ", "_"): v for k, v in record.items()}

        # Company fields
        normalized["company"] = self._find_field_value(
            lower_record, self.field_mapping.COMPANY_FIELDS
        )

        # Name fields
        first_name = self._find_field_value(
            lower_record, self.field_mapping.FIRST_NAME_FIELDS
        )
        last_name = self._find_field_value(
            lower_record, self.field_mapping.LAST_NAME_FIELDS
        )
        full_name = self._find_field_value(
            lower_record, self.field_mapping.FULL_NAME_FIELDS
        )

        # Handle name parsing
        if first_name and last_name:
            normalized["first_name"] = first_name
            normalized["last_name"] = last_name
        elif full_name:
            # Try to split full name
            parts = full_name.strip().split(None, 1)
            if len(parts) == 2:
                normalized["first_name"] = parts[0]
                normalized["last_name"] = parts[1]
            else:
                normalized["first_name"] = full_name
                normalized["last_name"] = ""
        else:
            normalized["first_name"] = first_name or ""
            normalized["last_name"] = last_name or ""

        # Contact info
        normalized["email"] = self._find_field_value(
            lower_record, self.field_mapping.EMAIL_FIELDS
        )
        normalized["phone"] = self._find_field_value(
            lower_record, self.field_mapping.PHONE_FIELDS
        )
        normalized["title"] = self._find_field_value(
            lower_record, self.field_mapping.TITLE_FIELDS
        )

        # Address fields
        normalized["street"] = self._find_field_value(
            lower_record, self.field_mapping.STREET_FIELDS
        )
        normalized["city"] = self._find_field_value(
            lower_record, self.field_mapping.CITY_FIELDS
        )
        normalized["state"] = self._find_field_value(
            lower_record, self.field_mapping.STATE_FIELDS
        )
        normalized["zip"] = self._find_field_value(
            lower_record, self.field_mapping.ZIP_FIELDS
        )
        normalized["country"] = self._find_field_value(
            lower_record, self.field_mapping.COUNTRY_FIELDS
        )

        # Company info
        normalized["website"] = self._find_field_value(
            lower_record, self.field_mapping.WEBSITE_FIELDS
        )
        normalized["linkedin"] = self._find_field_value(
            lower_record, self.field_mapping.LINKEDIN_FIELDS
        )
        normalized["industry"] = self._find_field_value(
            lower_record, self.field_mapping.INDUSTRY_FIELDS
        )

        # Numeric fields
        employee_count = self._find_field_value(
            lower_record, self.field_mapping.EMPLOYEE_COUNT_FIELDS
        )
        if employee_count:
            normalized["employee_count"] = self._parse_number(employee_count)

        revenue = self._find_field_value(
            lower_record, self.field_mapping.REVENUE_FIELDS
        )
        if revenue:
            normalized["revenue"] = self._parse_number(revenue)

        # Include original fields that weren't mapped
        normalized["_original"] = record
        normalized["_unmapped"] = {}

        # Find unmapped fields
        mapped_keys = set()
        for field_list in [
            self.field_mapping.COMPANY_FIELDS,
            self.field_mapping.FIRST_NAME_FIELDS,
            self.field_mapping.LAST_NAME_FIELDS,
            self.field_mapping.FULL_NAME_FIELDS,
            self.field_mapping.EMAIL_FIELDS,
            self.field_mapping.PHONE_FIELDS,
            self.field_mapping.TITLE_FIELDS,
            self.field_mapping.STREET_FIELDS,
            self.field_mapping.CITY_FIELDS,
            self.field_mapping.STATE_FIELDS,
            self.field_mapping.ZIP_FIELDS,
            self.field_mapping.COUNTRY_FIELDS,
            self.field_mapping.WEBSITE_FIELDS,
            self.field_mapping.LINKEDIN_FIELDS,
            self.field_mapping.INDUSTRY_FIELDS,
            self.field_mapping.EMPLOYEE_COUNT_FIELDS,
            self.field_mapping.REVENUE_FIELDS,
        ]:
            mapped_keys.update(field_list)

        for key, value in lower_record.items():
            if key not in mapped_keys and value:
                normalized["_unmapped"][key] = value

        return normalized

    def _find_field_value(
        self, record: Dict[str, Any], field_names: List[str]
    ) -> Optional[str]:
        """Find value for any of the given field names."""
        for field in field_names:
            if field in record and record[field]:
                value = record[field]
                if isinstance(value, str):
                    return value.strip()
                else:
                    return str(value)
        return None

    def _parse_number(self, value: str) -> Optional[float]:
        """Parse numeric value from string."""
        if not value:
            return None

        # Remove common formatting
        cleaned = re.sub(r"[,$£€¥]", "", str(value))
        cleaned = cleaned.strip()

        # Handle ranges (e.g., "100-500")
        if "-" in cleaned and not cleaned.startswith("-"):
            parts = cleaned.split("-")
            if len(parts) == 2:
                try:
                    # Take the midpoint of the range
                    low = float(parts[0])
                    high = float(parts[1])
                    return (low + high) / 2
                except:
                    pass

        # Handle "k" or "m" suffixes
        multiplier = 1
        if cleaned.lower().endswith("k"):
            multiplier = 1000
            cleaned = cleaned[:-1]
        elif cleaned.lower().endswith("m"):
            multiplier = 1000000
            cleaned = cleaned[:-1]
        elif cleaned.lower().endswith("b"):
            multiplier = 1000000000
            cleaned = cleaned[:-1]

        try:
            return float(cleaned) * multiplier
        except:
            return None

    def _detect_record_type(self, record: Dict[str, Any]) -> str:
        """Detect whether record is a lead, contact, or account."""
        # If we have email or name fields, likely a contact/lead
        has_email = bool(record.get("email"))
        has_name = bool(record.get("first_name") or record.get("last_name"))
        has_company = bool(record.get("company"))
        has_website = bool(record.get("website"))

        if has_email or has_name:
            if has_company:
                return "lead"  # Person with company info
            else:
                return "contact"  # Just person info
        elif has_company or has_website:
            return "account"  # Company info only
        else:
            return "unknown"

    async def _queue_for_matching(
        self,
        account_id: int,
        records: List[Dict[str, Any]],
        record_type: str,
        integration_id: Optional[str] = None,
    ):
        """Queue normalized records for matching against Salesforce."""
        if not self.redis_client:
            logger.warning("Redis client not available for queuing")
            return

        # Create batch job
        job_id = str(uuid.uuid4())
        job_data = {
            "id": job_id,
            "account_id": account_id,
            "record_type": record_type,
            "integration_id": integration_id,
            "records": records,
            "created_at": datetime.utcnow().isoformat(),
        }

        # Store job data
        await self.redis_client.set(
            f"webhook:matching:job:{job_id}",
            json.dumps(job_data),
            ex=86400,  # Expire after 24 hours
        )

        # Queue for processing
        await self.redis_client.lpush("webhook:matching:queue", job_id)

        logger.info(
            f"Queued {len(records)} {record_type} records for matching (job {job_id})"
        )

    async def auto_configure_mapping(
        self,
        sample_records: List[Dict[str, Any]],
        account_id: int,
        session: Optional[AsyncSession] = None,
    ) -> Dict[str, Any]:
        """
        Use intelligence API to auto-configure field mappings.

        Analyzes sample records and suggests optimal field mappings.
        """
        if not sample_records:
            return {"mappings": {}, "confidence": 0}

        # Analyze field patterns
        field_analysis = {}
        for record in sample_records[:10]:  # Analyze first 10 records
            for key, value in record.items():
                if key not in field_analysis:
                    field_analysis[key] = {
                        "samples": [],
                        "types": set(),
                        "patterns": set(),
                    }

                if value:
                    field_analysis[key]["samples"].append(str(value)[:100])
                    field_analysis[key]["types"].add(type(value).__name__)

                    # Detect patterns
                    if isinstance(value, str):
                        if "@" in value:
                            field_analysis[key]["patterns"].add("email")
                        elif re.match(r"^[\d\s\-\+\(\)]+$", value):
                            field_analysis[key]["patterns"].add("phone")
                        elif re.match(r"^https?://", value):
                            field_analysis[key]["patterns"].add("url")
                        elif re.match(r"^\d{5}(-\d{4})?$", value):
                            field_analysis[key]["patterns"].add("zip")

        # Generate mapping suggestions
        mappings = {}
        confidence_scores = []

        for field, analysis in field_analysis.items():
            field_lower = field.lower().replace(" ", "_")
            confidence = 0
            suggested_mapping = None

            # Check patterns first (highest confidence)
            if "email" in analysis["patterns"]:
                suggested_mapping = "email"
                confidence = 0.95
            elif "phone" in analysis["patterns"]:
                suggested_mapping = "phone"
                confidence = 0.9
            elif "url" in analysis["patterns"]:
                if "linkedin" in field_lower:
                    suggested_mapping = "linkedin"
                else:
                    suggested_mapping = "website"
                confidence = 0.85
            elif "zip" in analysis["patterns"]:
                suggested_mapping = "zip"
                confidence = 0.9
            else:
                # Check field name similarity
                best_match = self._find_best_field_match(field_lower)
                if best_match:
                    suggested_mapping = best_match[0]
                    confidence = best_match[1]

            if suggested_mapping and confidence > 0.5:
                mappings[field] = {
                    "target": suggested_mapping,
                    "confidence": confidence,
                    "samples": analysis["samples"][:3],
                }
                confidence_scores.append(confidence)

        # Calculate overall confidence
        overall_confidence = (
            sum(confidence_scores) / len(confidence_scores) if confidence_scores else 0
        )

        return {
            "mappings": mappings,
            "confidence": overall_confidence,
            "fields_analyzed": len(field_analysis),
            "fields_mapped": len(mappings),
        }

    def _find_best_field_match(self, field_name: str) -> Optional[Tuple[str, float]]:
        """Find best matching standard field for a given field name."""
        best_match = None
        best_score = 0

        field_mappings = {
            "company": self.field_mapping.COMPANY_FIELDS,
            "first_name": self.field_mapping.FIRST_NAME_FIELDS,
            "last_name": self.field_mapping.LAST_NAME_FIELDS,
            "email": self.field_mapping.EMAIL_FIELDS,
            "phone": self.field_mapping.PHONE_FIELDS,
            "title": self.field_mapping.TITLE_FIELDS,
            "street": self.field_mapping.STREET_FIELDS,
            "city": self.field_mapping.CITY_FIELDS,
            "state": self.field_mapping.STATE_FIELDS,
            "zip": self.field_mapping.ZIP_FIELDS,
            "country": self.field_mapping.COUNTRY_FIELDS,
            "website": self.field_mapping.WEBSITE_FIELDS,
            "linkedin": self.field_mapping.LINKEDIN_FIELDS,
            "industry": self.field_mapping.INDUSTRY_FIELDS,
            "employee_count": self.field_mapping.EMPLOYEE_COUNT_FIELDS,
            "revenue": self.field_mapping.REVENUE_FIELDS,
        }

        for target_field, variations in field_mappings.items():
            for variation in variations:
                # Calculate similarity score
                score = self._calculate_similarity(field_name, variation)
                if score > best_score:
                    best_score = score
                    best_match = (target_field, score)

        return best_match if best_score > 0.5 else None

    def _calculate_similarity(self, str1: str, str2: str) -> float:
        """Calculate similarity score between two strings."""
        # Exact match
        if str1 == str2:
            return 1.0

        # One contains the other
        if str1 in str2 or str2 in str1:
            return 0.8

        # Levenshtein distance-based similarity
        distance = self._levenshtein_distance(str1, str2)
        max_len = max(len(str1), len(str2))
        if max_len == 0:
            return 0

        return 1 - (distance / max_len)

    def _levenshtein_distance(self, s1: str, s2: str) -> int:
        """Calculate Levenshtein distance between two strings."""
        if len(s1) < len(s2):
            return self._levenshtein_distance(s2, s1)

        if len(s2) == 0:
            return len(s1)

        previous_row = range(len(s2) + 1)
        for i, c1 in enumerate(s1):
            current_row = [i + 1]
            for j, c2 in enumerate(s2):
                # j+1 instead of j since previous_row and current_row are one character longer
                insertions = previous_row[j + 1] + 1
                deletions = current_row[j] + 1
                substitutions = previous_row[j] + (c1 != c2)
                current_row.append(min(insertions, deletions, substitutions))
            previous_row = current_row

        return previous_row[-1]
