# saas/services/instant_survey.py
"""
Instant survey that runs automatically after Salesforce connection.
Analyzes data quality and finds duplicates using the core engine.
All matching delegated to EngineClient - no local fuzzy logic.
"""

from typing import Dict, Any, List, Optional
from datetime import datetime
import asyncio
import logging
import pandas as pd

# Use the engine for ALL matching operations
from .engine_client import get_engine_client
from .rate_limiter import SmartRateLimiter
from .discovery_service import DiscoveryService
from .utils.data_profile import normalize_scalar

logger = logging.getLogger(__name__)

# Constants
PERSONAL_DOMAINS = {
    "gmail.com",
    "yahoo.com",
    "hotmail.com",
    "outlook.com",
    "aol.com",
    "icloud.com",
    "me.com",
    "mac.com",
}

SAMPLE_SIZE = 1000  # Per object
MAX_SCAN_TIME = 90  # seconds


class InstantSurvey:
    """Performs instant analysis of Salesforce data quality using the core engine."""

    def __init__(self, sf_client, progress_callback=None, account_id=None):
        self.sf = sf_client
        self.engine = get_engine_client()  # Use engine for all matching
        from .rate_limiter import RateLimitConfig

        self.rate_limiter = SmartRateLimiter(RateLimitConfig(requests_per_second=5))
        self.progress_callback = progress_callback or (lambda x: None)
        self.start_time = None
        self.account_id = account_id
        self.discovery_service = DiscoveryService()

    async def run(self) -> Dict[str, Any]:
        """Main entry point - runs complete survey."""
        self.start_time = datetime.utcnow()

        try:
            self.progress_callback(
                {"phase": "starting", "message": "Initializing survey..."}
            )

            # Run surveys for each object in parallel
            results = await asyncio.gather(
                self._survey_object("Account"),
                self._survey_object("Lead"),
                self._survey_object("Contact"),
                return_exceptions=True,
            )

            # Combine results
            combined = self._combine_results(results)

            # Calculate overall metrics
            combined["summary"] = self._calculate_summary(combined)
            combined["duration_seconds"] = (
                datetime.utcnow() - self.start_time
            ).total_seconds()

            self.progress_callback({"phase": "complete", "message": "Survey complete!"})
            return combined

        except Exception as e:
            logger.error(f"Survey failed: {e}")
            self.progress_callback({"phase": "error", "message": str(e)})
            raise

    async def _survey_object(self, object_type: str) -> Dict[str, Any]:
        """Survey a single Salesforce object."""
        self.progress_callback(
            {
                "phase": "scanning",
                "message": f"Analyzing {object_type} records...",
                "object": object_type,
            }
        )

        try:
            # 1. Get schema
            schema = await self.sf.describe_object(object_type)
            fields_meta = {f["name"]: f for f in schema.get("fields", [])}

            # 2. Identify key fields
            key_fields = self._identify_key_fields(object_type, fields_meta)

            # 3. Sample records
            sample = await self.sf.get_sample_records(
                object_type, list(key_fields.keys()), n=SAMPLE_SIZE
            )

            if not sample:
                return {"object": object_type, "error": "No records found"}

            # 4. Compute field statistics
            field_stats = self._compute_field_stats(sample, key_fields)

            # 5. Generate matching weights based on field quality
            weights = self._generate_weights(object_type, field_stats)

            # 6. Find duplicates using the ENGINE
            duplicates = await self._find_duplicates_with_engine(
                object_type, sample, weights
            )

            # 7. Identify enrichment opportunities
            enrichment = self._identify_enrichment_opportunities(sample, field_stats)

            return {
                "object": object_type,
                "records_sampled": len(sample),
                "field_stats": field_stats,
                "weights": weights,
                "duplicates": duplicates,
                "enrichment_opportunities": enrichment,
                "proposed_policy": self._generate_policy(object_type, weights),
            }

        except Exception as e:
            logger.error(f"Survey failed for {object_type}: {e}")
            return {"object": object_type, "error": str(e)}

    def _identify_key_fields(
        self, object_type: str, fields_meta: Dict
    ) -> Dict[str, str]:
        """Identify fields useful for matching."""
        key_fields = {}

        # Try to use discovered fields if available
        if self.account_id:
            try:
                discovery_policy = asyncio.create_task(
                    self.discovery_service.get_policy_for_matching(
                        self.account_id, object_type
                    )
                )
                policy = asyncio.run_coroutine_threadsafe(
                    discovery_policy, asyncio.get_event_loop()
                ).result(timeout=2)

                if policy and policy.get("fields"):
                    # Use discovered fields
                    for idx, field in enumerate(policy["fields"][:10]):
                        if field in fields_meta:
                            key_fields[field] = f"discovered_{idx}"

                    if key_fields:
                        logger.info(
                            f"Using {len(key_fields)} discovered fields for {object_type}"
                        )
                        return key_fields
            except Exception as e:
                logger.debug(f"Could not get discovery policy: {e}")

        # Fall back to standard fields by object
        if object_type == "Account":
            field_map = {
                "Id": "id",
                "Name": "name",
                "Website": "website",
                "Phone": "phone",
                "BillingCity": "city",
                "BillingState": "state",
                "BillingPostalCode": "postal",
            }
        elif object_type == "Lead":
            field_map = {
                "Id": "id",
                "FirstName": "first",
                "LastName": "last",
                "Company": "company",
                "Email": "email",
                "Website": "website",
                "Phone": "phone",
            }
        elif object_type == "Contact":
            field_map = {
                "Id": "id",
                "FirstName": "first",
                "LastName": "last",
                "Email": "email",
                "Phone": "phone",
                "AccountId": "account",
                "Title": "title",
            }
        else:
            field_map = {"Id": "id", "Name": "name"}

        # Only include fields that exist
        for field, alias in field_map.items():
            if field in fields_meta:
                key_fields[field] = alias

        return key_fields

    def _compute_field_stats(self, records: List[Dict], key_fields: Dict) -> Dict:
        """Compute statistics for each field."""
        stats = {}

        for field, alias in key_fields.items():
            values = [normalize_scalar(r.get(field)) for r in records]
            non_null = [v for v in values if v is not None]

            # Basic stats
            fill_rate = len(non_null) / max(len(records), 1)
            unique_rate = len(set(non_null)) / max(len(non_null), 1) if non_null else 0

            # Field-specific stats
            field_lower = field.lower()
            format_valid = 1.0
            personal_rate = 0.0

            if "email" in field_lower and non_null:
                valid = [
                    v
                    for v in non_null
                    if "@" in str(v) and "." in str(v).split("@")[-1]
                ]
                format_valid = len(valid) / len(non_null)
                domains = [
                    str(v).split("@")[-1].lower() for v in valid if "@" in str(v)
                ]
                personal = [d for d in domains if d in PERSONAL_DOMAINS]
                personal_rate = len(personal) / max(len(domains), 1)

            elif "phone" in field_lower and non_null:
                valid = [v for v in non_null if len(self._normalize_phone(v)) >= 10]
                format_valid = len(valid) / len(non_null)

            elif "website" in field_lower and non_null:
                valid = [v for v in non_null if "." in str(v)]
                format_valid = len(valid) / len(non_null)

            sample_values = []
            if non_null:
                seen = set()
                for value in non_null:
                    if value in seen:
                        continue
                    seen.add(value)
                    sample_values.append(value)
                    if len(sample_values) == 5:
                        break

            stats[field] = {
                "fill_rate": round(fill_rate, 3),
                "unique_rate": round(unique_rate, 3),
                "format_valid": round(format_valid, 3),
                "personal_rate": round(personal_rate, 3),
                "sample_values": sample_values,
            }

        return stats

    def _generate_weights(
        self, object_type: str, field_stats: Dict
    ) -> Dict[str, float]:
        """Generate matching weights based on field quality."""
        weights = {}

        for field, stats in field_stats.items():
            field_lower = field.lower()

            # Skip ID fields
            if field_lower in ["id", "_id", "recordid"]:
                continue

            # Base score from fill rate
            score = stats["fill_rate"]

            # Adjust based on field type
            if "email" in field_lower or "website" in field_lower:
                score *= 1.5  # High value for domain matching
                score *= stats["format_valid"]
                score *= 1 - stats["personal_rate"] * 0.5  # Reduce for personal emails
            elif "phone" in field_lower:
                score *= 1.3
                score *= stats["format_valid"]
            elif "name" in field_lower or "company" in field_lower:
                score *= 1.2
                score *= min(stats["unique_rate"] * 2, 1.0)  # Penalize if too uniform
            elif "accountid" in field_lower:
                score *= 0.5  # Lower weight for reference fields

            # Penalize if too sparse
            if stats["fill_rate"] < 0.3:
                score *= 0.5

            if score > 0.3:  # Threshold for inclusion
                weights[field] = round(score, 3)

        # Normalize weights to sum to 1
        total = sum(weights.values())
        if total > 0:
            weights = {k: round(v / total, 3) for k, v in weights.items()}

        return weights

    async def _find_duplicates_with_engine(
        self, object_type: str, records: List[Dict], weights: Dict
    ) -> List[Dict]:
        """Find duplicates using the core engine instead of local logic."""
        if not weights or len(records) < 2:
            return []

        # Convert records to DataFrame for engine
        df = pd.DataFrame(records)

        # Create mappings from weights
        mappings = []
        for field, weight in weights.items():
            if field not in df.columns:
                continue

            # Determine algorithm based on field type
            field_lower = field.lower()
            if "email" in field_lower or "website" in field_lower:
                algorithm = "Domain"
            elif "phone" in field_lower:
                algorithm = "Exact"
            elif "name" in field_lower or "company" in field_lower:
                algorithm = "TokenSetRatio"
            else:
                algorithm = "WRatio"

            mappings.append(
                {
                    "source_column": field,
                    "reference_column": field,  # Same for dedupe
                    "weight": weight,
                    "algorithm": algorithm,
                }
            )

        if not mappings:
            logger.warning(f"No valid mappings for {object_type}")
            return []

        # Create blocking strategy based on key fields
        blocking_strategy = None
        if "Email" in df.columns or "Website" in df.columns:
            blocking_strategy = {
                "source_column": "Email" if "Email" in df.columns else "Website",
                "key_length": 3,
            }
        elif "Phone" in df.columns:
            blocking_strategy = {"source_column": "Phone", "key_length": 7}

        # Use engine to find duplicates
        try:
            result = await self.engine.find_duplicates(
                data=df,
                mappings=mappings,
                threshold=0.75,  # Duplicate threshold
                blocking_strategy=blocking_strategy,
                id_column="Id" if "Id" in df.columns else None,
            )

            # Convert engine results to survey format
            duplicates = []
            matches_df = result.matches

            # Group matches by pairs
            processed_pairs = set()

            for _, match in matches_df.iterrows():
                src_id = match.get("source_id", match.get("src_id"))
                ref_id = match.get("duplicate_id", match.get("ref_id"))

                # Skip if already processed
                pair_key = tuple(sorted([src_id, ref_id]))
                if pair_key in processed_pairs:
                    continue
                processed_pairs.add(pair_key)

                # Find original records
                src_record = next(
                    (r for r in records if str(r.get("Id")) == str(src_id)), None
                )
                ref_record = next(
                    (r for r in records if str(r.get("Id")) == str(ref_id)), None
                )

                if not src_record or not ref_record:
                    continue

                duplicates.append(
                    {
                        "records": [
                            self._safe_record_summary(src_record),
                            self._safe_record_summary(ref_record),
                        ],
                        "score": float(match.get("score", match.get("match_score", 0))),
                        "contributions": match.get("feature_scores", {}),
                        "confidence_tier": self.engine._get_tier(
                            float(match.get("score", 0))
                        ),
                        "explanation": match.get(
                            "explanation", "High similarity detected"
                        ),
                        "visual": self._format_visual_bar(
                            int(float(match.get("score", 0)) * 100)
                        ),
                    }
                )

                if len(duplicates) >= 50:  # Cap for UI
                    break

            return sorted(duplicates, key=lambda x: x["score"], reverse=True)

        except Exception as e:
            logger.error(f"Engine duplicate detection failed: {e}")
            return []

    def _safe_record_summary(self, record: Dict) -> Dict:
        """Create a safe summary of a record for display."""
        summary = {"Id": record.get("Id", "Unknown")}

        # Add key fields
        for field in ["Name", "Company", "FirstName", "LastName", "Email", "Phone"]:
            if field in record and record[field]:
                summary[field] = str(record[field])[:100]  # Truncate long values

        return summary

    def _format_visual_bar(self, percent: int) -> str:
        """Create a visual progress bar."""
        filled = int(percent / 10)
        empty = 10 - filled
        return f"[{'█' * filled}{'░' * empty}] {percent}%"

    def _normalize_phone(self, value: str) -> str:
        """Normalize phone number."""
        return "".join(c for c in str(value) if c.isdigit())

    def _extract_domain(self, value: str) -> Optional[str]:
        """Extract domain from email or website."""
        if not value:
            return None
        s = str(value).lower().strip()
        if "@" in s:
            s = s.split("@")[-1]
        s = s.replace("http://", "").replace("https://", "")
        if s.startswith("www."):
            s = s[4:]
        return s if "." in s and len(s) >= 3 else None

    def _identify_enrichment_opportunities(
        self, records: List[Dict], field_stats: Dict
    ) -> List[Dict]:
        """Identify opportunities for data enrichment."""
        opportunities = []

        for field, stats in field_stats.items():
            if stats["fill_rate"] < 0.5:
                opportunities.append(
                    {
                        "field": field,
                        "fill_rate": stats["fill_rate"],
                        "impact": "high" if stats["fill_rate"] < 0.2 else "medium",
                        "suggestion": f"Consider enriching {field} - only {int(stats['fill_rate']*100)}% populated",
                    }
                )

        return sorted(opportunities, key=lambda x: x["fill_rate"])[:10]

    def _generate_policy(self, object_type: str, weights: Dict) -> Dict:
        """Generate a suggested matching policy."""
        return {
            "object_type": object_type,
            "field_weights": weights,
            "suggested_threshold": 0.75,
            "blocking_strategy": "domain"
            if any("email" in k.lower() or "website" in k.lower() for k in weights)
            else "name",
        }

    def _combine_results(self, results: List) -> Dict:
        """Combine results from multiple objects."""
        combined = {"objects": {}, "total_duplicates": 0, "total_records_sampled": 0}

        for result in results:
            if isinstance(result, dict) and "object" in result:
                obj_name = result["object"]
                combined["objects"][obj_name] = result

                if "duplicates" in result:
                    combined["total_duplicates"] += len(result.get("duplicates", []))
                if "records_sampled" in result:
                    combined["total_records_sampled"] += result["records_sampled"]

        return combined

    def _calculate_summary(self, combined: Dict) -> Dict:
        """Calculate overall summary metrics."""
        summary = {
            "total_objects_analyzed": len(combined["objects"]),
            "total_records_sampled": combined["total_records_sampled"],
            "total_duplicates_found": combined["total_duplicates"],
            "data_quality_score": 0.0,
        }

        # Calculate average data quality score
        quality_scores = []
        for obj_data in combined["objects"].values():
            if "field_stats" in obj_data:
                fill_rates = [s["fill_rate"] for s in obj_data["field_stats"].values()]
                if fill_rates:
                    quality_scores.append(sum(fill_rates) / len(fill_rates))

        if quality_scores:
            summary["data_quality_score"] = round(
                sum(quality_scores) / len(quality_scores), 2
            )

        return summary
