"""
Salesforce Data Validation and Quality Checks for FoundryMatch
==============================================================
Comprehensive data validation before executing Salesforce workflows to prevent errors
and ensure data quality.
"""

import re
import logging
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from typing import Dict, List, Optional, Any
import pandas as pd
import numpy as np

from .core import SalesforceIntegration, SalesforceField

log = logging.getLogger(__name__)


class ValidationSeverity(Enum):
    """Severity levels for validation issues."""

    ERROR = "error"  # Will prevent workflow execution
    WARNING = "warning"  # Should be reviewed but won't prevent execution
    INFO = "info"  # Informational, good to know


@dataclass
class ValidationIssue:
    """Represents a data validation issue."""

    severity: ValidationSeverity
    field_name: str
    message: str
    affected_records: List[str] = field(default_factory=list)
    suggestion: Optional[str] = None
    auto_fixable: bool = False


@dataclass
class ValidationResult:
    """Result of data validation."""

    is_valid: bool
    total_records: int
    issues: List[ValidationIssue] = field(default_factory=list)
    errors: List[ValidationIssue] = field(default_factory=list)
    warnings: List[ValidationIssue] = field(default_factory=list)
    info: List[ValidationIssue] = field(default_factory=list)
    validation_time: float = 0.0

    def __post_init__(self):
        """Categorize issues by severity."""
        self.errors = [
            issue for issue in self.issues if issue.severity == ValidationSeverity.ERROR
        ]
        self.warnings = [
            issue
            for issue in self.issues
            if issue.severity == ValidationSeverity.WARNING
        ]
        self.info = [
            issue for issue in self.issues if issue.severity == ValidationSeverity.INFO
        ]
        self.is_valid = len(self.errors) == 0


@dataclass
class DataQualityReport:
    """Comprehensive data quality analysis."""

    overall_score: float  # 0-100
    completeness_score: float
    validity_score: float
    consistency_score: float
    accuracy_score: float
    field_reports: Dict[str, Dict[str, Any]] = field(default_factory=dict)
    recommendations: List[str] = field(default_factory=list)


class SalesforceDataValidator:
    """Comprehensive data validator for Salesforce operations."""

    def __init__(self, sf_integration: SalesforceIntegration):
        self.sf_integration = sf_integration
        self.field_cache: Dict[str, List[SalesforceField]] = {}

        # Common validation patterns
        self.email_pattern = re.compile(
            r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
        )
        self.phone_pattern = re.compile(r"^[\+]?[1-9][\d]{0,15}$")
        self.url_pattern = re.compile(r"^https?://[^\s/$.?#].[^\s]*$")

        # Salesforce specific patterns
        self.sf_id_pattern = re.compile(r"^[a-zA-Z0-9]{15}$|^[a-zA-Z0-9]{18}$")

        # Common business domains for domain validation
        self.common_domains = {
            "gmail.com",
            "yahoo.com",
            "hotmail.com",
            "outlook.com",
            "aol.com",
            "icloud.com",
            "protonmail.com",
        }

    async def validate_for_workflow(
        self,
        data: pd.DataFrame,
        object_name: str,
        operation_type: str = "upsert",
        external_id_field: Optional[str] = None,
    ) -> ValidationResult:
        """
        Validate data for a specific Salesforce workflow.

        Args:
            data: DataFrame to validate
            object_name: Target Salesforce object
            operation_type: Type of operation (insert, update, upsert, delete)
            external_id_field: External ID field for upsert operations

        Returns:
            ValidationResult with detailed findings
        """
        start_time = datetime.now()
        issues = []

        try:
            log.info(
                f"Validating {len(data)} records for {operation_type} on {object_name}"
            )

            # Get object metadata
            object_fields = await self._get_object_fields(object_name)
            field_map = {field.name: field for field in object_fields}

            # 1. Schema validation
            schema_issues = await self._validate_schema(
                data, object_name, field_map, operation_type
            )
            issues.extend(schema_issues)

            # 2. Required fields validation
            required_issues = await self._validate_required_fields(
                data, field_map, operation_type
            )
            issues.extend(required_issues)

            # 3. Data type validation
            type_issues = await self._validate_data_types(data, field_map)
            issues.extend(type_issues)

            # 4. Field length validation
            length_issues = await self._validate_field_lengths(data, field_map)
            issues.extend(length_issues)

            # 5. Format validation (email, phone, etc.)
            format_issues = await self._validate_formats(data, field_map)
            issues.extend(format_issues)

            # 6. Business logic validation
            business_issues = await self._validate_business_logic(
                data, object_name, field_map
            )
            issues.extend(business_issues)

            # 7. External ID validation (for upserts)
            if operation_type == "upsert" and external_id_field:
                external_id_issues = await self._validate_external_id(
                    data, external_id_field, field_map
                )
                issues.extend(external_id_issues)

            # 8. Record limit validation
            limit_issues = await self._validate_record_limits(data, operation_type)
            issues.extend(limit_issues)

            # 9. Duplicate detection within dataset
            duplicate_issues = await self._validate_duplicates(
                data, external_id_field or "Id"
            )
            issues.extend(duplicate_issues)

            execution_time = (datetime.now() - start_time).total_seconds()

            result = ValidationResult(
                is_valid=not any(
                    issue.severity == ValidationSeverity.ERROR for issue in issues
                ),
                total_records=len(data),
                issues=issues,
                validation_time=execution_time,
            )

            log.info(
                f"Validation completed: {len(result.errors)} errors, {len(result.warnings)} warnings"
            )
            return result

        except Exception as e:
            log.error(f"Validation failed: {e}", exc_info=True)
            execution_time = (datetime.now() - start_time).total_seconds()

            return ValidationResult(
                is_valid=False,
                total_records=len(data),
                issues=[
                    ValidationIssue(
                        severity=ValidationSeverity.ERROR,
                        field_name="validation",
                        message=f"Validation process failed: {str(e)}",
                    )
                ],
                validation_time=execution_time,
            )

    async def analyze_data_quality(
        self, data: pd.DataFrame, object_name: str
    ) -> DataQualityReport:
        """
        Perform comprehensive data quality analysis.

        Args:
            data: DataFrame to analyze
            object_name: Target Salesforce object

        Returns:
            DataQualityReport with quality metrics and recommendations
        """
        try:
            log.info(f"Analyzing data quality for {len(data)} records")

            # Get object metadata
            object_fields = await self._get_object_fields(object_name)
            field_map = {field.name: field for field in object_fields}

            # Calculate quality scores
            completeness_score = self._calculate_completeness_score(data, field_map)
            validity_score = self._calculate_validity_score(data, field_map)
            consistency_score = self._calculate_consistency_score(data, field_map)
            accuracy_score = self._calculate_accuracy_score(data, field_map)

            # Overall score (weighted average)
            overall_score = (
                completeness_score * 0.3
                + validity_score * 0.3
                + consistency_score * 0.2
                + accuracy_score * 0.2
            )

            # Generate field-level reports
            field_reports = {}
            for column in data.columns:
                if column in field_map:
                    field_reports[column] = self._analyze_field_quality(
                        data[column], field_map[column]
                    )

            # Generate recommendations
            recommendations = self._generate_quality_recommendations(
                data,
                field_map,
                completeness_score,
                validity_score,
                consistency_score,
                accuracy_score,
            )

            return DataQualityReport(
                overall_score=overall_score,
                completeness_score=completeness_score,
                validity_score=validity_score,
                consistency_score=consistency_score,
                accuracy_score=accuracy_score,
                field_reports=field_reports,
                recommendations=recommendations,
            )

        except Exception as e:
            log.error(f"Data quality analysis failed: {e}", exc_info=True)
            return DataQualityReport(
                overall_score=0.0,
                completeness_score=0.0,
                validity_score=0.0,
                consistency_score=0.0,
                accuracy_score=0.0,
                recommendations=[f"Analysis failed: {str(e)}"],
            )

    async def _get_object_fields(self, object_name: str) -> List[SalesforceField]:
        """Get and cache object field metadata."""
        if object_name not in self.field_cache:
            self.field_cache[object_name] = await self.sf_integration.get_object_fields(
                object_name
            )
        return self.field_cache[object_name]

    async def _validate_schema(
        self,
        data: pd.DataFrame,
        object_name: str,
        field_map: Dict[str, SalesforceField],
        operation_type: str,
    ) -> List[ValidationIssue]:
        """Validate data schema against Salesforce object."""
        issues = []

        # Check for unknown fields
        unknown_fields = set(data.columns) - set(field_map.keys())
        if unknown_fields:
            issues.append(
                ValidationIssue(
                    severity=ValidationSeverity.WARNING,
                    field_name="schema",
                    message=f"Unknown fields found: {', '.join(unknown_fields)}",
                    suggestion="Remove unknown fields or verify field names",
                )
            )

        # Check for required fields missing in schema
        if operation_type == "insert":
            required_fields = [
                field.name
                for field in field_map.values()
                if not field.nullable and field.name != "Id"
            ]
            missing_required = set(required_fields) - set(data.columns)
            if missing_required:
                issues.append(
                    ValidationIssue(
                        severity=ValidationSeverity.ERROR,
                        field_name="schema",
                        message=f"Required fields missing: {', '.join(missing_required)}",
                        suggestion="Add required fields to your data",
                    )
                )

        return issues

    async def _validate_required_fields(
        self,
        data: pd.DataFrame,
        field_map: Dict[str, SalesforceField],
        operation_type: str,
    ) -> List[ValidationIssue]:
        """Validate required field values."""
        issues = []

        for field_name, field in field_map.items():
            if field_name in data.columns and not field.nullable:
                null_mask = data[field_name].isna()
                null_count = null_mask.sum()

                if null_count > 0:
                    null_records = data[null_mask].index.tolist()[
                        :10
                    ]  # First 10 for performance

                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.ERROR,
                            field_name=field_name,
                            message=f"Required field has {null_count} null values",
                            affected_records=[str(idx) for idx in null_records],
                            suggestion=f"Provide values for required field {field_name}",
                        )
                    )

        return issues

    async def _validate_data_types(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> List[ValidationIssue]:
        """Validate data types against Salesforce field types."""
        issues = []

        for field_name, field in field_map.items():
            if field_name not in data.columns:
                continue

            series = data[field_name].dropna()
            if series.empty:
                continue

            # Validate based on Salesforce field type
            if field.type in ["int", "double", "currency", "percent"]:
                non_numeric = []
                for idx, value in series.items():
                    try:
                        float(value)
                    except (ValueError, TypeError):
                        non_numeric.append(str(idx))
                        if len(non_numeric) >= 10:  # Limit for performance
                            break

                if non_numeric:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.ERROR,
                            field_name=field_name,
                            message="Non-numeric values found in numeric field",
                            affected_records=non_numeric,
                            suggestion=f"Ensure all values in {field_name} are numeric",
                        )
                    )

            elif field.type == "boolean":
                invalid_boolean = []
                for idx, value in series.items():
                    if str(value).lower() not in [
                        "true",
                        "false",
                        "1",
                        "0",
                        "yes",
                        "no",
                    ]:
                        invalid_boolean.append(str(idx))
                        if len(invalid_boolean) >= 10:
                            break

                if invalid_boolean:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.ERROR,
                            field_name=field_name,
                            message="Invalid boolean values found",
                            affected_records=invalid_boolean,
                            suggestion=f"Use true/false, 1/0, or yes/no for {field_name}",
                        )
                    )

            elif field.type == "date":
                invalid_dates = []
                for idx, value in series.items():
                    try:
                        pd.to_datetime(value)
                    except (ValueError, TypeError):
                        invalid_dates.append(str(idx))
                        if len(invalid_dates) >= 10:
                            break

                if invalid_dates:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.ERROR,
                            field_name=field_name,
                            message="Invalid date values found",
                            affected_records=invalid_dates,
                            suggestion=f"Use ISO date format (YYYY-MM-DD) for {field_name}",
                        )
                    )

        return issues

    async def _validate_field_lengths(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> List[ValidationIssue]:
        """Validate field length constraints."""
        issues = []

        for field_name, field in field_map.items():
            if field_name not in data.columns or not field.length:
                continue

            series = data[field_name].dropna().astype(str)
            if series.empty:
                continue

            too_long_mask = series.str.len() > field.length
            too_long_count = too_long_mask.sum()

            if too_long_count > 0:
                too_long_records = data[too_long_mask].index.tolist()[:10]

                issues.append(
                    ValidationIssue(
                        severity=ValidationSeverity.ERROR,
                        field_name=field_name,
                        message=f"{too_long_count} values exceed maximum length of {field.length}",
                        affected_records=[str(idx) for idx in too_long_records],
                        suggestion=f"Truncate or clean values in {field_name}",
                        auto_fixable=True,
                    )
                )

        return issues

    async def _validate_formats(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> List[ValidationIssue]:
        """Validate format patterns (email, phone, URL, etc.)."""
        issues = []

        for field_name, field in field_map.items():
            if field_name not in data.columns:
                continue

            series = data[field_name].dropna().astype(str)
            if series.empty:
                continue

            # Email validation
            if "email" in field_name.lower() or field.type == "email":
                invalid_emails = []
                for idx, value in series.items():
                    if not self.email_pattern.match(value):
                        invalid_emails.append(str(idx))
                        if len(invalid_emails) >= 10:
                            break

                if invalid_emails:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.ERROR,
                            field_name=field_name,
                            message="Invalid email format found",
                            affected_records=invalid_emails,
                            suggestion=f"Ensure {field_name} contains valid email addresses",
                        )
                    )

            # Phone validation
            elif "phone" in field_name.lower():
                invalid_phones = []
                for idx, value in series.items():
                    # Remove common formatting characters
                    clean_phone = re.sub(r"[^\d+]", "", value)
                    if not self.phone_pattern.match(clean_phone):
                        invalid_phones.append(str(idx))
                        if len(invalid_phones) >= 10:
                            break

                if invalid_phones:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.WARNING,
                            field_name=field_name,
                            message="Invalid phone format found",
                            affected_records=invalid_phones,
                            suggestion=f"Standardize phone format in {field_name}",
                        )
                    )

            # URL validation
            elif "url" in field_name.lower() or "website" in field_name.lower():
                invalid_urls = []
                for idx, value in series.items():
                    if not self.url_pattern.match(value):
                        invalid_urls.append(str(idx))
                        if len(invalid_urls) >= 10:
                            break

                if invalid_urls:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.WARNING,
                            field_name=field_name,
                            message="Invalid URL format found",
                            affected_records=invalid_urls,
                            suggestion=f"Ensure URLs in {field_name} start with http:// or https://",
                        )
                    )

            # Salesforce ID validation
            elif (
                field_name == "Id"
                or field_name.endswith("__c")
                and "id" in field_name.lower()
            ):
                invalid_ids = []
                for idx, value in series.items():
                    if not self.sf_id_pattern.match(value):
                        invalid_ids.append(str(idx))
                        if len(invalid_ids) >= 10:
                            break

                if invalid_ids:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.ERROR,
                            field_name=field_name,
                            message="Invalid Salesforce ID format found",
                            affected_records=invalid_ids,
                            suggestion=f"Ensure {field_name} contains valid 15 or 18 character Salesforce IDs",
                        )
                    )

        return issues

    async def _validate_business_logic(
        self,
        data: pd.DataFrame,
        object_name: str,
        field_map: Dict[str, SalesforceField],
    ) -> List[ValidationIssue]:
        """Validate business logic rules."""
        issues = []

        # Object-specific validations
        if object_name == "Lead":
            issues.extend(self._validate_lead_specific(data, field_map))
        elif object_name == "Account":
            issues.extend(self._validate_account_specific(data, field_map))
        elif object_name == "Contact":
            issues.extend(self._validate_contact_specific(data, field_map))
        elif object_name == "Opportunity":
            issues.extend(self._validate_opportunity_specific(data, field_map))

        return issues

    def _validate_lead_specific(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> List[ValidationIssue]:
        """Lead-specific business logic validation."""
        issues = []

        # Leads with AccountId should have appropriate status
        if "AccountId" in data.columns and "Status" in data.columns:
            leads_with_account = data[data["AccountId"].notna()]
            if not leads_with_account.empty:
                unconverted_with_account = leads_with_account[
                    ~leads_with_account["Status"].isin(["Converted", "Qualified"])
                ]
                if not unconverted_with_account.empty:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.WARNING,
                            field_name="Status",
                            message="Leads with AccountId should typically have 'Converted' or 'Qualified' status",
                            affected_records=unconverted_with_account.index.astype(
                                str
                            ).tolist()[:10],
                            suggestion="Review Lead status for leads with Account associations",
                        )
                    )

        return issues

    def _validate_account_specific(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> List[ValidationIssue]:
        """Account-specific business logic validation."""
        issues = []

        # Check for potential personal email domains in business accounts
        if "Name" in data.columns and any(
            "email" in col.lower() for col in data.columns
        ):
            email_cols = [col for col in data.columns if "email" in col.lower()]
            for email_col in email_cols:
                if email_col in data.columns:
                    for idx, row in data.iterrows():
                        email = str(row[email_col]).lower()
                        if "@" in email:
                            domain = email.split("@")[-1]
                            if domain in self.common_domains:
                                issues.append(
                                    ValidationIssue(
                                        severity=ValidationSeverity.WARNING,
                                        field_name=email_col,
                                        message="Personal email domain found in business account",
                                        affected_records=[str(idx)],
                                        suggestion="Verify business email addresses for B2B accounts",
                                    )
                                )

        return issues

    def _validate_contact_specific(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> List[ValidationIssue]:
        """Contact-specific business logic validation."""
        issues = []

        # Contacts should have either FirstName+LastName or just LastName
        if "FirstName" in data.columns and "LastName" in data.columns:
            missing_names = data[
                (
                    data["FirstName"].isna()
                    | (data["FirstName"].astype(str).str.strip() == "")
                )
                & (
                    data["LastName"].isna()
                    | (data["LastName"].astype(str).str.strip() == "")
                )
            ]
            if not missing_names.empty:
                issues.append(
                    ValidationIssue(
                        severity=ValidationSeverity.ERROR,
                        field_name="LastName",
                        message="Contacts must have at least a LastName",
                        affected_records=missing_names.index.astype(str).tolist()[:10],
                        suggestion="Provide LastName for all Contact records",
                    )
                )

        return issues

    def _validate_opportunity_specific(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> List[ValidationIssue]:
        """Opportunity-specific business logic validation."""
        issues = []

        # Closed Won opportunities should have positive amounts
        if "StageName" in data.columns and "Amount" in data.columns:
            closed_won = data[data["StageName"] == "Closed Won"]
            if not closed_won.empty:
                zero_amount = closed_won[
                    (closed_won["Amount"].isna()) | (closed_won["Amount"] <= 0)
                ]
                if not zero_amount.empty:
                    issues.append(
                        ValidationIssue(
                            severity=ValidationSeverity.WARNING,
                            field_name="Amount",
                            message="Closed Won opportunities should have positive amounts",
                            affected_records=zero_amount.index.astype(str).tolist()[
                                :10
                            ],
                            suggestion="Review Amount values for Closed Won opportunities",
                        )
                    )

        return issues

    async def _validate_external_id(
        self,
        data: pd.DataFrame,
        external_id_field: str,
        field_map: Dict[str, SalesforceField],
    ) -> List[ValidationIssue]:
        """Validate external ID field for upsert operations."""
        issues = []

        if external_id_field not in data.columns:
            issues.append(
                ValidationIssue(
                    severity=ValidationSeverity.ERROR,
                    field_name=external_id_field,
                    message=f"External ID field '{external_id_field}' not found in data",
                    suggestion=f"Add {external_id_field} column to your data",
                )
            )
            return issues

        # Check for null external IDs
        null_external_ids = data[data[external_id_field].isna()]
        if not null_external_ids.empty:
            issues.append(
                ValidationIssue(
                    severity=ValidationSeverity.ERROR,
                    field_name=external_id_field,
                    message=f"External ID field has {len(null_external_ids)} null values",
                    affected_records=null_external_ids.index.astype(str).tolist()[:10],
                    suggestion=f"Provide values for external ID field {external_id_field}",
                )
            )

        return issues

    async def _validate_record_limits(
        self, data: pd.DataFrame, operation_type: str
    ) -> List[ValidationIssue]:
        """Validate against Salesforce record limits."""
        issues = []

        # Bulk API limits
        bulk_limit = 10000000  # 10M records per 24 hours
        if len(data) > bulk_limit:
            issues.append(
                ValidationIssue(
                    severity=ValidationSeverity.ERROR,
                    field_name="record_count",
                    message=f"Record count ({len(data):,}) exceeds bulk API daily limit ({bulk_limit:,})",
                    suggestion="Split data into smaller batches or use multiple days",
                )
            )

        # Single batch limit
        batch_limit = 10000
        if len(data) > batch_limit:
            issues.append(
                ValidationIssue(
                    severity=ValidationSeverity.WARNING,
                    field_name="record_count",
                    message=f"Large batch size ({len(data):,}) may require chunking",
                    suggestion="Consider processing in smaller batches for better performance",
                )
            )

        return issues

    async def _validate_duplicates(
        self, data: pd.DataFrame, key_field: str
    ) -> List[ValidationIssue]:
        """Validate for duplicates within the dataset."""
        issues = []

        if key_field not in data.columns:
            return issues

        # Find duplicates
        duplicates = data[data.duplicated(subset=[key_field], keep=False)]
        if not duplicates.empty:
            duplicate_values = duplicates[key_field].value_counts()

            issues.append(
                ValidationIssue(
                    severity=ValidationSeverity.WARNING,
                    field_name=key_field,
                    message=f"Found {len(duplicate_values)} duplicate values in {key_field}",
                    affected_records=duplicates.index.astype(str).tolist()[:10],
                    suggestion=f"Remove or consolidate duplicate {key_field} values",
                )
            )

        return issues

    def _calculate_completeness_score(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> float:
        """Calculate data completeness score."""
        if data.empty:
            return 0.0

        total_fields = len([f for f in field_map.keys() if f in data.columns])
        if total_fields == 0:
            return 0.0

        completeness_scores = []
        for field_name in data.columns:
            if field_name in field_map:
                non_null_ratio = data[field_name].count() / len(data)
                completeness_scores.append(non_null_ratio)

        return np.mean(completeness_scores) * 100 if completeness_scores else 0.0

    def _calculate_validity_score(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> float:
        """Calculate data validity score based on format and type compliance."""
        if data.empty:
            return 0.0

        validity_scores = []

        for field_name, field in field_map.items():
            if field_name not in data.columns:
                continue

            series = data[field_name].dropna()
            if series.empty:
                continue

            valid_count = len(series)

            # Type validation
            if field.type in ["int", "double", "currency", "percent"]:
                try:
                    pd.to_numeric(series, errors="raise")
                except ValueError:
                    valid_count -= len(
                        series[pd.to_numeric(series, errors="coerce").isna()]
                    )

            # Format validation
            if "email" in field_name.lower():
                valid_count -= len(
                    [v for v in series if not self.email_pattern.match(str(v))]
                )

            validity_ratio = valid_count / len(series) if len(series) > 0 else 1.0
            validity_scores.append(validity_ratio)

        return np.mean(validity_scores) * 100 if validity_scores else 100.0

    def _calculate_consistency_score(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> float:
        """Calculate data consistency score."""
        if data.empty:
            return 0.0

        consistency_scores = []

        for field_name in data.columns:
            if field_name in field_map:
                series = data[field_name].dropna().astype(str)
                if series.empty:
                    continue

                # Measure consistency in formats
                if len(series) > 1:
                    # Simple consistency check: standard deviation of string lengths
                    lengths = series.str.len()
                    if lengths.std() == 0:  # All same length
                        consistency_scores.append(1.0)
                    else:
                        # Normalize by mean length
                        consistency = 1.0 - min(1.0, lengths.std() / lengths.mean())
                        consistency_scores.append(consistency)
                else:
                    consistency_scores.append(1.0)

        return np.mean(consistency_scores) * 100 if consistency_scores else 100.0

    def _calculate_accuracy_score(
        self, data: pd.DataFrame, field_map: Dict[str, SalesforceField]
    ) -> float:
        """Calculate data accuracy score (simplified heuristic)."""
        # This is a simplified accuracy check
        # In practice, accuracy would require external validation sources

        if data.empty:
            return 0.0

        accuracy_scores = []

        # Check for obvious data quality issues
        for field_name in data.columns:
            if field_name in field_map:
                series = data[field_name].dropna().astype(str)
                if series.empty:
                    continue

                # Check for placeholder values
                placeholder_patterns = [
                    "test",
                    "example",
                    "xxx",
                    "n/a",
                    "tbd",
                    "unknown",
                ]
                placeholder_count = 0

                for value in series:
                    if any(
                        pattern in value.lower() for pattern in placeholder_patterns
                    ):
                        placeholder_count += 1

                accuracy = 1.0 - (placeholder_count / len(series))
                accuracy_scores.append(accuracy)

        return np.mean(accuracy_scores) * 100 if accuracy_scores else 100.0

    def _analyze_field_quality(
        self, series: pd.Series, field: SalesforceField
    ) -> Dict[str, Any]:
        """Analyze quality metrics for a specific field."""
        analysis = {
            "field_name": field.name,
            "field_type": field.type,
            "total_records": len(series),
            "non_null_count": series.count(),
            "null_count": series.isna().sum(),
            "completeness_rate": series.count() / len(series) if len(series) > 0 else 0,
            "unique_count": series.nunique(),
            "uniqueness_rate": series.nunique() / series.count()
            if series.count() > 0
            else 0,
        }

        # Type-specific analysis
        non_null_series = series.dropna()
        if not non_null_series.empty:
            if field.type in ["string", "textarea"]:
                str_series = non_null_series.astype(str)
                analysis.update(
                    {
                        "avg_length": str_series.str.len().mean(),
                        "min_length": str_series.str.len().min(),
                        "max_length": str_series.str.len().max(),
                        "empty_strings": (str_series.str.strip() == "").sum(),
                    }
                )

            elif field.type in ["int", "double", "currency"]:
                try:
                    numeric_series = pd.to_numeric(non_null_series, errors="coerce")
                    analysis.update(
                        {
                            "min_value": numeric_series.min(),
                            "max_value": numeric_series.max(),
                            "mean_value": numeric_series.mean(),
                            "invalid_numeric": numeric_series.isna().sum(),
                        }
                    )
                except Exception:
                    analysis["numeric_conversion_error"] = True

        return analysis

    def _generate_quality_recommendations(
        self,
        data: pd.DataFrame,
        field_map: Dict[str, SalesforceField],
        completeness: float,
        validity: float,
        consistency: float,
        accuracy: float,
    ) -> List[str]:
        """Generate data quality improvement recommendations."""
        recommendations = []

        if completeness < 80:
            recommendations.append(
                f"Low completeness score ({completeness:.1f}%). "
                "Consider filling missing values or marking optional fields."
            )

        if validity < 90:
            recommendations.append(
                f"Validity issues detected ({validity:.1f}%). "
                "Review data formats and types before import."
            )

        if consistency < 85:
            recommendations.append(
                f"Consistency issues found ({consistency:.1f}%). "
                "Standardize data formats across all records."
            )

        if accuracy < 90:
            recommendations.append(
                f"Potential accuracy issues ({accuracy:.1f}%). "
                "Review for placeholder or test data."
            )

        # Field-specific recommendations
        for field_name in data.columns:
            if field_name in field_map:
                series = data[field_name]
                null_rate = series.isna().sum() / len(series)

                if null_rate > 0.5:
                    recommendations.append(
                        f"Field '{field_name}' has high null rate ({null_rate:.1%}). "
                        "Consider if this field is necessary."
                    )

        return recommendations


# Helper functions for integration


async def validate_before_salesforce_operation(
    sf_integration: SalesforceIntegration,
    data: pd.DataFrame,
    object_name: str,
    operation_type: str,
    external_id_field: Optional[str] = None,
) -> ValidationResult:
    """Convenience function to validate data before Salesforce operations."""
    validator = SalesforceDataValidator(sf_integration)
    return await validator.validate_for_workflow(
        data, object_name, operation_type, external_id_field
    )


async def get_data_quality_report(
    sf_integration: SalesforceIntegration, data: pd.DataFrame, object_name: str
) -> DataQualityReport:
    """Convenience function to get data quality report."""
    validator = SalesforceDataValidator(sf_integration)
    return await validator.analyze_data_quality(data, object_name)


def fix_auto_fixable_issues(
    data: pd.DataFrame, validation_result: ValidationResult
) -> pd.DataFrame:
    """Automatically fix issues that can be auto-corrected."""
    fixed_data = data.copy()

    for issue in validation_result.issues:
        if not issue.auto_fixable:
            continue

        # Fix field length issues by truncating
        if "exceed maximum length" in issue.message:
            # Extract max length from message
            import re

            length_match = re.search(r"maximum length of (\d+)", issue.message)
            if length_match:
                max_length = int(length_match.group(1))
                fixed_data[issue.field_name] = (
                    fixed_data[issue.field_name].astype(str).str[:max_length]
                )
                log.info(
                    f"Auto-fixed length issue in {issue.field_name} by truncating to {max_length} characters"
                )

    return fixed_data
