"""
CXR Kernel - Full Context Layer Implementation (Heavyweight Solution)
Version: 3.1 + Full Context
Purpose: Transform rule engine into intelligent reasoning system

7 Context Types:
1. Patient Context - Historical patterns, prior denials
2. Provider Context - Performance, specialty, training needs  
3. Payer Context - Payer-specific denial rates, strictness
4. Temporal Context - Seasonality, claim age, urgency
5. Financial Context - Outlier detection, amount validation
6. Relationship Context - Bundling, family claims, resubmissions
7. Policy Context - CMS changes, effective dates, transitions
"""

import pyodbc
import json
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple
import statistics
from qdrant_client import QdrantClient
from sentence_transformers import SentenceTransformer
import torch
import sys
import os

# Add current directory to path for correction_engine import
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from correction_engine import CorrectionEngine, extract_policy_snippets

# =====================================================================================
# v3.2 FIX: THRESHOLD CALIBRATION & DECISION LOGIC
# =====================================================================================

ARCHETYPE_THRESHOLDS = {
    "DEFAULT":               (0.70, 0.40),
    "PTP_SEPARATE_PROCEDURE":(0.80, 0.45),
    "MUE_Risk":              (0.85, 0.50),
    "COVERAGE":              (0.75, 0.40),
    "FINANCIAL_OUTLIER":     (0.78, 0.40),
    "POST_OP_COMPLICATION":  (0.70, 0.35),
    "HIGH_RISK_PROVIDER":    (0.75, 0.40),
    "Compliant":             (0.60, 0.30),
}

def _get_archetype_thresholds(archetype_code: str) -> Tuple[float, float]:
    """v3.2 FIX: Get archetype-aware thresholds with fallback to DEFAULT"""
    return ARCHETYPE_THRESHOLDS.get(archetype_code, ARCHETYPE_THRESHOLDS["DEFAULT"])

def _confidence_interval(score: float, k_sources: int = 3) -> Tuple[float, float]:
    """v3.2 FIX: Calculate 95% confidence interval for fused score"""
    k_sources = max(1, k_sources)
    se = (score * (1.0 - score) / k_sources) ** 0.5
    margin = 1.96 * se
    return max(0.0, score - margin), min(1.0, score + margin)

def _detect_conflict(structured_conf: float, semantic_conf: float) -> bool:
    """v3.2 FIX: Detect when rule and semantic evidence strongly conflict"""
    return (structured_conf <= 0.20 and semantic_conf >= 0.80) or \
           (structured_conf >= 0.80 and semantic_conf <= 0.20)

def _dynamic_fusion_weights(context: dict, archetype_code: str) -> dict:
    """v3.2 FIX: Dynamic weights based on risk and archetype type"""
    overall_risk = context.get("overall_risk_score", 0.0)
    
    # High-risk claims need more context influence
    if "PTP" in archetype_code or "SURG" in archetype_code or overall_risk >= 0.5:
        return {"rule": 0.40, "vector": 0.20, "context": 0.40}
    if "MUE" in archetype_code:
        return {"rule": 0.55, "vector": 0.25, "context": 0.20}
    # Default balanced
    return {"rule": 0.60, "vector": 0.25, "context": 0.15}

def _fuse_scores(rule_conf: float, vec_conf: float, ctx_conf: float, weights: dict) -> float:
    """v3.2 FIX: Fuse three evidence sources with dynamic weights"""
    return (
        rule_conf * weights["rule"] +
        vec_conf  * weights["vector"] +
        ctx_conf  * weights["context"]
    )

def _context_adjustment_v32(context_summary: dict) -> float:
    """v3.2 FIX: Stronger, consistent context penalties (range -0.30 to 0.0)"""
    adj = 0.0
    
    # Provider risk
    prov_rate = context_summary.get("provider_denial_rate", 0.0)
    if prov_rate >= 0.8:        adj -= 0.20
    elif prov_rate >= 0.5:      adj -= 0.10

    # Payer strictness
    payer_rate = context_summary.get("payer_denial_rate", 0.0)
    if payer_rate >= 0.7:       adj -= 0.10

    # Financial outlier
    z = context_summary.get("financial_zscore", 0.0)
    if abs(z) > 3.0:            adj -= 0.10
    elif abs(z) > 2.5:          adj -= 0.05

    # Temporal recency
    days_active = context_summary.get("policy_days_active", 365)
    if days_active < 30:        adj -= 0.05

    return max(-0.30, min(0.0, adj))

def _risk_override(decision: str, overall_risk: float) -> str:
    """v3.2 FIX: Sliding risk override without hard cutoff at 0.7"""
    if overall_risk >= 0.75 and decision == "AUTO_CORRECT":
        return "MANUAL_REVIEW"
    if overall_risk >= 0.65 and decision == "AUTO_CORRECT":
        return "MANUAL_REVIEW"
    return decision

def _detect_additional_archetypes(claim: dict, context_summary: dict) -> list:
    """v3.2 FIX: Detect post-op, financial outlier, high-risk provider archetypes"""
    matches = []

    # Post-op complication via ICD Z98/Z99 codes
    icds = claim.get("icd_codes", []) or [claim.get("icd")] if claim.get("icd") else []
    if any(str(code).upper().startswith(("Z98", "Z99")) for code in icds):
        matches.append("POST_OP_COMPLICATION")

    # Financial outlier via z-score
    z = context_summary.get("financial_zscore", 0.0)
    if abs(z) > 3.0:
        matches.append("FINANCIAL_OUTLIER")

    # High-risk provider
    prov_rate = context_summary.get("provider_denial_rate", 0.0)
    if prov_rate >= 0.5:
        matches.append("HIGH_RISK_PROVIDER")

    return matches

# =====================================================================================
# CONFIGURATION
# =====================================================================================

SERVER = "localhost,1433"
DATABASE = "_archetype"
USERNAME = "SA"
PASSWORD = "Bbanwo@1980!"

# =====================================================================================
# CONTEXT COLLECTOR (7 Context Types)
# =====================================================================================

class ContextCollector:
    """
    Comprehensive context gathering for intelligent reasoning
    Implements all 7 context types
    """
    
    def __init__(self, conn):
        self.conn = conn
    
    def gather_full_context(self, claim: Dict, archetype_code: str) -> Dict:
        """
        Gather all 7 types of context
        
        Returns: Complete context dictionary with all factors
        """
        
        print(f"\n  Gathering Context (7 types):")
        print("  " + "-" * 68)
        
        context = {
            '1_patient': self.get_patient_context(claim),
            '2_provider': self.get_provider_context(claim),
            '3_payer': self.get_payer_context(claim, archetype_code),
            '4_temporal': self.get_temporal_context(claim),
            '5_financial': self.get_financial_context(claim),
            '6_relationship': self.get_relationship_context(claim),
            '7_policy': self.get_policy_context(claim, archetype_code)
        }
        
        # Calculate aggregate scores
        context['overall_risk_score'] = self._calculate_overall_risk(context)
        context['confidence_adjustment'] = self._calculate_confidence_adjustment(context)
        
        print(f"  [OK] Overall Risk Score: {context['overall_risk_score']:.3f}")
        print(f"  [OK] Confidence Adjustment: {context['confidence_adjustment']:+.3f}")
        
        return context
    
    # =================================================================================
    # 1. PATIENT CONTEXT
    # =================================================================================
    
    def get_patient_context(self, claim: Dict) -> Dict:
        """
        Patient claim history and denial patterns
        """
        patient_id = claim.get('patient_id', 'UNKNOWN')
        
        # Mock data - replace with actual queries to your claims system
        # In production: Query your claims database for this patient
        
        patient_context = {
            'patient_id': patient_id,
            'patient_gender': claim.get('patient_gender', 'U'),
            'patient_age': claim.get('patient_age', 50),
            'patient_total_claims_12mo': 5,  # Would query: SELECT COUNT(*)...
            'patient_prior_denials_total': 1,  # Would query past denials
            'patient_similar_denials_6mo': 0,  # Same archetype denials
            'patient_denial_rate': 0.20,  # 1/5 = 20%
            'patient_risk_flag': False,
            'patient_chronic_conditions': claim.get('chronic_conditions', '')
        }
        
        print(f"    1. Patient: {patient_context['patient_total_claims_12mo']} claims, "
              f"{patient_context['patient_denial_rate']:.0%} denial rate")
        
        return patient_context
    
    # =================================================================================
    # 2. PROVIDER CONTEXT
    # =================================================================================
    
    def get_provider_context(self, claim: Dict) -> Dict:
        """
        Provider performance and specialty validation
        """
        provider_id = claim.get('provider_id', 'UNKNOWN')
        
        # Mock data - in production, query provider performance from CXR_Claims_Analysis
        cursor = self.conn.cursor()
        
        # Try to get actual provider stats from logged claims
        try:
            query = """
            SELECT 
                COUNT(*) as total,
                SUM(CASE WHEN resolved = 0 THEN 1 ELSE 0 END) as denials
            FROM CXR_Claims_Analysis
            WHERE claim_id LIKE ?
              AND detected_on > DATEADD(MONTH, -6, GETDATE())
            """
            
            cursor.execute(query, (f'%{provider_id}%',))
            row = cursor.fetchone()
            
            total_claims = row.total if row.total else 10
            denials = row.denials if row.denials else 2
            denial_rate = denials / max(total_claims, 1)
            
        except:
            total_claims = 10
            denials = 2
            denial_rate = 0.20
        
        cursor.close()
        
        provider_context = {
            'provider_id': provider_id,
            'provider_npi': claim.get('provider_npi', ''),
            'provider_specialty': claim.get('provider_specialty', 'General'),
            'provider_facility_type': claim.get('facility_type', 'Office'),
            'provider_total_claims_6mo': total_claims,
            'provider_denial_count_6mo': denials,
            'provider_denial_rate': denial_rate,
            'provider_risk_level': 'HIGH' if denial_rate > 0.3 else 'MODERATE' if denial_rate > 0.15 else 'LOW',
            'provider_needs_training': denial_rate > 0.3,
            'provider_avg_claim_amount': 2500.00
        }
        
        print(f"    2. Provider: {provider_context['provider_denial_rate']:.0%} denial rate "
              f"({provider_context['provider_risk_level']} risk)")
        
        return provider_context
    
    # =================================================================================
    # 3. PAYER CONTEXT
    # =================================================================================
    
    def get_payer_context(self, claim: Dict, archetype_code: str) -> Dict:
        """
        Payer-specific denial patterns for this archetype
        """
        payer_id = claim.get('payer_id', 'UNKNOWN')
        payer_name = claim.get('payer_name', 'Unknown Payer')
        
        # v3.2 FIX: Use REALISTIC defaults instead of hardcoded 65%
        # Different payers have different archetype-specific denial rates
        # Clean/Compliant archetypes: typically 15-25% denial
        # High-risk archetypes (e.g., MUE, Coverage): typically 40-60% denial
        
        # Determine realistic payer denial rate based on archetype
        if any(x in archetype_code for x in ['DEFAULT', 'COMPLIANT', 'CLEAN']):
            # Low-risk archetype → lower payer denial rate
            payer_arch_denial_rate = 0.20  # 20% (realistic for standard claims)
        elif any(x in archetype_code for x in ['MUE', 'PTP', 'COVERAGE', 'MEDICAL_NECESSITY']):
            # Higher-risk archetype → higher payer denial rate
            payer_arch_denial_rate = 0.50  # 50% (realistic for policy-driven denials)
        else:
            # Default to moderate
            payer_arch_denial_rate = 0.30  # 30% (middle ground)
        
        # Mock data - in production, query payer-archetype specific patterns
        payer_context = {
            'payer_id': payer_id,
            'payer_name': payer_name,
            'payer_type': claim.get('payer_type', 'Commercial'),
            'payer_overall_denial_rate': 0.25,
            'payer_denial_rate_for_archetype': payer_arch_denial_rate,
            'payer_is_strict': payer_arch_denial_rate > 0.4,  # Strict if >40%
            'payer_avg_processing_days': 45,
            'payer_requires_extra_docs': payer_arch_denial_rate > 0.4
        }
        
        print(f"    3. Payer: {payer_context['payer_denial_rate_for_archetype']:.0%} denial rate "
              f"for {archetype_code} (strict: {payer_context['payer_is_strict']})")
        
        return payer_context
    
    # =================================================================================
    # 4. TEMPORAL CONTEXT
    # =================================================================================
    
    def get_temporal_context(self, claim: Dict) -> Dict:
        """
        Time-based context (seasonality, claim age, urgency)
        """
        service_date = claim.get('service_date', datetime.now().date())
        submission_date = datetime.now().date()
        
        if isinstance(service_date, str):
            service_date = datetime.fromisoformat(service_date).date()
        elif isinstance(service_date, datetime):
            service_date = service_date.date()
        
        claim_age = (submission_date - service_date).days
        
        # Determine quarter
        month = submission_date.month
        quarter = f"{submission_date.year}Q{(month-1)//3 + 1}"
        
        # Timely filing (typically 365 days)
        filing_deadline = service_date + timedelta(days=365)
        days_until_deadline = (filing_deadline - submission_date).days
        
        # Claim staleness
        if claim_age < 30:
            staleness = 'FRESH'
        elif claim_age < 90:
            staleness = 'AGING'
        else:
            staleness = 'STALE'
        
        temporal_context = {
            'service_date': service_date.isoformat(),
            'submission_date': submission_date.isoformat(),
            'claim_age_days': claim_age,
            'claim_staleness': staleness,
            'claim_quarter': quarter,
            'is_Q4': quarter.endswith('Q4'),
            'is_end_of_year': submission_date.month == 12,
            'is_audit_period': submission_date.month in [9, 10, 11],  # Sept-Nov audit season
            'days_until_timely_filing': days_until_deadline,
            'timely_filing_risk': days_until_deadline < 30
        }
        
        print(f"    4. Temporal: {staleness} claim ({claim_age} days old), "
              f"{quarter}, timely filing: {days_until_deadline} days")
        
        return temporal_context
    
    # =================================================================================
    # 5. FINANCIAL CONTEXT
    # =================================================================================
    
    def get_financial_context(self, claim: Dict) -> Dict:
        """
        Financial outlier detection and validation
        """
        claim_amount = claim.get('total_amount', 0)
        
        # Get average for this procedure (mock - query your data in production)
        # In production: Query actual averages from claims history
        avg_amount = 2500.00  # Would be: SELECT AVG(amount) WHERE cpt_code = ?
        std_dev = 800.00
        
        # Calculate outlier metrics
        amount_ratio = claim_amount / max(avg_amount, 1)
        z_score = (claim_amount - avg_amount) / max(std_dev, 1)
        is_outlier = abs(z_score) > 3
        
        if abs(z_score) > 5:
            outlier_severity = 'HIGH'
        elif abs(z_score) > 3:
            outlier_severity = 'MODERATE'
        else:
            outlier_severity = 'LOW'
        
        financial_context = {
            'claim_amount': claim_amount,
            'procedure_avg_amount': avg_amount,
            'amount_vs_avg_ratio': amount_ratio,
            'amount_std_dev': std_dev,
            'z_score': z_score,
            'is_financial_outlier': is_outlier,
            'outlier_severity': outlier_severity,
            'requires_justification': is_outlier
        }
        
        print(f"    5. Financial: ${claim_amount:,.2f} ({amount_ratio:.1f}x avg), "
              f"outlier: {is_outlier}, z-score: {z_score:.1f}")
        
        return financial_context
    
    # =================================================================================
    # 6. RELATIONSHIP CONTEXT
    # =================================================================================
    
    def get_relationship_context(self, claim: Dict) -> Dict:
        """
        Related claims, bundling, resubmissions
        """
        # Mock data - in production, query for same-day claims, family claims
        same_day = claim.get('same_day_claims_count', 0)
        
        relationship_context = {
            'same_day_claims_count': same_day,
            'family_claims_same_day': 0,
            'is_bundled_service': same_day > 0,
            'bundling_risk_flag': same_day > 5,
            'is_resubmission': claim.get('is_resubmission', False),
            'previous_submission_id': claim.get('previous_claim_id', ''),
            'resubmission_count': 0,
            'related_claims_count': same_day
        }
        
        print(f"    6. Relationship: {same_day} same-day claims, "
              f"bundling risk: {relationship_context['bundling_risk_flag']}")
        
        return relationship_context
    
    # =================================================================================
    # 7. POLICY CONTEXT
    # =================================================================================
    
    def get_policy_context(self, claim: Dict, archetype_code: str) -> Dict:
        """
        CMS policy change context and validity
        """
        cursor = self.conn.cursor()
        
        # Get archetype's effective date
        cursor.execute("""
            SELECT effective_date, version
            FROM CXR_Archetypes
            WHERE archetype_code = ?
        """, (archetype_code,))
        
        row = cursor.fetchone()
        cursor.close()
        
        if row and row.effective_date:
            effective_date = row.effective_date
            days_since_effective = (datetime.now().date() - effective_date).days
            is_new = days_since_effective < 90
            
            # Check if policy was valid at service date
            service_date = claim.get('service_date', datetime.now().date())
            if isinstance(service_date, str):
                service_date = datetime.fromisoformat(service_date).date()
            elif isinstance(service_date, datetime):
                service_date = service_date.date()
            
            policy_valid = service_date >= effective_date
        else:
            days_since_effective = 365
            is_new = False
            policy_valid = True
            effective_date = datetime.now().date()
        
        policy_context = {
            'policy_effective_date': effective_date.isoformat() if isinstance(effective_date, datetime) else str(effective_date),
            'days_since_policy_effective': days_since_effective,
            'is_new_policy': is_new,
            'policy_version': row.version if row else '2025Q1',
            'cms_update_quarter': f"{datetime.now().year}Q{(datetime.now().month-1)//3 + 1}",
            'policy_valid_at_service': policy_valid,
            'lcd_jurisdiction': claim.get('jurisdiction', 'N/A')
        }
        
        print(f"    7. Policy: Effective {days_since_effective} days ago, "
              f"valid at service: {policy_valid}, new: {is_new}")
        
        return policy_context
    
    # =================================================================================
    # CONTEXT AGGREGATION
    # =================================================================================
    
    def _calculate_overall_risk(self, context: Dict) -> float:
        """
        Calculate aggregate risk score from all contexts (0.0-1.0)
        Higher = more likely to be denied despite archetype match
        """
        risk_components = []
        
        # Patient risk
        patient = context.get('1_patient', {})
        if patient.get('patient_risk_flag'):
            risk_components.append(0.4)
        elif patient.get('patient_denial_rate', 0) > 0.3:
            risk_components.append(0.3)
        
        # Provider risk  
        provider = context.get('2_provider', {})
        provider_rate = provider.get('provider_denial_rate', 0)
        if provider_rate > 0.4:
            risk_components.append(0.5)
        elif provider_rate > 0.2:
            risk_components.append(0.3)
        
        # Payer risk (weighted heavily!)
        payer = context.get('3_payer', {})
        payer_arch_rate = payer.get('payer_denial_rate_for_archetype', 0)
        risk_components.append(payer_arch_rate * 0.8)  # Payer archetype-specific rate is key
        
        # Financial risk
        financial = context.get('5_financial', {})
        if financial.get('is_financial_outlier'):
            outlier_severity = financial.get('outlier_severity', 'LOW')
            if outlier_severity == 'HIGH':
                risk_components.append(0.6)
            elif outlier_severity == 'MODERATE':
                risk_components.append(0.4)
        
        # Temporal risk
        temporal = context.get('4_temporal', {})
        if temporal.get('is_Q4'):
            risk_components.append(0.2)
        if temporal.get('claim_staleness') == 'STALE':
            risk_components.append(0.3)
        
        # Relationship risk
        relationship = context.get('6_relationship', {})
        if relationship.get('bundling_risk_flag'):
            risk_components.append(0.4)
        
        # Policy risk
        policy = context.get('7_policy', {})
        if policy.get('is_new_policy'):
            risk_components.append(0.25)
        if not policy.get('policy_valid_at_service'):
            risk_components.append(0.8)  # Major risk!
        
        # Average risk
        if risk_components:
            overall_risk = sum(risk_components) / len(risk_components)
        else:
            overall_risk = 0.3  # Default moderate risk
        
        return min(1.0, overall_risk)
    
    def _calculate_confidence_adjustment(self, context: Dict) -> float:
        """
        Calculate confidence adjustment based on context (-0.15 to +0.15)
        Negative = higher denial risk, Positive = lower denial risk
        
        Note: Reduced from ±0.3 to ±0.15 to prevent context from dominating fusion
        """
        adjustment = 0.0
        
        # Patient history (reduced from 0.15 → 0.08)
        patient = context.get('1_patient', {})
        similar_denials = patient.get('patient_similar_denials_6mo', 0)
        if similar_denials >= 3:
            adjustment -= 0.08  # Pattern of this specific denial
        elif similar_denials == 0 and patient.get('patient_denial_rate', 0) < 0.1:
            adjustment += 0.03  # Clean history
        
        # Provider performance (reduced from 0.10 → 0.05)
        provider = context.get('2_provider', {})
        provider_rate = provider.get('provider_denial_rate', 0)
        if provider_rate > 0.4:
            adjustment -= 0.05  # High-risk provider
        elif provider_rate < 0.05:
            adjustment += 0.05  # Excellent provider
        
        # Payer behavior (reduced from 0.20 → 0.10)
        payer = context.get('3_payer', {})
        payer_rate = payer.get('payer_denial_rate_for_archetype', 0)
        if payer_rate > 0.7:
            adjustment -= 0.10  # Very strict payer for this archetype
        elif payer_rate < 0.1:
            adjustment += 0.08  # Lenient payer
        
        # Temporal factors (reduced from 0.05/0.10 → 0.03/0.05)
        temporal = context.get('4_temporal', {})
        if temporal.get('is_Q4'):
            adjustment -= 0.03  # Higher scrutiny
        if temporal.get('timely_filing_risk'):
            adjustment -= 0.05  # Urgency increases risk
        if temporal.get('claim_staleness') == 'STALE':
            adjustment -= 0.05  # Old claims get more scrutiny
        
        # Financial outliers (only penalize if SEVERE, not test data)
        financial = context.get('5_financial', {})
        outlier_severity = financial.get('outlier_severity', 'LOW')
        if financial.get('is_financial_outlier') and outlier_severity == 'HIGH':
            adjustment -= 0.03  # Only penalize severe outliers
        
        # Policy validity (reduced from 0.30/0.10 → 0.15/0.05)
        policy = context.get('7_policy', {})
        if not policy.get('policy_valid_at_service'):
            adjustment -= 0.15  # Policy not active = very high risk
        if policy.get('is_new_policy'):
            adjustment -= 0.05  # New policies = uncertainty
        
        # Clamp to range (reduced from ±0.3 → ±0.15)
        return max(-0.15, min(0.15, adjustment))

# =====================================================================================
# ENHANCED CXR KERNEL WITH FULL CONTEXT
# =====================================================================================

class CXRKernelFullContext:
    """
    Production CXR Kernel with comprehensive context layer
    
    Features:
    - Two-phase archetype detection (archetype_7 → archetype_24)
    - 7-type context gathering
    - Evidence fusion (rule + vector + context)
    - Context-aware decision making
    - Full context logging to SQL
    """
    
    def __init__(self, qdrant_url: str = "http://localhost:6333"):
        connection_string = (
            f"DRIVER={{ODBC Driver 17 for SQL Server}};"
            f"SERVER={SERVER};"
            f"DATABASE={DATABASE};"
            f"UID={USERNAME};"
            f"PWD={PASSWORD};"
            "TrustServerCertificate=yes;"
        )
        
        self.conn = pyodbc.connect(connection_string)
        self.context_collector = ContextCollector(self.conn)
        
        # Initialize Qdrant client
        try:
            self.qdrant_client = QdrantClient(url=qdrant_url)
            collections = [c.name for c in self.qdrant_client.get_collections().collections]
            self.policy_collections = [c for c in collections if c.startswith("claims__")]
            self.qdrant_available = True
        except Exception as e:
            print(f"   Warning: Qdrant not available: {e}")
            self.qdrant_client = None
            self.policy_collections = []
            self.qdrant_available = False
        
        # Initialize embedder
        try:
            self.embedder = SentenceTransformer(
                "nomic-ai/nomic-embed-text-v1.5",
                device="cuda" if torch.cuda.is_available() else "cpu",
                trust_remote_code=True
            )
            self.embedder_available = True
        except Exception as e:
            print(f"   Warning: Embedder not available: {e}")
            self.embedder = None
            self.embedder_available = False
        
        # Load archetypes
        self.archetypes_by_category = self._load_archetypes()
        
        # Initialize correction engine
        self.correction_engine = CorrectionEngine()
        
        # Evidence fusion weights
        self.fusion_weights = {
            'rule': 0.6,
            'vector': 0.3,
            'context': 0.1
        }
        
        print(f"[OK] CXR Kernel Full Context initialized")
        print(f"   Archetypes: {sum(len(v) for v in self.archetypes_by_category.values())}")
        print(f"   Context Types: 7")
        print(f"   Qdrant Collections: {len(self.policy_collections)}")
        print(f"   Evidence Fusion: rule(60%) + vector(30%) + context(10%)")
    
    def _load_archetypes(self):
        """Load archetypes grouped by archetype_7"""
        cursor = self.conn.cursor()
        
        query = """
        SELECT archetype_7, archetype_code, archetype_json
        FROM CXR_Archetypes
        WHERE status = 'active'
        ORDER BY archetype_7, priority_level
        """
        
        cursor.execute(query)
        
        grouped = {}
        for row in cursor.fetchall():
            archetype_7 = row.archetype_7
            if archetype_7 not in grouped:
                grouped[archetype_7] = []
            
            arch_data = json.loads(row.archetype_json) if row.archetype_json else {}
            arch_data['archetype_code'] = row.archetype_code
            arch_data['archetype_7'] = archetype_7
            grouped[archetype_7].append(arch_data)
        
        cursor.close()
        return grouped
    
    # =================================================================================
    # SEMANTIC EVIDENCE GATHERING (Qdrant)
    # =================================================================================
    
    def _get_semantic_evidence(self, claim: Dict, archetype_code: str) -> tuple:
        """
        Get semantic evidence from Qdrant policy collections
        
        Returns: (vector_confidence, semantic_docs)
        """
        
        if not self.qdrant_available or not self.embedder_available:
            # Fallback to placeholder if Qdrant not available
            print(f"   [QDRANT] Qdrant available: {self.qdrant_available}, Embedder available: {self.embedder_available}")
            return (0.85, [])
        
        try:
            # Build query based on claim
            query_parts = []
            if claim.get('cpt'):
                query_parts.append(f"CPT {claim['cpt']}")
            if claim.get('icd10_code'):
                query_parts.append(f"ICD {claim['icd10_code']}")
            if archetype_code:
                query_parts.append(archetype_code.replace('_', ' '))
            
            query_text = " ".join(query_parts)
            
            # If no query parts, use generic search
            if not query_text.strip():
                query_text = "medical claim review"
            
            print(f"   [QDRANT] Query text: '{query_text}'")
            print(f"   [QDRANT] Embedding query with GPU...")
            
            # Embed query
            query_vector = self.embedder.encode(query_text).tolist()
            
            print(f"   [QDRANT] Vector size: {len(query_vector)}")
            print(f"   [QDRANT] Searching {len(self.policy_collections[:3])} collections...")
            
            # Search across policy collections
            all_results = []
            for collection in self.policy_collections[:3]:  # Limit to 3 collections for speed
                try:
                    print(f"   [QDRANT] Searching collection: {collection}")
                    results = self.qdrant_client.query_points(
                        collection_name=collection,
                        query=query_vector,
                        limit=5,
                        score_threshold=0.5,  # Lower threshold to get more results
                        with_payload=True,
                        with_vectors=False
                    ).points
                    print(f"   [QDRANT] Found {len(results)} results in {collection}")
                    
                    for hit in results:
                        doc = hit.payload.copy() if hit.payload else {}
                        doc['score'] = hit.score
                        doc['collection'] = collection
                        all_results.append(doc)
                
                except Exception as e:
                    print(f"   Warning: Search failed for {collection}: {e}")
                    continue
            
            # Sort by score
            all_results.sort(key=lambda x: x.get('score', 0), reverse=True)
            
            # Calculate average confidence from top results
            if all_results:
                top_scores = [r['score'] for r in all_results[:5]]
                vector_confidence = sum(top_scores) / len(top_scores)
                print(f"   [QDRANT] Total results: {len(all_results)}, Top scores: {[f'{s:.3f}' for s in top_scores]}")
                print(f"   [QDRANT] Vector confidence: {vector_confidence:.3f}")
            else:
                vector_confidence = 0.50  # Low confidence if no matches
                print(f"   [QDRANT] NO RESULTS FOUND - using fallback 0.50")
            
            return (vector_confidence, all_results[:5])
        
        except Exception as e:
            print(f"   Warning: Semantic evidence gathering failed: {e}")
            return (0.85, [])
    
    # =================================================================================
    # MAIN PROCESSING WITH FULL CONTEXT
    # =================================================================================
    
    def process_claim(self, claim: Dict, verbose: bool = True) -> Dict:
        """
        Full context-aware claim processing
        
        Returns: Complete result with context influence tracking
        """
        
        if verbose:
            print(f"\n{'='*70}")
            print(f"🧠 CXR Kernel - Full Context Processing")
            print(f"Claim: {claim.get('claim_id')}")
            print(f"{'='*70}")
        
        # PHASE 1 & 2: Detect archetype
        broad_matches = self._detect_broad(claim)
        if not broad_matches:
            return {'clean': True}
        
        specific_matches = self._detect_specific(claim, broad_matches)
        if not specific_matches:
            return {'clean': True}
        
        top_match = specific_matches[0]
        
        if verbose:
            print(f"\n[PHASE] Phase 1-2: Archetype Detection")
            print(f"  [OK] Matched: {top_match['archetype_code']}")
            print(f"  [OK] Base Confidence: {top_match['confidence']:.3f}")
        
        # PHASE 3: GATHER FULL CONTEXT
        context = self.context_collector.gather_full_context(claim, top_match['archetype_code'])
        
        # PHASE 4: EVIDENCE FUSION
        if verbose:
            print(f"\n[PHASE] Phase 3: Evidence Fusion")
            print("  " + "-" * 68)
        
        # Get evidence from all 3 planes
        rule_confidence = top_match['confidence']
        
        # PHASE 3B: Get semantic evidence from Qdrant
        vector_confidence, semantic_docs = self._get_semantic_evidence(claim, top_match['archetype_code'])
        
        context_score = 1.0 - (context['overall_risk_score'] * 0.5)  # Invert risk to score
        
        # v3.2 FIX: Use dynamic fusion weights based on risk and archetype
        weights = _dynamic_fusion_weights(context, top_match['archetype_code'])
        fused_confidence = _fuse_scores(rule_confidence, vector_confidence, context_score, weights)
        
        # v3.2 FIX: Check for conflict between structured and semantic evidence
        is_conflict = _detect_conflict(rule_confidence, vector_confidence)
        if is_conflict:
            print(f"  [!] CONFLICT: Structured ({rule_confidence:.3f}) vs Semantic ({vector_confidence:.3f})")
        
        # v3.2 FIX: Use stronger, more consistent context adjustment
        ctx_adjustment_dict = {
            "provider_denial_rate": context.get('2_provider', {}).get('provider_denial_rate', 0.0),
            "payer_denial_rate": context.get('3_payer', {}).get('payer_denial_rate_for_archetype', 0.0),
            "financial_zscore": context.get('5_financial', {}).get('z_score', 0.0),
            "policy_days_active": 365  # Placeholder; enhance with actual data
        }
        context_adjustment = _context_adjustment_v32(ctx_adjustment_dict)
        
        final_confidence = max(0.0, min(1.0, fused_confidence + context_adjustment))
        
        if verbose:
            print(f"  Rule Confidence:    {rule_confidence:.3f} (weight: {weights['rule']:.0%})")
            print(f"  Vector Confidence:  {vector_confidence:.3f} (weight: {weights['vector']:.0%})")
            print(f"  Context Score:      {context_score:.3f} (weight: {weights['context']:.0%})")
            print(f"  Fused:              {fused_confidence:.3f}")
            print(f"  Context Adjustment: {context_adjustment:+.3f} (v3.2 stronger)")
            print(f"  Final Confidence:   {final_confidence:.3f}")
            if is_conflict:
                print(f"  [WARNING] Conflict detected - forcing MANUAL_REVIEW")
        
        # PHASE 5: CONTEXT-AWARE DECISION
        original_action = top_match.get('recommended_action', 'MANUAL_REVIEW')
        
        # v3.2 FIX: Add archetype to context for decision logic
        context['archetype_code'] = top_match['archetype_code']
        
        # v3.2 FIX: Detect additional archetypes (post-op, financial outlier, high-risk provider)
        additional_archetypes = _detect_additional_archetypes(claim, ctx_adjustment_dict)
        all_detected_archetypes = [top_match['archetype_code']] + additional_archetypes
        
        # v3.2 FIX: If conflict detected, force MANUAL_REVIEW
        if is_conflict:
            final_action = 'MANUAL_REVIEW'
            override_reason = "Conflict: Structured evidence disagrees with semantic evidence"
        else:
            final_action, override_reason = self._make_context_aware_decision(
                original_action, final_confidence, context
            )
        
        if verbose and final_action != original_action:
            print(f"\n  [!] Action Override:")
            print(f"      Original: {original_action}")
            print(f"      Final: {final_action}")
            print(f"      Reason: {override_reason}")
        
        # PHASE 6: Extract corrections and policy support from semantic docs
        # Map final_action to decision string for correction engine
        decision_map_for_engine = {
            'ESCALATE': 'FAIL',
            'MANUAL_REVIEW': 'REVIEW',
            'AUTO_CORRECT': 'PASS',
            'AUTO_APPROVE': 'PASS'
        }
        decision_for_engine = decision_map_for_engine.get(final_action, 'REVIEW')
        
        print(f"   [CORRECTIONS] Decision: {decision_for_engine}, Semantic docs: {len(semantic_docs)}")
        corrections = self.correction_engine.generate(semantic_docs, decision_for_engine, claim)
        print(f"   [CORRECTIONS] Found {len(corrections)} corrections")
        
        policy_support = extract_policy_snippets(semantic_docs, top_n=5)
        print(f"   [POLICY] Found {len(policy_support)} policy snippets")
        
        correction = {
            'corrected': final_action == 'AUTO_CORRECT',
            'amount_saved': claim.get('total_amount', 0) if final_action == 'AUTO_CORRECT' else 0,
            'suggestions': corrections
        }
        
        # PHASE 7: LOG WITH FULL CONTEXT
        analysis_id = self._log_claim_analysis(claim, top_match, final_confidence, final_action, override_reason != None, override_reason, correction)
        self._log_full_context(analysis_id, claim, context)
        
        if verbose:
            print(f"\n  [OK] Logged to SQL (analysis_id: {analysis_id})")
        
        # PHASE 8: PRINT CONTEXT ANALYSIS SUMMARY
        
        # Extract risk factors
        risk_factors = []
        patient = context.get('1_patient', {})
        if patient.get('patient_similar_denials_6mo', 0) >= 3:
            risk_factors.append(f"Patient: {patient.get('patient_similar_denials_6mo', 0)} similar denials in 6mo")
        
        provider = context.get('2_provider', {})
        if provider.get('provider_denial_rate', 0) > 0.3:
            risk_factors.append(f"Provider: {provider.get('provider_denial_rate', 0):.0%} denial rate (HIGH risk)")
        
        payer = context.get('3_payer', {})
        if payer.get('payer_denial_rate_for_archetype', 0) > 0.5:
            risk_factors.append(f"Payer: {payer.get('payer_denial_rate_for_archetype', 0):.0%} denial rate for {top_match['archetype_code']} (strict: {payer.get('payer_is_strict', False)})")
        
        financial = context.get('5_financial', {})
        if financial.get('is_financial_outlier', False):
            z_score = financial.get('z_score', 0)
            risk_factors.append(f"Financial: Outlier claim (z-score: {z_score:.1f})")
        
        policy = context.get('7_policy', {})
        if not policy.get('policy_valid_at_service', True):
            risk_factors.append(f"Policy: Not valid at service date")
        
        # Build summary dict
        summary = {
            "claim_id": claim.get('claim_id', 'UNKNOWN'),
            "archetype": top_match.get('archetype_code', 'UNKNOWN'),
            "detected_archetypes": all_detected_archetypes,  # v3.2 FIX
            "fused_score": round(final_confidence, 3),
            "structured_score": round(rule_confidence, 3),
            "semantic_score": round(vector_confidence, 3),
            "context_score": round(context_score, 3),
            "context_penalty": round(context_adjustment, 3),  # v3.2 stronger penalty
            "decision": final_action,
            "thresholds": {
                "T_pass": round(t_pass, 2),  # v3.2 archetype-specific
                "T_fail": round(t_fail, 2)
            },
            "fusion_weights": weights,  # v3.2 dynamic weights
            "confidence_interval": {  # v3.2 NEW
                "lower": round(ci_low, 3),
                "upper": round(ci_high, 3)
            },
            "conflict_detected": is_conflict,  # v3.2 NEW
            "top_policy_snippets": [
                {
                    "source": s.get('source', 'Unknown'),
                    "confidence": round(s.get('confidence', 0), 3)
                } for s in policy_support[:2]
            ],
            "risk_factors": risk_factors if risk_factors else ["No significant risk factors"],
            "corrections_count": len(corrections),
            "trust_index": round(1.0 - (context.get('overall_risk_score', 0) * 0.5), 2),
            "timestamp": datetime.now().isoformat()
        }
        
        # Print as beautifully formatted summary
        print()
        print("╔" + "═"*68 + "╗")
        print("║" + " "*18 + "CONTEXT ANALYSIS SUMMARY" + " "*26 + "║")
        print("╚" + "═"*68 + "╝")
        print()
        
        # Claim & Archetype
        print(f"  Claim ID:        {summary['claim_id']}")
        print(f"  Archetype:       {summary['archetype']}")
        print()
        
        # Score Breakdown
        print("  ┌─ SCORE BREAKDOWN ─────────────────────────────────────────┐")
        print(f"  │ Structured (SQL Rules):    {summary['structured_score']:.3f} ████████████░░░░")
        print(f"  │ Semantic (Qdrant):         {summary['semantic_score']:.3f} ████████░░░░░░░░")
        print(f"  │ Context (7 Types):         {summary['context_score']:.3f} ████████████░░░░")
        print(f"  │ Context Penalty:           {summary['context_penalty']:+.3f}")
        print("  │ " + "─"*62 + " │")
        decision_color = "[PASS]" if summary['decision'] == 'AUTO_CORRECT' else "[REVIEW]" if summary['decision'] == 'MANUAL_REVIEW' else "[FAIL]"
        print(f"  │ FUSED SCORE:               {summary['fused_score']:.3f}  {decision_color}")
        print("  └" + "─"*62 + "┘")
        print()
        
        # Thresholds
        print("  ┌─ DECISION THRESHOLDS ─────────────────────────────────────┐")
        print(f"  │ T_pass (Auto-Correct):  >= {t_pass:.2f}")
        print(f"  │ T_fail (Escalate):      <  {t_fail:.2f}")
        print(f"  │ T_review (Manual):      {t_fail:.2f} - {t_pass:.2f}")
        print(f"  │ Your Score:             {summary['fused_score']:.3f}  →  {decision_color}")
        print("  └" + "─"*62 + "┘")
        print()
        
        # v3.2 FIX: Show dynamic weights
        print("  ┌─ v3.2 DYNAMIC FUSION WEIGHTS ────────────────────────────────┐")
        print(f"  │ Structured (SQL):   {weights['rule']:.0%}")
        print(f"  │ Semantic (Qdrant):  {weights['vector']:.0%}")
        print(f"  │ Context (7 types):  {weights['context']:.0%}")
        print("  └" + "─"*62 + "┘")
        print()
        
        # v3.2 FIX: Show confidence interval
        print("  ┌─ v3.2 CONFIDENCE INTERVAL ────────────────────────────────────┐")
        print(f"  │ Score: {summary['fused_score']:.3f}")
        print(f"  │ 95% CI: [{summary['confidence_interval']['lower']:.3f}, {summary['confidence_interval']['upper']:.3f}]")
        print(f"  │ Conflict Detected: {'YES - REVIEW' if summary['conflict_detected'] else 'NO'}")
        print("  └" + "─"*62 + "┘")
        print()
        
        # v3.2 FIX: Show detected archetypes
        if summary['detected_archetypes']:
            print("  ┌─ DETECTED ARCHETYPES (v3.2) ──────────────────────────────────┐")
            for arch in summary['detected_archetypes']:
                print(f"  │ - {arch}")
            print("  └" + "─"*62 + "┘")
            print()
        
        # Policy Support
        if summary['top_policy_snippets']:
            print("  ┌─ TOP POLICY CITATIONS ─────────────────────────────────────┐")
            for i, snippet in enumerate(summary['top_policy_snippets'], 1):
                conf = snippet['confidence']
                bar = "█" * int(conf * 10) + "░" * (10 - int(conf * 10))
                print(f"  │ {i}. {snippet['source']}")
                print(f"  │    Confidence: {conf:.3f} {bar}")
            print("  └" + "─"*62 + "┘")
            print()
        
        # Risk Factors
        print("  ┌─ RISK ASSESSMENT ──────────────────────────────────────────┐")
        if summary['risk_factors']:
            for risk in summary['risk_factors']:
                print(f"  │ [!] {risk}")
        else:
            print(f"  │ [OK] No significant risk factors")
        print("  └" + "─"*62 + "┘")
        print()
        
        # Summary Stats
        print("  ┌─ ANALYSIS METRICS ────────────────────────────────────────┐")
        print(f"  │ Corrections Found:  {summary['corrections_count']}")
        print(f"  │ Trust Index:        {summary['trust_index']:.0%}")
        print(f"  │ Timestamp:          {summary['timestamp']}")
        print("  └" + "─"*62 + "┘")
        print()
        
        # Pretty JSON at the end
        print("  FULL DATA (JSON):")
        print("  " + "─"*62)
        json_output = json.dumps(summary, indent=2)
        for line in json_output.split('\n'):
            print("    " + line)
        print("  " + "─"*62)
        
        return {
            'claim_id': claim['claim_id'],
            'archetype': top_match,
            'context': context,
            'base_confidence': rule_confidence,
            'vector_confidence': vector_confidence,
            'context_score': context_score,
            'fused_confidence': fused_confidence,
            'final_confidence': final_confidence,
            'original_action': original_action,
            'final_action': final_action,
            'action_overridden': final_action != original_action,
            'correction': correction,
            'corrections': corrections,  # Extracted corrections
            'policy_support': policy_support,  # Extracted policy snippets
            'semantic_docs': semantic_docs
        }
    
    def _extract_corrections_from_docs(self, semantic_docs: List[Dict], claim: Dict, archetype: Dict) -> List[Dict]:
        """Extract correction suggestions from Qdrant semantic docs"""
        corrections = []
        
        for doc in semantic_docs[:5]:  # Top 5 docs
            text = str(doc.get('text', '') or doc.get('content', '') or doc.get('chunk', '')).lower()
            
            # Check metadata for modifiers
            modifiers = doc.get('modifiers', [])
            if modifiers and isinstance(modifiers, list) and len(modifiers) > 0:
                corrections.append({
                    'action': f'Consider adding modifier {modifiers[0]}',
                    'rationale': f'Policy document suggests modifier {modifiers[0]} may apply',
                    'source': doc.get('source', 'Policy'),
                    'score': doc.get('score', 0),
                    'metadata': {'modifiers': modifiers}
                })
            
            # Check text for common correction patterns
            modifier_keywords = ['modifier 59', 'modifier xe', 'modifier 25', 'modifier 91', 'modifier', 'separately billable']
            coverage_keywords = ['covered when', 'coverage requires', 'not covered', 'bundled', 'component of']
            
            matched = False
            for keyword in modifier_keywords:
                if keyword in text:
                    corrections.append({
                        'action': f'Review policy for {keyword}',
                        'rationale': text[:250],
                        'source': doc.get('source', 'Policy'),
                        'score': doc.get('score', 0)
                    })
                    matched = True
                    break
            
            if not matched:
                for keyword in coverage_keywords:
                    if keyword in text:
                        corrections.append({
                            'action': f'Review coverage requirement: {keyword}',
                            'rationale': text[:250],
                            'source': doc.get('source', 'Policy'),
                            'score': doc.get('score', 0)
                        })
                        break
        
        return corrections[:5]  # Top 5
    
    def _extract_policy_snippets(self, semantic_docs: List[Dict], archetype: Dict) -> List[Dict]:
        """Extract authoritative policy snippets from Qdrant docs"""
        snippets = []
        
        for doc in semantic_docs[:5]:
            text = str(doc.get('text', '') or doc.get('content', '') or doc.get('chunk', ''))
            source = doc.get('source', '') or doc.get('policy_name', '') or doc.get('collection', 'Policy Document')
            
            # Build richer source reference
            source_parts = [source]
            if doc.get('chapter'):
                source_parts.append(f"Ch.{doc.get('chapter')}")
            if doc.get('section'):
                source_parts.append(f"§{doc.get('section')}")
            source_ref = ' '.join(source_parts)
            
            if text and len(text) > 50:  # Only substantial snippets
                snippets.append({
                    'source': source_ref,
                    'excerpt': text[:400],  # Increased from 300 to 400 chars
                    'score': doc.get('score', 0),
                    'collection': doc.get('collection', 'unknown'),
                    'metadata': {
                        'cpt_codes': doc.get('cpt_codes', []),
                        'modifiers': doc.get('modifiers', []),
                        'page': doc.get('page')
                    }
                })
        
        return snippets[:5]  # Top 5 snippets
    
    def _make_context_aware_decision(self, original_action, final_confidence, context):
        """
        v3.2 FIX: Context-aware decision with archetype thresholds, CI, and sliding risk
        
        Thresholds are now archetype-specific (see ARCHETYPE_THRESHOLDS dict)
        Includes confidence interval for boundary cases
        Uses sliding risk override (no hard 0.7 cliff)
        """
        
        # Get archetype-specific thresholds
        archetype_code = context.get('archetype_code', 'DEFAULT')
        t_pass, t_fail = _get_archetype_thresholds(archetype_code)
        
        # Calculate confidence interval for uncertainty handling
        ci_low, ci_high = _confidence_interval(final_confidence, k_sources=3)
        
        # Check if threshold is within uncertainty interval
        if ci_low < t_pass < ci_high:
            decision = 'MANUAL_REVIEW'
            reason = f"Uncertain: CI [{ci_low:.3f}, {ci_high:.3f}] overlaps pass threshold ({t_pass:.3f})"
            return decision, reason
        
        # Standard threshold check
        if final_confidence >= t_pass:
            decision = 'AUTO_CORRECT'
            reason = f"Score {final_confidence:.3f} >= pass threshold {t_pass:.3f}"
        elif final_confidence < t_fail:
            decision = 'ESCALATE'
            reason = f"Score {final_confidence:.3f} < fail threshold {t_fail:.3f}"
        else:
            decision = 'MANUAL_REVIEW'
            reason = f"Score {final_confidence:.3f} in review zone ({t_fail:.3f}-{t_pass:.3f})"
        
        # v3.2 FIX: Sliding risk override (not hard cutoff)
        overall_risk = context.get('overall_risk_score', 0.0)
        decision = _risk_override(decision, overall_risk)
        if overall_risk >= 0.65:
            reason += f" | Context risk {overall_risk:.3f} may force review"
        
        return decision, reason
    
    def _log_claim_analysis(self, claim, match, final_conf, final_action, overridden, override_reason, correction):
        """Log to CXR_Claims_Analysis"""
        cursor = self.conn.cursor()
        
        cursor.execute("""
            INSERT INTO CXR_Claims_Analysis (
                claim_id, archetype_code, matched_confidence,
                final_confidence, recommended_action,
                auto_corrected, amount_saved,
                action_overridden, action_override_reason
            )
            OUTPUT INSERTED.analysis_id
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            claim['claim_id'], match['archetype_code'], match['confidence'],
            final_conf, final_action, correction.get('corrected', False),
            correction.get('amount_saved', 0), overridden, override_reason
        ))
        
        analysis_id = cursor.fetchone()[0]
        self.conn.commit()
        cursor.close()
        
        return analysis_id
    
    def _log_full_context(self, analysis_id, claim, context):
        """Log full context to CXR_Claim_Context"""
        try:
            cursor = self.conn.cursor()
            
            patient = context['1_patient']
            provider = context['2_provider']
            payer = context['3_payer']
            temporal = context['4_temporal']
            financial = context['5_financial']
            relationship = context['6_relationship']
            policy = context['7_policy']
            
            # Simplified INSERT - just log the key metrics
            cursor.execute("""
                INSERT INTO CXR_Claim_Context (
                    analysis_id, claim_id,
                        patient_id, provider_id, payer_id,
                        patient_denial_rate, provider_denial_rate,
                        payer_denial_rate_for_archetype, payer_is_strict,
                        claim_age_days, is_financial_outlier, z_score,
                    overall_risk_score, context_confidence_adjustment,
                    context_json
                )
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                analysis_id, claim['claim_id'],
                    patient.get('patient_id', 'UNKNOWN'), 
                    provider.get('provider_id', 'UNKNOWN'), 
                    payer.get('payer_id', 'UNKNOWN'),
                    patient.get('patient_denial_rate', 0), 
                    provider.get('provider_denial_rate', 0),
                    payer.get('payer_denial_rate_for_archetype', 0), 
                    payer.get('payer_is_strict', False),
                    temporal.get('claim_age_days', 0), 
                    financial.get('is_financial_outlier', False), 
                    financial.get('z_score', 0),
                    context.get('overall_risk_score', 0), 
                    context.get('confidence_adjustment', 0),
                json.dumps(context)
            ))
            
            self.conn.commit()
            cursor.close()
        except Exception as e:
            print(f"   [DB] Context logging skipped: {str(e)}")
            # Don't fail the entire analysis if context logging fails
    
    # =================================================================================
    # ARCHETYPE DETECTION (Phase 1 & 2)
    # =================================================================================
    
    def _detect_broad(self, claim):
        """Phase 1: Broad category detection using SQL queries"""
        matches = []
        
        # Try SQL-based detection first
        sql_matches = self._detect_archetypes_from_sql(claim)
        if sql_matches:
            return sql_matches
        
        # Fallback to keyword-based detection
        for archetype_7 in self.archetypes_by_category.keys():
            if self._quick_category_check(claim, archetype_7):
                matches.append(archetype_7)
        
        # If no matches found, default to DEFAULT category for clean claims
        if not matches:
            matches.append('DEFAULT')
        
        return matches
    
    def _detect_archetypes_from_sql(self, claim: Dict) -> List[str]:
        """Query SQL databases to detect archetypes from actual policy rules"""
        matches = []
        
        try:
            cpt_code = self._get_cpt_code(claim)
            icd_code = self._get_icd_code(claim)
            
            if not cpt_code:
                return []
            
            # Get all procedure codes from claim
            proc_codes = claim.get('procedure_codes', {})
            if isinstance(proc_codes, dict):
                all_cpts = [v for v in proc_codes.values() if v]
            else:
                all_cpts = [cpt_code]
            
            cursor = self.conn.cursor()
            
            # Query for PTP conflicts BETWEEN the actual procedures in this claim
            if len(all_cpts) > 1:
                # Check if any pair of CPTs in this claim has a PTP edit
                ptp_found = False
                for i, cpt1 in enumerate(all_cpts[:-1]):
                    for cpt2 in all_cpts[i+1:]:
                        cursor.execute("""
                        SELECT TOP 1 ptp_edit_rationale
                        FROM _ncci_.dbo.vw_NCCI_PTP_Unified_Edits
                        WHERE ((primary_cpt_hcpcs_code = ? AND secondary_cpt_hcpcs_code = ?)
                           OR  (primary_cpt_hcpcs_code = ? AND secondary_cpt_hcpcs_code = ?))
                        AND deletion_date IS NULL
                        """, (cpt1, cpt2, cpt2, cpt1))
                        
                        if cursor.fetchone():
                            ptp_found = True
                            break
                    if ptp_found:
                        break
                
                if ptp_found:
                    print(f"   [SQL] Found PTP conflict between CPTs in claim")
                    matches.append('NCCI_PTP_Conflict')
            
            # Query for MUE edits (single CPT analysis)
            cursor.execute("""
            SELECT TOP 5 mue_rationale
            FROM _ncci_.dbo.vw_NCCI_MUE_Unified_Edits
            WHERE hcpcs_cpt_code = ?
            """, (cpt_code,))
            
            mue_results = cursor.fetchall()
            if mue_results:
                print(f"   [SQL] Found {len(mue_results)} MUE edits for CPT {cpt_code}")
                # Only flag as MUE if there's actually a concern
                # For now, we'll let the scoring determine if it's real
                # matches.append('MUE')
            
            cursor.close()
            
        except Exception as e:
            print(f"   [SQL] SQL-based detection failed: {str(e)}")
        
        return matches
    
    def _quick_category_check(self, claim: Dict, archetype_7: str) -> bool:
        """Fast category-level check - improved with actual matching"""
        
        cpt_code = self._get_cpt_code(claim)
        icd_code = self._get_icd_code(claim)
        
        if archetype_7 == 'NCCI_PTP_Conflict':
            # PTP conflicts require SQL detection
            # Don't flag just because there are multiple procedures
            # Multiple procedures != PTP conflict (they might be routine tests)
            return False  # Let SQL handle this
        
        elif archetype_7 == 'MUE_Risk':
            # MUE requires procedure codes to check for units
            proc_codes = claim.get('procedure_codes', [])
            if isinstance(proc_codes, dict):
                proc_codes = [v for v in proc_codes.values() if v]
            return len(proc_codes) > 0  # Only if we have procedures
        
        elif archetype_7 == 'COVERAGE':
            # Coverage issues: Don't auto-flag for routine claims
            # Only return True if there's explicit coverage concern (requires SQL)
            return False  # Let SQL handle coverage detection
        
        elif archetype_7 == 'MEDICAL_NECESSITY':
            # Medical necessity requires diagnosis matching procedure
            return bool(icd_code and cpt_code)
        
        elif archetype_7 == 'AUTHORIZATION':
            # Prior auth issues - only if explicitly flagged
            return claim.get('prior_auth_required', False)
        
        elif archetype_7 == 'DEFAULT':
            # Default/Compliant - only if no other archetype matches
            return False  # Let this be a fallback
        
        return False
    
    def _get_cpt_code(self, claim: Dict) -> str:
        """Extract CPT code from claim"""
        if claim.get('cpt'):
            return str(claim.get('cpt'))
        
        # Check procedure_codes dict
        proc_codes = claim.get('procedure_codes', {})
        if isinstance(proc_codes, dict):
            return str(proc_codes.get('HCPCS_CD_1') or '')
        
        return ''
    
    def _get_icd_code(self, claim: Dict) -> str:
        """Extract primary ICD code from claim"""
        if claim.get('icd10_code'):
            return str(claim.get('icd10_code'))
        if claim.get('icd9_code'):
            return str(claim.get('icd9_code'))
        
        # Check diagnosis_codes dict
        diag_codes = claim.get('diagnosis_codes', {})
        if isinstance(diag_codes, dict):
            return str(diag_codes.get('ICD9_DGNS_CD_1') or diag_codes.get('ICD_DGNS_CD_1') or '')
        
        return ''
    
    def _detect_specific(self, claim, broad):
        """Phase 2: Specific archetype detection with scoring"""
        matches = []
        
        cpt_code = self._get_cpt_code(claim)
        icd_code = self._get_icd_code(claim)
        
        for archetype_7 in broad:
            # Get all archetypes in this category
            candidates = self.archetypes_by_category.get(archetype_7, [])
            
            for arch in candidates:
                # Score this archetype match
                base_confidence = arch.get('confidence', 0.75)
                match_score = self._score_archetype_match(arch, claim, cpt_code, icd_code)
                
                final_confidence = (base_confidence * 0.6) + (match_score * 0.4)
                
                matches.append({
                    'archetype_code': arch['archetype_code'],
                    'archetype_7': archetype_7,
                    'archetype_name': arch.get('archetype_name', archetype_7),
                    'confidence': final_confidence,
                    'recommended_action': arch.get('recommended_action', 'MANUAL_REVIEW'),
                    'risk_category': arch.get('risk_category', 'MEDIUM')
                })
        
        # Sort by confidence (highest first)
        matches.sort(key=lambda x: x['confidence'], reverse=True)
        
        # If no specific matches found but DEFAULT category exists, use Compliant
        if not matches:
            default_candidates = self.archetypes_by_category.get('DEFAULT', [])
            if default_candidates:
                compliant = default_candidates[0]
                matches.append({
                    'archetype_code': compliant.get('archetype_code', 'Compliant'),
                    'archetype_7': 'DEFAULT',
                    'archetype_name': compliant.get('archetype_name', 'Compliant'),
                    'confidence': 0.95,  # High confidence for clean claims
                    'recommended_action': 'APPROVE',
                    'risk_category': 'LOW'
                })
        
        return matches
    
    def _score_archetype_match(self, arch: Dict, claim: Dict, cpt_code: str, icd_code: str) -> float:
        """Score how well a claim matches an archetype (0.0 to 1.0)"""
        score = 0.5  # Base score
        
        arch_code = arch.get('archetype_code', '').upper()
        arch_name = arch.get('archetype_name', '').upper()
        
        # Be conservative: For coverage/NCD, only score if there's real evidence
        # Clean claims should NOT match these archetypes
        if 'NCD' in arch_code or 'TERMINATED' in arch_name:
            # NCD issues require explicit SQL detection, not keyword matching
            return 0.1  # Very low score for fallback
        
        if 'COVERAGE' in arch_code or 'COVERAGE' in arch_name:
            # Coverage issues also require SQL detection
            if not (icd_code):  # Only if we have an unusual ICD
                return 0.1  # Low score
            score += 0.15  # Conservative scoring
        
        if 'MUE' in arch_code or 'MUE' in arch_name:
            if cpt_code and len(cpt_code) > 0:
                score += 0.15  # Reduced from 0.20
        
        if 'PTP' in arch_code or 'SEPARATE' in arch_name:
            if cpt_code and len(cpt_code) > 0:
                score += 0.15
        
        if 'AUTH' in arch_code or 'AUTH' in arch_name:
            if claim.get('prior_auth_required'):
                score += 0.30
        
        if 'MEDICAL' in arch_code or 'NECESSITY' in arch_name:
            if cpt_code and icd_code:
                score += 0.20
        
        if 'COMPLIANT' in arch_code or 'DEFAULT' in arch_name:
            # For clean claims with proper codes, Compliant should score highest
            if cpt_code and icd_code:
                score = 0.95  # Compliant is best match for clean, coded claims
            elif not cpt_code and not icd_code:
                score = 0.85
            else:
                score = 0.80
        
        return min(1.0, score)

# =====================================================================================
# DEMO
# =====================================================================================

if __name__ == "__main__":
    print("="*70)
    print("CXR KERNEL - FULL CONTEXT DEMO")
    print("="*70)
    
    kernel = CXRKernelFullContext()
    
    # High-risk claim (multiple context red flags)
    risky_claim = {
        'claim_id': 'HIGH-RISK-001',
        'procedure_codes': ['27132', '27130'],
        'total_amount': 15000.00,  # 6x average
        'service_date': '2024-06-15',
        'patient_id': 'PT-12345',
        'patient_gender': 'F',
        'patient_age': 65,
        'provider_id': 'PROV-999',
        'provider_specialty': 'Orthopedic Surgery',
        'payer_id': 'PAYER-STRICT',
        'payer_name': 'StrictHealth',
        'payer_type': 'Commercial',
        'same_day_claims_count': 1
    }
    
    result = kernel.process_claim(risky_claim, verbose=True)
    
    print(f"\n{'='*70}")
    print("RESULT:")
    print(f"{'='*70}")
    print(f"Base → Final Confidence: {result['base_confidence']:.3f} → {result['final_confidence']:.3f}")
    print(f"Action: {result['original_action']} → {result['final_action']}")
    print(f"Context Influenced: {result['action_overridden']}")

