"""Benchmark orchestration harness."""

import time
from datetime import datetime
from typing import Dict, Any, List
from pathlib import Path

from .config import BenchmarkConfig
from .evaluators import get_evaluator
from .datasets import get_dataset


class BenchmarkHarness:
    """Orchestrates benchmark execution."""
    
    def __init__(self, config: BenchmarkConfig):
        self.config = config
        self.evaluator = get_evaluator(config.evaluator)
        self.dataset = get_dataset(config.dataset)
        
    def run(self) -> Dict[str, Any]:
        """Execute benchmark and return results."""
        start_time = time.time()
        
        results = {
            "benchmark": self.config.name,
            "version": self.config.version,
            "timestamp": datetime.utcnow().isoformat() + "Z",
            "config": self.config.model_dump(),
            "duration_seconds": 0.0,
            "metrics": {},
            "per_item_results": [],
            "warnings": [],
            "errors": []
        }
        
        try:
            # Load dataset
            items = self.dataset.load()
            
            # Evaluate each item
            for item in items:
                try:
                    item_result = self.evaluator.evaluate(item)
                    results["per_item_results"].append(item_result)
                except Exception as e:
                    results["errors"].append(f"Item {item.get('id', 'unknown')}: {str(e)}")
            
            # Compute aggregate metrics
            results["metrics"] = self.compute_metrics(results["per_item_results"])
            
        except Exception as e:
            results["errors"].append(f"Benchmark failed: {str(e)}")
        
        finally:
            results["duration_seconds"] = time.time() - start_time
        
        return results
    
    def compute_metrics(self, item_results: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Compute aggregate metrics from per-item results."""
        if not item_results:
            return {}
        
        metrics = {
            "total_items": len(item_results),
            "successful_items": len([r for r in item_results if not r.get("error")]),
        }
        
        # Compute type-specific metrics
        if self.config.evaluator.type == "multi_model":
            metrics.update(self._compute_multi_model_metrics(item_results))
        elif self.config.evaluator.type == "rv_telos":
            metrics.update(self._compute_rv_metrics(item_results))
        elif self.config.evaluator.type == "cuda_gates":
            metrics.update(self._compute_cuda_metrics(item_results))
        
        return metrics
    
    def _compute_multi_model_metrics(self, results: List[Dict]) -> Dict[str, Any]:
        """Compute multi-model consensus metrics."""
        import numpy as np
        
        agreements = [r.get("agreement", 0.0) for r in results if "agreement" in r]
        confidences = []
        for r in results:
            if "decisions" in r:
                confidences.extend([d["confidence"] for d in r["decisions"].values()])
        
        return {
            "mean_agreement": float(np.mean(agreements)) if agreements else 0.0,
            "std_agreement": float(np.std(agreements)) if agreements else 0.0,
            "mean_confidence": float(np.mean(confidences)) if confidences else 0.0,
        }
    
    def _compute_rv_metrics(self, results: List[Dict]) -> Dict[str, Any]:
        """Compute RV telos measurement metrics."""
        import numpy as np
        
        rv_scores = [r.get("rv_score", 1.0) for r in results if "rv_score" in r]
        ttr_ratios = [r.get("ttr_ratio", 1.0) for r in results if "ttr_ratio" in r]
        
        return {
            "mean_rv": float(np.mean(rv_scores)) if rv_scores else 1.0,
            "std_rv": float(np.std(rv_scores)) if rv_scores else 0.0,
            "mean_ttr_ratio": float(np.mean(ttr_ratios)) if ttr_ratios else 1.0,
        }
    
    def _compute_cuda_metrics(self, results: List[Dict]) -> Dict[str, Any]:
        """Compute CUDA acceleration metrics."""
        import numpy as np
        
        latencies = [r.get("latency_ms", 0.0) for r in results if "latency_ms" in r]
        
        if not latencies:
            return {}
        
        return {
            "latency_p50_ms": float(np.percentile(latencies, 50)),
            "latency_p95_ms": float(np.percentile(latencies, 95)),
            "latency_p99_ms": float(np.percentile(latencies, 99)),
            "throughput_per_sec": len(latencies) / sum(latencies) * 1000 if sum(latencies) > 0 else 0.0,
        }
