#!/usr/bin/env python3
"""
Cross-question concordance analysis for benchmark results.

Analyzes all benchmark results across all research questions in the database
and computes concordance matrices comparing each model to reference models
(Claude Opus/Sonnet).

Outputs:
1. Score concordance matrix (agreement within ±1 tolerance)
2. Inclusion/exclusion agreement matrix (binary decision agreement)
3. Per-model statistics compared to reference models

Usage:
    python scripts/concordance_analysis.py [--output-dir DIR] [--format csv|json]
"""

import argparse
import csv
import json
import sys
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Optional

# Add src to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))

from bmlibrarian_lite.config import LiteConfig
from bmlibrarian_lite.storage import LiteStorage
from bmlibrarian_lite.constants import DEFAULT_MIN_SCORE
from bmlibrarian_lite.benchmarking.statistics import (
    compute_agreement,
    compute_inclusion_agreement,
    compute_kendall_tau,
    compute_mean_absolute_difference,
)


# Reference model patterns (Claude Opus and Sonnet variants)
REFERENCE_MODEL_PATTERNS = [
    "claude-opus",
    "claude-sonnet",
]


@dataclass
class EvaluatorScoreData:
    """Aggregated score data for an evaluator across all questions."""

    evaluator_name: str
    evaluator_id: str
    total_documents: int
    scores: list[int]
    document_ids: list[str]
    latencies_ms: list[Optional[int]]  # Latency per document (ms)
    total_latency_ms: int = 0  # Total latency across all documents
    avg_latency_ms: float = 0.0  # Average ms per document


@dataclass
class PairwiseConcordance:
    """Concordance metrics between two evaluators."""

    evaluator1: str
    evaluator2: str
    documents_compared: int
    score_agreement: float  # Within ±1 tolerance
    exact_agreement: float  # Exact match
    inclusion_agreement: float  # Binary include/exclude decision
    mean_absolute_difference: float
    kendall_tau: Optional[float]


@dataclass
class EvaluatorTimingData:
    """Timing statistics for an evaluator."""

    evaluator_name: str
    total_documents: int
    total_latency_ms: int
    avg_latency_ms: float


@dataclass
class ConcordanceReport:
    """Complete concordance report across all questions."""

    generated_at: str
    total_questions: int
    total_documents: int
    total_evaluators: int
    evaluator_names: list[str]
    reference_models: list[str]
    score_agreement_matrix: dict[tuple[str, str], float]
    inclusion_agreement_matrix: dict[tuple[str, str], float]
    pairwise_concordances: list[PairwiseConcordance]
    reference_concordance_summary: dict[str, dict[str, float]]
    evaluator_timing: dict[str, EvaluatorTimingData]  # Timing data per evaluator


def is_reference_model(evaluator_name: str) -> bool:
    """Check if evaluator is a reference model (Opus or Sonnet)."""
    name_lower = evaluator_name.lower()
    return any(pattern in name_lower for pattern in REFERENCE_MODEL_PATTERNS)


def collect_all_scores(storage: LiteStorage) -> dict[str, EvaluatorScoreData]:
    """
    Collect all scores from all evaluators across all questions.

    This queries the scored_documents table directly to find ALL scores,
    not just those from formal benchmark runs.

    Returns:
        Dict mapping evaluator_name -> EvaluatorScoreData
    """
    # Get all research questions
    questions = storage.get_unique_research_questions(limit=1000)

    # Store both score and latency per document
    all_evaluator_scores: dict[str, dict[str, int]] = defaultdict(dict)
    all_evaluator_latencies: dict[str, dict[str, Optional[int]]] = defaultdict(dict)
    evaluator_id_map: dict[str, str] = {}

    print(f"Found {len(questions)} research questions in database")

    # Get all evaluators that have scores
    evaluators = storage.get_evaluators()
    print(f"Found {len(evaluators)} evaluators with scores")

    for evaluator in evaluators:
        evaluator_id_map[evaluator.display_name] = evaluator.id

    # For each question, get all document IDs and then query scores
    for q_summary in questions:
        question = q_summary.question
        doc_ids = storage.get_document_ids_for_question(question)

        if not doc_ids:
            continue

        # For each evaluator, check for scores on these documents FOR THIS QUESTION
        for evaluator in evaluators:
            for doc_id in doc_ids:
                scored_doc = storage.get_scored_document_for_question(
                    doc_id, evaluator.id, question
                )
                if scored_doc and 1 <= scored_doc.score <= 5:
                    # Use composite key (doc_id, question) to track question-specific scores
                    score_key = f"{doc_id}|{question[:50]}"
                    all_evaluator_scores[evaluator.display_name][score_key] = scored_doc.score
                    all_evaluator_latencies[evaluator.display_name][score_key] = scored_doc.latency_ms

    # Convert to EvaluatorScoreData
    # Note: Keys are now composite "doc_id|question" to ensure question-specific scores
    result = {}
    for evaluator_name, doc_scores in all_evaluator_scores.items():
        if not doc_scores:
            continue  # Skip evaluators with no valid scores

        # score_keys are composite keys: "doc_id|question"
        score_keys = list(doc_scores.keys())
        scores = [doc_scores[key] for key in score_keys]
        latencies = [all_evaluator_latencies[evaluator_name].get(key) for key in score_keys]

        # Calculate total and average latency (only from non-None values)
        valid_latencies = [lat for lat in latencies if lat is not None]
        total_latency = sum(valid_latencies) if valid_latencies else 0
        avg_latency = total_latency / len(valid_latencies) if valid_latencies else 0.0

        result[evaluator_name] = EvaluatorScoreData(
            evaluator_name=evaluator_name,
            evaluator_id=evaluator_id_map[evaluator_name],
            total_documents=len(scores),
            scores=scores,
            document_ids=score_keys,  # Composite keys for question-specific tracking
            latencies_ms=latencies,
            total_latency_ms=total_latency,
            avg_latency_ms=avg_latency,
        )

    return result


def compute_pairwise_concordance(
    eval1_data: EvaluatorScoreData,
    eval2_data: EvaluatorScoreData,
    inclusion_threshold: int = DEFAULT_MIN_SCORE,
) -> Optional[PairwiseConcordance]:
    """
    Compute concordance metrics between two evaluators.

    Only compares documents scored by both evaluators.
    """
    # Find common documents
    common_docs = set(eval1_data.document_ids) & set(eval2_data.document_ids)

    if len(common_docs) < 2:
        return None

    # Build aligned score lists
    doc_to_idx1 = {doc_id: i for i, doc_id in enumerate(eval1_data.document_ids)}
    doc_to_idx2 = {doc_id: i for i, doc_id in enumerate(eval2_data.document_ids)}

    scores1 = []
    scores2 = []
    for doc_id in common_docs:
        scores1.append(eval1_data.scores[doc_to_idx1[doc_id]])
        scores2.append(eval2_data.scores[doc_to_idx2[doc_id]])

    return PairwiseConcordance(
        evaluator1=eval1_data.evaluator_name,
        evaluator2=eval2_data.evaluator_name,
        documents_compared=len(common_docs),
        score_agreement=compute_agreement(scores1, scores2, tolerance=1),
        exact_agreement=compute_agreement(scores1, scores2, tolerance=0),
        inclusion_agreement=compute_inclusion_agreement(
            scores1, scores2, inclusion_threshold
        ),
        mean_absolute_difference=compute_mean_absolute_difference(scores1, scores2),
        kendall_tau=compute_kendall_tau(scores1, scores2),
    )


def build_concordance_report(
    evaluator_data: dict[str, EvaluatorScoreData],
    inclusion_threshold: int = DEFAULT_MIN_SCORE,
) -> ConcordanceReport:
    """
    Build complete concordance report from evaluator data.
    """
    evaluator_names = sorted(evaluator_data.keys())
    reference_models = [name for name in evaluator_names if is_reference_model(name)]

    print(f"Found {len(evaluator_names)} evaluators")
    print(f"Reference models: {reference_models}")

    # Compute all pairwise concordances
    pairwise_concordances = []
    score_agreement_matrix: dict[tuple[str, str], float] = {}
    inclusion_agreement_matrix: dict[tuple[str, str], float] = {}

    for i, name1 in enumerate(evaluator_names):
        for name2 in evaluator_names[i:]:
            if name1 == name2:
                score_agreement_matrix[(name1, name2)] = 1.0
                inclusion_agreement_matrix[(name1, name2)] = 1.0
                continue

            concordance = compute_pairwise_concordance(
                evaluator_data[name1],
                evaluator_data[name2],
                inclusion_threshold,
            )

            if concordance:
                pairwise_concordances.append(concordance)
                score_agreement_matrix[(name1, name2)] = concordance.score_agreement
                score_agreement_matrix[(name2, name1)] = concordance.score_agreement
                inclusion_agreement_matrix[(name1, name2)] = (
                    concordance.inclusion_agreement
                )
                inclusion_agreement_matrix[(name2, name1)] = (
                    concordance.inclusion_agreement
                )

    # Build reference model concordance summary
    reference_summary: dict[str, dict[str, float]] = {}
    for evaluator_name in evaluator_names:
        if evaluator_name in reference_models:
            continue

        ref_scores = {}
        for ref_model in reference_models:
            key = (evaluator_name, ref_model)
            if key in score_agreement_matrix:
                ref_scores[ref_model] = {
                    "score_agreement": score_agreement_matrix[key],
                    "inclusion_agreement": inclusion_agreement_matrix[key],
                }

        if ref_scores:
            reference_summary[evaluator_name] = ref_scores

    # Count total documents and questions
    total_docs = sum(d.total_documents for d in evaluator_data.values())
    unique_doc_ids: set[str] = set()
    for data in evaluator_data.values():
        unique_doc_ids.update(data.document_ids)

    # Build timing data for each evaluator
    evaluator_timing: dict[str, EvaluatorTimingData] = {}
    for name, data in evaluator_data.items():
        evaluator_timing[name] = EvaluatorTimingData(
            evaluator_name=name,
            total_documents=data.total_documents,
            total_latency_ms=data.total_latency_ms,
            avg_latency_ms=data.avg_latency_ms,
        )

    return ConcordanceReport(
        generated_at=datetime.now().isoformat(),
        total_questions=len(unique_doc_ids),  # Approximation
        total_documents=len(unique_doc_ids),
        total_evaluators=len(evaluator_names),
        evaluator_names=evaluator_names,
        reference_models=reference_models,
        score_agreement_matrix=score_agreement_matrix,
        inclusion_agreement_matrix=inclusion_agreement_matrix,
        pairwise_concordances=pairwise_concordances,
        reference_concordance_summary=reference_summary,
        evaluator_timing=evaluator_timing,
    )


def format_matrix_table(
    matrix: dict[tuple[str, str], float],
    evaluator_names: list[str],
    title: str,
) -> str:
    """Format a concordance matrix as an ASCII table."""
    # Truncate names for display
    max_name_len = 25
    short_names = [name[:max_name_len] for name in evaluator_names]

    lines = [f"\n{'=' * 80}", title, "=" * 80]

    # Header row
    header = f"{'Evaluator':<{max_name_len}} |"
    for short_name in short_names:
        header += f" {short_name[:8]:>8}"
    lines.append(header)
    lines.append("-" * len(header))

    # Data rows
    for i, name in enumerate(evaluator_names):
        row = f"{short_names[i]:<{max_name_len}} |"
        for name2 in evaluator_names:
            value = matrix.get((name, name2), 0.0)
            row += f" {value * 100:>7.1f}%"
        lines.append(row)

    return "\n".join(lines)


def format_reference_summary(report: ConcordanceReport) -> str:
    """Format reference model concordance summary."""
    lines = [
        f"\n{'=' * 120}",
        "CONCORDANCE WITH REFERENCE MODELS (Claude Opus/Sonnet)",
        "=" * 120,
        "",
        f"{'Model':<40} | {'Ref Model':<25} | {'Score Agr':>10} | {'Incl Agr':>10} | {'Total (s)':>12} | {'s/doc':>8}",
        "-" * 120,
    ]

    # Sort by average concordance with reference models
    model_avg_concordance = {}
    for model_name, ref_data in report.reference_concordance_summary.items():
        avg_score = sum(d["score_agreement"] for d in ref_data.values()) / len(ref_data)
        model_avg_concordance[model_name] = avg_score

    sorted_models = sorted(
        model_avg_concordance.keys(), key=lambda x: model_avg_concordance[x], reverse=True
    )

    for model_name in sorted_models:
        ref_data = report.reference_concordance_summary[model_name]
        timing = report.evaluator_timing.get(model_name)

        for i, (ref_model, metrics) in enumerate(sorted(ref_data.items())):
            display_name = model_name if i == 0 else ""
            # Only show timing on first row for this model
            if i == 0 and timing and timing.total_latency_ms > 0:
                total_secs = timing.total_latency_ms / 1000
                avg_secs = timing.avg_latency_ms / 1000
                total_time_str = f"{total_secs:>12,.1f}"
                avg_time_str = f"{avg_secs:>8.1f}"
            else:
                total_time_str = f"{'':>12}"
                avg_time_str = f"{'':>8}"

            lines.append(
                f"{display_name:<40} | {ref_model:<25} | "
                f"{metrics['score_agreement'] * 100:>9.1f}% | "
                f"{metrics['inclusion_agreement'] * 100:>9.1f}% | "
                f"{total_time_str} | {avg_time_str}"
            )
        if len(ref_data) > 1:
            avg_score = sum(d["score_agreement"] for d in ref_data.values()) / len(
                ref_data
            )
            avg_incl = sum(d["inclusion_agreement"] for d in ref_data.values()) / len(
                ref_data
            )
            lines.append(
                f"{'':<40} | {'[Average]':<25} | "
                f"{avg_score * 100:>9.1f}% | {avg_incl * 100:>9.1f}% | "
                f"{'':>12} | {'':>8}"
            )
        lines.append("")

    return "\n".join(lines)


def export_to_csv(report: ConcordanceReport, output_dir: Path) -> None:
    """Export concordance matrices to CSV files."""
    output_dir.mkdir(parents=True, exist_ok=True)

    # Score agreement matrix
    score_csv = output_dir / "score_agreement_matrix.csv"
    with open(score_csv, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["Evaluator"] + report.evaluator_names)
        for name1 in report.evaluator_names:
            row = [name1]
            for name2 in report.evaluator_names:
                value = report.score_agreement_matrix.get((name1, name2), 0.0)
                row.append(f"{value:.4f}")
            writer.writerow(row)
    print(f"Wrote: {score_csv}")

    # Inclusion agreement matrix
    incl_csv = output_dir / "inclusion_agreement_matrix.csv"
    with open(incl_csv, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["Evaluator"] + report.evaluator_names)
        for name1 in report.evaluator_names:
            row = [name1]
            for name2 in report.evaluator_names:
                value = report.inclusion_agreement_matrix.get((name1, name2), 0.0)
                row.append(f"{value:.4f}")
            writer.writerow(row)
    print(f"Wrote: {incl_csv}")

    # Reference summary with timing
    ref_csv = output_dir / "reference_concordance.csv"
    with open(ref_csv, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(
            ["Model", "Reference Model", "Score Agreement", "Inclusion Agreement",
             "Total Documents", "Total Time (s)", "Time per Doc (s)"]
        )
        for model_name, ref_data in report.reference_concordance_summary.items():
            timing = report.evaluator_timing.get(model_name)
            for ref_model, metrics in ref_data.items():
                writer.writerow(
                    [
                        model_name,
                        ref_model,
                        f"{metrics['score_agreement']:.4f}",
                        f"{metrics['inclusion_agreement']:.4f}",
                        timing.total_documents if timing else "",
                        f"{timing.total_latency_ms / 1000:.1f}" if timing else "",
                        f"{timing.avg_latency_ms / 1000:.1f}" if timing else "",
                    ]
                )
    print(f"Wrote: {ref_csv}")


def export_to_json(report: ConcordanceReport, output_dir: Path) -> None:
    """Export concordance report to JSON."""
    output_dir.mkdir(parents=True, exist_ok=True)

    # Convert tuple keys to string for JSON
    json_data = {
        "generated_at": report.generated_at,
        "total_questions": report.total_questions,
        "total_documents": report.total_documents,
        "total_evaluators": report.total_evaluators,
        "evaluator_names": report.evaluator_names,
        "reference_models": report.reference_models,
        "score_agreement_matrix": {
            f"{k[0]}|{k[1]}": v for k, v in report.score_agreement_matrix.items()
        },
        "inclusion_agreement_matrix": {
            f"{k[0]}|{k[1]}": v for k, v in report.inclusion_agreement_matrix.items()
        },
        "pairwise_concordances": [
            {
                "evaluator1": pc.evaluator1,
                "evaluator2": pc.evaluator2,
                "documents_compared": pc.documents_compared,
                "score_agreement": pc.score_agreement,
                "exact_agreement": pc.exact_agreement,
                "inclusion_agreement": pc.inclusion_agreement,
                "mean_absolute_difference": pc.mean_absolute_difference,
                "kendall_tau": pc.kendall_tau,
            }
            for pc in report.pairwise_concordances
        ],
        "reference_concordance_summary": report.reference_concordance_summary,
        "evaluator_timing": {
            name: {
                "total_documents": timing.total_documents,
                "total_time_seconds": timing.total_latency_ms / 1000,
                "time_per_doc_seconds": timing.avg_latency_ms / 1000,
            }
            for name, timing in report.evaluator_timing.items()
        },
    }

    json_path = output_dir / "concordance_report.json"
    with open(json_path, "w") as f:
        json.dump(json_data, f, indent=2)
    print(f"Wrote: {json_path}")


def export_to_markdown(
    report: ConcordanceReport,
    output_dir: Path,
    inclusion_threshold: int,
) -> None:
    """Export concordance report to Markdown format."""
    output_dir.mkdir(parents=True, exist_ok=True)

    lines = [
        "# Concordance Analysis Report",
        "",
        f"**Generated:** {report.generated_at}",
        "",
        "## Summary",
        "",
        "| Metric | Value |",
        "|--------|-------|",
        f"| Total Documents Analyzed | {report.total_documents} |",
        f"| Total Evaluators | {report.total_evaluators} |",
        f"| Reference Models | {', '.join(report.reference_models) or 'None found'} |",
        f"| Inclusion Threshold | >= {inclusion_threshold} |",
        "",
    ]

    # Reference model concordance summary (most important table)
    if report.reference_concordance_summary:
        lines.extend([
            "## Concordance with Reference Models",
            "",
            "Models ranked by score agreement with Claude Opus/Sonnet:",
            "",
            "| Model | Reference Model | Score Agreement | Inclusion Agreement | Total (s) | s/doc |",
            "|-------|-----------------|----------------:|--------------------:|----------:|------:|",
        ])

        # Sort by average concordance
        model_avg = {}
        for model_name, ref_data in report.reference_concordance_summary.items():
            avg = sum(d["score_agreement"] for d in ref_data.values()) / len(ref_data)
            model_avg[model_name] = avg

        sorted_models = sorted(
            model_avg.keys(), key=lambda x: model_avg[x], reverse=True
        )

        for model_name in sorted_models:
            ref_data = report.reference_concordance_summary[model_name]
            timing = report.evaluator_timing.get(model_name)

            for i, (ref_model, metrics) in enumerate(sorted(ref_data.items())):
                score_pct = metrics["score_agreement"] * 100
                incl_pct = metrics["inclusion_agreement"] * 100

                # Only show timing on first row for this model
                if i == 0 and timing and timing.total_latency_ms > 0:
                    total_secs = timing.total_latency_ms / 1000
                    avg_secs = timing.avg_latency_ms / 1000
                    total_str = f"{total_secs:,.1f}"
                    avg_str = f"{avg_secs:.1f}"
                else:
                    total_str = ""
                    avg_str = ""

                lines.append(
                    f"| {model_name} | {ref_model} | {score_pct:.1f}% | {incl_pct:.1f}% | {total_str} | {avg_str} |"
                )

        lines.append("")

    # Score agreement matrix
    lines.extend([
        "## Score Agreement Matrix",
        "",
        "Percentage of documents where evaluators agree within ±1 score point:",
        "",
    ])
    lines.append(_format_markdown_matrix(
        report.score_agreement_matrix,
        report.evaluator_names,
    ))
    lines.append("")

    # Inclusion agreement matrix
    lines.extend([
        "## Inclusion Agreement Matrix",
        "",
        f"Percentage of documents where evaluators agree on include/exclude decision (threshold >= {inclusion_threshold}):",
        "",
    ])
    lines.append(_format_markdown_matrix(
        report.inclusion_agreement_matrix,
        report.evaluator_names,
    ))
    lines.append("")

    # Detailed pairwise statistics
    if report.pairwise_concordances:
        lines.extend([
            "## Detailed Pairwise Statistics",
            "",
            "| Evaluator 1 | Evaluator 2 | Docs | Score Agr | Exact Agr | Incl Agr | MAD | Kendall τ |",
            "|-------------|-------------|-----:|----------:|----------:|---------:|----:|----------:|",
        ])

        # Sort by score agreement descending
        sorted_pairs = sorted(
            report.pairwise_concordances,
            key=lambda x: x.score_agreement,
            reverse=True,
        )

        for pc in sorted_pairs:
            tau_str = f"{pc.kendall_tau:.3f}" if pc.kendall_tau is not None else "N/A"
            lines.append(
                f"| {pc.evaluator1} | {pc.evaluator2} | {pc.documents_compared} | "
                f"{pc.score_agreement * 100:.1f}% | {pc.exact_agreement * 100:.1f}% | "
                f"{pc.inclusion_agreement * 100:.1f}% | {pc.mean_absolute_difference:.2f} | {tau_str} |"
            )

        lines.append("")

    # Methodology notes
    lines.extend([
        "## Methodology",
        "",
        "- **Score Agreement**: Percentage of documents where scores differ by at most 1 point (on 1-5 scale)",
        "- **Exact Agreement**: Percentage of documents with identical scores",
        "- **Inclusion Agreement**: Percentage of documents where both evaluators agree on include/exclude decision",
        "- **MAD (Mean Absolute Difference)**: Average absolute score difference (0-4 scale)",
        "- **Kendall τ**: Rank correlation coefficient (-1 to +1, higher = better agreement on relative ordering)",
        "",
    ])

    md_path = output_dir / "concordance_report.md"
    with open(md_path, "w") as f:
        f.write("\n".join(lines))
    print(f"Wrote: {md_path}")


def _format_markdown_matrix(
    matrix: dict[tuple[str, str], float],
    evaluator_names: list[str],
) -> str:
    """Format a concordance matrix as a Markdown table."""
    # Create short names for column headers
    short_names = []
    for i, name in enumerate(evaluator_names):
        # Use abbreviation: first letters of each part
        parts = name.replace(":", " ").replace("-", " ").replace("_", " ").split()
        abbrev = "".join(p[0].upper() for p in parts[:4])
        short_names.append(f"{abbrev}[{i+1}]")

    lines = []

    # Header row
    header = "| Model |"
    for short in short_names:
        header += f" {short} |"
    lines.append(header)

    # Separator row
    sep = "|-------|"
    for _ in short_names:
        sep += "-------:|"
    lines.append(sep)

    # Data rows
    for i, name in enumerate(evaluator_names):
        row = f"| **[{i+1}]** {name} |"
        for name2 in evaluator_names:
            value = matrix.get((name, name2), 0.0)
            row += f" {value * 100:.1f}% |"
        lines.append(row)

    return "\n".join(lines)


def main() -> None:
    """Main entry point."""
    parser = argparse.ArgumentParser(
        description="Analyze concordance across all benchmark results"
    )
    parser.add_argument(
        "--output-dir",
        type=Path,
        default=Path("concordance_output"),
        help="Directory for output files (default: concordance_output)",
    )
    parser.add_argument(
        "--format",
        choices=["csv", "json", "markdown", "md", "both", "all"],
        default="all",
        help="Output format: csv, json, markdown/md, both (csv+json), all (default: all)",
    )
    parser.add_argument(
        "--inclusion-threshold",
        type=int,
        default=DEFAULT_MIN_SCORE,
        help=f"Score threshold for inclusion decision (default: {DEFAULT_MIN_SCORE})",
    )
    args = parser.parse_args()

    print("Loading configuration and storage...")
    config = LiteConfig.load()
    storage = LiteStorage(config)

    print("\nCollecting scores from all benchmark runs...")
    evaluator_data = collect_all_scores(storage)

    if not evaluator_data:
        print("No benchmark scores found in database.")
        sys.exit(1)

    print(f"\nFound scores from {len(evaluator_data)} evaluators:")
    for name, data in sorted(evaluator_data.items()):
        if data.total_latency_ms > 0:
            total_secs = data.total_latency_ms / 1000
            avg_secs = data.avg_latency_ms / 1000
            print(f"  - {name}: {data.total_documents} documents, "
                  f"{total_secs:,.1f}s total, {avg_secs:.1f}s/doc")
        else:
            print(f"  - {name}: {data.total_documents} documents (no timing data)")

    print("\nBuilding concordance report...")
    report = build_concordance_report(evaluator_data, args.inclusion_threshold)

    # Print summary to console
    print(f"\n{'=' * 80}")
    print("CONCORDANCE ANALYSIS SUMMARY")
    print("=" * 80)
    print(f"Generated: {report.generated_at}")
    print(f"Total documents analyzed: {report.total_documents}")
    print(f"Total evaluators: {report.total_evaluators}")
    print(f"Reference models found: {', '.join(report.reference_models)}")

    # Print score agreement matrix
    if report.score_agreement_matrix:
        print(
            format_matrix_table(
                report.score_agreement_matrix,
                report.evaluator_names,
                "SCORE AGREEMENT MATRIX (within ±1 tolerance)",
            )
        )

    # Print inclusion agreement matrix
    if report.inclusion_agreement_matrix:
        print(
            format_matrix_table(
                report.inclusion_agreement_matrix,
                report.evaluator_names,
                f"INCLUSION AGREEMENT MATRIX (threshold >= {args.inclusion_threshold})",
            )
        )

    # Print reference model summary
    if report.reference_concordance_summary:
        print(format_reference_summary(report))

    # Export to files
    if args.format in ("csv", "both", "all"):
        export_to_csv(report, args.output_dir)

    if args.format in ("json", "both", "all"):
        export_to_json(report, args.output_dir)

    if args.format in ("markdown", "md", "both", "all"):
        export_to_markdown(report, args.output_dir, args.inclusion_threshold)

    print(f"\nDone! Output written to: {args.output_dir}")


if __name__ == "__main__":
    main()
