# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Familiar v1.4.0 - CI/CD Evaluation Integration

Provides automated evaluation integration for CI/CD pipelines:
- CLI runner for evaluations
- GitHub Actions workflow templates
- GitLab CI templates
- JUnit XML output for test reporting
- Threshold-based pass/fail decisions
- Environment variable configuration
- Artifact generation
"""

import argparse
import json
import logging
import os
import sys
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from pathlib import Path
from typing import Callable, Dict, List, Optional, Union
from xml.dom import minidom

logger = logging.getLogger(__name__)


# ============================================================
# CONSTANTS
# ============================================================


def _utcnow() -> datetime:
    """Get current UTC time."""
    return datetime.now(timezone.utc)


# Default thresholds
DEFAULT_PASS_RATE_THRESHOLD = 0.8  # 80%
DEFAULT_MAX_LATENCY_MS = 5000
DEFAULT_MAX_COST_USD = 1.0
DEFAULT_CRITICAL_FAILURES_ALLOWED = 0


# ============================================================
# ENUMS & CONFIG
# ============================================================


class OutputFormat(str, Enum):
    """Output format options."""

    JSON = "json"
    JUNIT = "junit"
    MARKDOWN = "markdown"
    CONSOLE = "console"
    ALL = "all"


class CIProvider(str, Enum):
    """Supported CI providers."""

    GITHUB_ACTIONS = "github"
    GITLAB_CI = "gitlab"
    GENERIC = "generic"


@dataclass
class EvalThresholds:
    """
    Thresholds for pass/fail decisions in CI.

    Evaluation fails if ANY threshold is exceeded.
    """

    min_pass_rate: float = DEFAULT_PASS_RATE_THRESHOLD
    max_critical_failures: int = DEFAULT_CRITICAL_FAILURES_ALLOWED
    max_high_failures: int = 5
    max_avg_latency_ms: float = DEFAULT_MAX_LATENCY_MS
    max_total_cost_usd: float = DEFAULT_MAX_COST_USD
    fail_on_any_critical: bool = True

    def to_dict(self) -> dict:
        return {
            "min_pass_rate": self.min_pass_rate,
            "max_critical_failures": self.max_critical_failures,
            "max_high_failures": self.max_high_failures,
            "max_avg_latency_ms": self.max_avg_latency_ms,
            "max_total_cost_usd": self.max_total_cost_usd,
            "fail_on_any_critical": self.fail_on_any_critical,
        }

    @classmethod
    def from_dict(cls, data: dict) -> "EvalThresholds":
        return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})

    @classmethod
    def from_env(cls) -> "EvalThresholds":
        """Load thresholds from environment variables."""
        return cls(
            min_pass_rate=float(os.getenv("FAMILIAR_MIN_PASS_RATE", DEFAULT_PASS_RATE_THRESHOLD)),
            max_critical_failures=int(
                os.getenv("FAMILIAR_MAX_CRITICAL_FAILURES", DEFAULT_CRITICAL_FAILURES_ALLOWED)
            ),
            max_high_failures=int(os.getenv("FAMILIAR_MAX_HIGH_FAILURES", 5)),
            max_avg_latency_ms=float(os.getenv("FAMILIAR_MAX_LATENCY_MS", DEFAULT_MAX_LATENCY_MS)),
            max_total_cost_usd=float(os.getenv("FAMILIAR_MAX_COST_USD", DEFAULT_MAX_COST_USD)),
            fail_on_any_critical=os.getenv("FAMILIAR_FAIL_ON_CRITICAL", "true").lower() == "true",
        )


@dataclass
class CIConfig:
    """
    Configuration for CI/CD evaluation runs.
    """

    suite_path: str
    thresholds: EvalThresholds = field(default_factory=EvalThresholds)
    output_dir: str = "./eval-results"
    output_formats: List[OutputFormat] = field(default_factory=lambda: [OutputFormat.ALL])
    provider: Optional[str] = None  # Auto-detected
    model: str = "gpt-4"
    api_key_env: str = "OPENAI_API_KEY"
    parallel: bool = False
    max_workers: int = 4
    timeout_seconds: int = 300
    verbose: bool = False
    dry_run: bool = False

    def __post_init__(self):
        if self.provider is None:
            self.provider = detect_ci_provider()

    def to_dict(self) -> dict:
        return {
            "suite_path": self.suite_path,
            "thresholds": self.thresholds.to_dict(),
            "output_dir": self.output_dir,
            "output_formats": [f.value for f in self.output_formats],
            "provider": self.provider,
            "model": self.model,
            "parallel": self.parallel,
            "max_workers": self.max_workers,
            "timeout_seconds": self.timeout_seconds,
        }

    @classmethod
    def from_file(cls, path: Union[str, Path]) -> "CIConfig":
        """Load config from JSON file."""
        data = json.loads(Path(path).read_text())
        thresholds = EvalThresholds.from_dict(data.get("thresholds", {}))
        formats = [OutputFormat(f) for f in data.get("output_formats", ["all"])]
        return cls(
            suite_path=data["suite_path"],
            thresholds=thresholds,
            output_dir=data.get("output_dir", "./eval-results"),
            output_formats=formats,
            model=data.get("model", "gpt-4"),
            api_key_env=data.get("api_key_env", "OPENAI_API_KEY"),
            parallel=data.get("parallel", False),
            max_workers=data.get("max_workers", 4),
            timeout_seconds=data.get("timeout_seconds", 300),
        )


# ============================================================
# CI PROVIDER DETECTION
# ============================================================


def detect_ci_provider() -> str:
    """Detect the current CI provider from environment."""
    if os.getenv("GITHUB_ACTIONS") == "true":
        return CIProvider.GITHUB_ACTIONS.value
    if os.getenv("GITLAB_CI") == "true":
        return CIProvider.GITLAB_CI.value
    if os.getenv("CI") == "true":
        return CIProvider.GENERIC.value
    return "local"


def is_ci_environment() -> bool:
    """Check if running in a CI environment."""
    return os.getenv("CI") == "true" or detect_ci_provider() != "local"


def get_ci_metadata() -> dict:
    """Get CI-specific metadata."""
    provider = detect_ci_provider()

    metadata = {
        "provider": provider,
        "is_ci": is_ci_environment(),
        "timestamp": _utcnow().isoformat(),
    }

    if provider == CIProvider.GITHUB_ACTIONS.value:
        metadata.update(
            {
                "repository": os.getenv("GITHUB_REPOSITORY", ""),
                "ref": os.getenv("GITHUB_REF", ""),
                "sha": os.getenv("GITHUB_SHA", ""),
                "run_id": os.getenv("GITHUB_RUN_ID", ""),
                "run_number": os.getenv("GITHUB_RUN_NUMBER", ""),
                "actor": os.getenv("GITHUB_ACTOR", ""),
                "workflow": os.getenv("GITHUB_WORKFLOW", ""),
                "event_name": os.getenv("GITHUB_EVENT_NAME", ""),
            }
        )
    elif provider == CIProvider.GITLAB_CI.value:
        metadata.update(
            {
                "project_path": os.getenv("CI_PROJECT_PATH", ""),
                "ref": os.getenv("CI_COMMIT_REF_NAME", ""),
                "sha": os.getenv("CI_COMMIT_SHA", ""),
                "pipeline_id": os.getenv("CI_PIPELINE_ID", ""),
                "job_id": os.getenv("CI_JOB_ID", ""),
                "user": os.getenv("GITLAB_USER_LOGIN", ""),
                "pipeline_source": os.getenv("CI_PIPELINE_SOURCE", ""),
            }
        )

    return metadata


# ============================================================
# THRESHOLD CHECKING
# ============================================================


@dataclass
class ThresholdResult:
    """Result of threshold checking."""

    passed: bool
    violations: List[str] = field(default_factory=list)
    warnings: List[str] = field(default_factory=list)

    def to_dict(self) -> dict:
        return {"passed": self.passed, "violations": self.violations, "warnings": self.warnings}


def check_thresholds(report_dict: dict, thresholds: EvalThresholds) -> ThresholdResult:
    """
    Check evaluation report against thresholds.

    Args:
        report_dict: Evaluation report as dictionary
        thresholds: Threshold configuration

    Returns:
        ThresholdResult with pass/fail and violation details
    """
    violations = []
    warnings = []

    # Pass rate
    pass_rate = report_dict.get("pass_rate", 0)
    if pass_rate < thresholds.min_pass_rate:
        violations.append(f"Pass rate {pass_rate:.1%} below minimum {thresholds.min_pass_rate:.1%}")
    elif pass_rate < thresholds.min_pass_rate + 0.1:
        warnings.append(
            f"Pass rate {pass_rate:.1%} close to threshold {thresholds.min_pass_rate:.1%}"
        )

    # Critical failures
    critical = report_dict.get("critical_failures", 0)
    if thresholds.fail_on_any_critical and critical > 0:
        violations.append(f"{critical} critical failure(s) detected")
    elif critical > thresholds.max_critical_failures:
        violations.append(
            f"{critical} critical failures exceed maximum {thresholds.max_critical_failures}"
        )

    # High severity failures
    high = report_dict.get("high_failures", 0)
    if high > thresholds.max_high_failures:
        violations.append(
            f"{high} high-severity failures exceed maximum {thresholds.max_high_failures}"
        )
    elif high > thresholds.max_high_failures * 0.8:
        warnings.append(
            f"{high} high-severity failures approaching limit {thresholds.max_high_failures}"
        )

    # Latency
    latency = report_dict.get("avg_latency_ms", 0)
    if latency > thresholds.max_avg_latency_ms:
        violations.append(
            f"Average latency {latency:.0f}ms exceeds maximum {thresholds.max_avg_latency_ms:.0f}ms"
        )

    # Cost
    cost = report_dict.get("total_cost_usd", 0)
    if cost > thresholds.max_total_cost_usd:
        violations.append(
            f"Total cost ${cost:.4f} exceeds maximum ${thresholds.max_total_cost_usd:.4f}"
        )
    elif cost > thresholds.max_total_cost_usd * 0.8:
        warnings.append(
            f"Total cost ${cost:.4f} approaching limit ${thresholds.max_total_cost_usd:.4f}"
        )

    return ThresholdResult(passed=len(violations) == 0, violations=violations, warnings=warnings)


# ============================================================
# OUTPUT FORMATTERS
# ============================================================


def to_junit_xml(report_dict: dict, ci_metadata: Optional[dict] = None) -> str:
    """
    Convert evaluation report to JUnit XML format.

    This format is widely supported by CI systems for test reporting.
    """
    testsuites = ET.Element("testsuites")
    testsuites.set("name", "Familiar Evaluation")
    testsuites.set("tests", str(report_dict.get("total_tests", 0)))
    testsuites.set("failures", str(report_dict.get("failed_tests", 0)))
    testsuites.set("time", str(report_dict.get("duration_seconds", 0)))

    # Create testsuite
    testsuite = ET.SubElement(testsuites, "testsuite")
    testsuite.set("name", report_dict.get("suite_name", "evaluation"))
    testsuite.set("tests", str(report_dict.get("total_tests", 0)))
    testsuite.set("failures", str(report_dict.get("failed_tests", 0)))
    testsuite.set("time", str(report_dict.get("duration_seconds", 0)))
    testsuite.set("timestamp", report_dict.get("started_at", ""))

    # Add properties
    properties = ET.SubElement(testsuite, "properties")

    prop = ET.SubElement(properties, "property")
    prop.set("name", "pass_rate")
    prop.set("value", f"{report_dict.get('pass_rate', 0):.2%}")

    prop = ET.SubElement(properties, "property")
    prop.set("name", "avg_latency_ms")
    prop.set("value", str(report_dict.get("avg_latency_ms", 0)))

    prop = ET.SubElement(properties, "property")
    prop.set("name", "total_cost_usd")
    prop.set("value", f"{report_dict.get('total_cost_usd', 0):.4f}")

    if ci_metadata:
        for key, value in ci_metadata.items():
            if value:
                prop = ET.SubElement(properties, "property")
                prop.set("name", f"ci.{key}")
                prop.set("value", str(value))

    # Add test cases
    for result in report_dict.get("results", []):
        testcase = ET.SubElement(testsuite, "testcase")
        testcase.set("name", result.get("test_name", "unknown"))
        testcase.set("classname", f"familiar.{report_dict.get('suite_name', 'eval')}")
        testcase.set("time", str(result.get("latency_ms", 0) / 1000))

        if not result.get("passed", False):
            failure = ET.SubElement(testcase, "failure")
            failure.set("type", result.get("severity", "medium"))
            failures = result.get("failures", [])
            failure.text = "\n".join(failures) if failures else "Test failed"

    # Pretty print
    xml_str = ET.tostring(testsuites, encoding="unicode")
    dom = minidom.parseString(xml_str)
    return dom.toprettyxml(indent="  ")


def to_markdown_report(
    report_dict: dict, threshold_result: ThresholdResult, ci_metadata: Optional[dict] = None
) -> str:
    """
    Generate a Markdown report suitable for PR comments.
    """
    lines = []

    # Header with status
    status_emoji = "✅" if threshold_result.passed else "❌"
    status_text = "PASSED" if threshold_result.passed else "FAILED"
    lines.append(f"## {status_emoji} Evaluation Report: {status_text}")
    lines.append("")

    # Summary table
    lines.append("### Summary")
    lines.append("")
    lines.append("| Metric | Value |")
    lines.append("|--------|-------|")
    lines.append(f"| Suite | {report_dict.get('suite_name', 'N/A')} |")
    lines.append(f"| Pass Rate | {report_dict.get('pass_rate', 0):.1%} |")
    lines.append(
        f"| Tests | {report_dict.get('passed_tests', 0)}/{report_dict.get('total_tests', 0)} |"
    )
    lines.append(f"| Critical Failures | {report_dict.get('critical_failures', 0)} |")
    lines.append(f"| Avg Latency | {report_dict.get('avg_latency_ms', 0):.0f}ms |")
    lines.append(f"| Total Cost | ${report_dict.get('total_cost_usd', 0):.4f} |")
    lines.append(f"| Duration | {report_dict.get('duration_seconds', 0):.1f}s |")
    lines.append("")

    # Violations
    if threshold_result.violations:
        lines.append("### ❌ Threshold Violations")
        lines.append("")
        for violation in threshold_result.violations:
            lines.append(f"- {violation}")
        lines.append("")

    # Warnings
    if threshold_result.warnings:
        lines.append("### ⚠️ Warnings")
        lines.append("")
        for warning in threshold_result.warnings:
            lines.append(f"- {warning}")
        lines.append("")

    # Failed tests (limit to 10)
    failed = [r for r in report_dict.get("results", []) if not r.get("passed", False)]
    if failed:
        lines.append("### Failed Tests")
        lines.append("")
        lines.append("<details>")
        lines.append(f"<summary>Show {len(failed)} failed test(s)</summary>")
        lines.append("")

        for result in failed[:10]:
            severity = result.get("severity", "medium").upper()
            lines.append(f"**[{severity}] {result.get('test_name', 'unknown')}**")
            for failure in result.get("failures", [])[:3]:
                lines.append(f"- {failure}")
            lines.append("")

        if len(failed) > 10:
            lines.append(f"*...and {len(failed) - 10} more failed tests*")

        lines.append("</details>")
        lines.append("")

    # CI metadata
    if ci_metadata and ci_metadata.get("sha"):
        lines.append("### CI Information")
        lines.append("")
        lines.append(f"- **Commit**: `{ci_metadata.get('sha', '')[:8]}`")
        if ci_metadata.get("ref"):
            lines.append(f"- **Branch**: `{ci_metadata.get('ref', '')}`")
        if ci_metadata.get("run_id"):
            lines.append(f"- **Run ID**: {ci_metadata.get('run_id', '')}")
        lines.append("")

    lines.append("---")
    lines.append(f"*Generated by Familiar v1.4.0 at {_utcnow().isoformat()}*")

    return "\n".join(lines)


def to_console_output(
    report_dict: dict, threshold_result: ThresholdResult, verbose: bool = False
) -> str:
    """Generate console output with ANSI colors."""
    lines = []

    # Colors (only if terminal supports it)
    use_colors = sys.stdout.isatty()
    GREEN = "\033[92m" if use_colors else ""
    RED = "\033[91m" if use_colors else ""
    YELLOW = "\033[93m" if use_colors else ""
    BOLD = "\033[1m" if use_colors else ""
    RESET = "\033[0m" if use_colors else ""

    # Header
    status_color = GREEN if threshold_result.passed else RED
    status = "PASSED" if threshold_result.passed else "FAILED"
    lines.append("")
    lines.append(f"{BOLD}{'=' * 60}{RESET}")
    lines.append(f"{BOLD}FAMILIAR EVALUATION: {status_color}{status}{RESET}")
    lines.append(f"{BOLD}{'=' * 60}{RESET}")
    lines.append("")

    # Summary
    lines.append(f"Suite: {report_dict.get('suite_name', 'N/A')}")
    lines.append(f"Pass Rate: {report_dict.get('pass_rate', 0):.1%}")
    lines.append(f"Tests: {report_dict.get('passed_tests', 0)}/{report_dict.get('total_tests', 0)}")
    lines.append(f"Critical Failures: {report_dict.get('critical_failures', 0)}")
    lines.append(f"Avg Latency: {report_dict.get('avg_latency_ms', 0):.0f}ms")
    lines.append(f"Total Cost: ${report_dict.get('total_cost_usd', 0):.4f}")
    lines.append("")

    # Violations
    if threshold_result.violations:
        lines.append(f"{RED}{BOLD}Threshold Violations:{RESET}")
        for v in threshold_result.violations:
            lines.append(f"  {RED}✗{RESET} {v}")
        lines.append("")

    # Warnings
    if threshold_result.warnings:
        lines.append(f"{YELLOW}{BOLD}Warnings:{RESET}")
        for w in threshold_result.warnings:
            lines.append(f"  {YELLOW}⚠{RESET} {w}")
        lines.append("")

    # Failed tests
    if verbose:
        failed = [r for r in report_dict.get("results", []) if not r.get("passed")]
        if failed:
            lines.append(f"{BOLD}Failed Tests:{RESET}")
            for r in failed[:5]:
                lines.append(
                    f"  {RED}✗{RESET} [{r.get('severity', 'medium').upper()}] {r.get('test_name')}"
                )
            if len(failed) > 5:
                lines.append(f"  ... and {len(failed) - 5} more")
            lines.append("")

    lines.append(f"{'=' * 60}")

    return "\n".join(lines)


# ============================================================
# GITHUB ACTIONS WORKFLOW GENERATOR
# ============================================================


def generate_github_workflow(
    suite_path: str = "tests/eval_suite.json",
    model: str = "gpt-4",
    python_version: str = "3.11",
    trigger_on_pr: bool = True,
    trigger_on_push: bool = True,
    branches: List[str] = None,
    thresholds: Optional[EvalThresholds] = None,
) -> str:
    """
    Generate a GitHub Actions workflow for evaluation.

    Args:
        suite_path: Path to evaluation suite JSON
        model: LLM model to use
        python_version: Python version
        trigger_on_pr: Trigger on pull requests
        trigger_on_push: Trigger on pushes
        branches: Branches to trigger on (default: main)
        thresholds: Custom thresholds

    Returns:
        YAML workflow content
    """
    if branches is None:
        branches = ["main"]

    if thresholds is None:
        thresholds = EvalThresholds()

    triggers = []
    if trigger_on_push:
        triggers.append(f"""  push:
    branches: [{", ".join(branches)}]""")
    if trigger_on_pr:
        triggers.append(f"""  pull_request:
    branches: [{", ".join(branches)}]""")

    workflow = f'''# Familiar LLM Evaluation Workflow
# Generated by Familiar v1.4.0 CI/CD Integration

name: LLM Evaluation

on:
{chr(10).join(triggers)}
  workflow_dispatch:
    inputs:
      suite:
        description: 'Evaluation suite path'
        required: false
        default: '{suite_path}'
      model:
        description: 'Model to evaluate'
        required: false
        default: '{model}'

env:
  FAMILIAR_MIN_PASS_RATE: "{thresholds.min_pass_rate}"
  FAMILIAR_MAX_CRITICAL_FAILURES: "{thresholds.max_critical_failures}"
  FAMILIAR_MAX_HIGH_FAILURES: "{thresholds.max_high_failures}"
  FAMILIAR_MAX_LATENCY_MS: "{thresholds.max_avg_latency_ms}"
  FAMILIAR_MAX_COST_USD: "{thresholds.max_total_cost_usd}"

jobs:
  evaluate:
    name: Run LLM Evaluation
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '{python_version}'
          cache: 'pip'

      - name: Install dependencies
        run: |
          pip install --upgrade pip
          pip install familiar-framework
          # Or: pip install -e .

      - name: Run evaluation
        id: eval
        env:
          OPENAI_API_KEY: ${{{{ secrets.OPENAI_API_KEY }}}}
          ANTHROPIC_API_KEY: ${{{{ secrets.ANTHROPIC_API_KEY }}}}
        run: |
          python -m familiar.eval \\
            --suite "${{{{ github.event.inputs.suite || '{suite_path}' }}}}" \\
            --model "${{{{ github.event.inputs.model || '{model}' }}}}" \\
            --output-dir ./eval-results \\
            --format all

      - name: Upload evaluation results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: eval-results-${{{{ github.run_id }}}}
          path: ./eval-results/
          retention-days: 30

      - name: Publish test results
        uses: EnricoMi/publish-unit-test-result-action@v2
        if: always()
        with:
          files: ./eval-results/junit.xml
          check_name: "Evaluation Results"
          comment_mode: always

      - name: Comment PR with results
        uses: actions/github-script@v7
        if: github.event_name == 'pull_request'
        with:
          script: |
            const fs = require('fs');
            const report = fs.readFileSync('./eval-results/report.md', 'utf8');

            github.rest.issues.createComment({{
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: report
            }});
'''

    return workflow


# ============================================================
# GITLAB CI TEMPLATE GENERATOR
# ============================================================


def generate_gitlab_ci(
    suite_path: str = "tests/eval_suite.json",
    model: str = "gpt-4",
    python_version: str = "3.11",
    thresholds: Optional[EvalThresholds] = None,
) -> str:
    """
    Generate a GitLab CI configuration for evaluation.

    Args:
        suite_path: Path to evaluation suite JSON
        model: LLM model to use
        python_version: Python version
        thresholds: Custom thresholds

    Returns:
        YAML GitLab CI content
    """
    if thresholds is None:
        thresholds = EvalThresholds()

    config = f'''# Familiar LLM Evaluation Pipeline
# Generated by Familiar v1.4.0 CI/CD Integration

stages:
  - evaluate

variables:
  FAMILIAR_MIN_PASS_RATE: "{thresholds.min_pass_rate}"
  FAMILIAR_MAX_CRITICAL_FAILURES: "{thresholds.max_critical_failures}"
  FAMILIAR_MAX_HIGH_FAILURES: "{thresholds.max_high_failures}"
  FAMILIAR_MAX_LATENCY_MS: "{thresholds.max_avg_latency_ms}"
  FAMILIAR_MAX_COST_USD: "{thresholds.max_total_cost_usd}"
  EVAL_SUITE: "{suite_path}"
  EVAL_MODEL: "{model}"

.eval_template:
  image: python:{python_version}-slim
  stage: evaluate
  before_script:
    - pip install --upgrade pip
    - pip install familiar-framework
  artifacts:
    when: always
    paths:
      - eval-results/
    reports:
      junit: eval-results/junit.xml
    expire_in: 30 days

evaluate:
  extends: .eval_template
  script:
    - |
      python -m familiar.eval \\
        --suite "$EVAL_SUITE" \\
        --model "$EVAL_MODEL" \\
        --output-dir ./eval-results \\
        --format all
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_PIPELINE_SOURCE == "web"

evaluate:manual:
  extends: .eval_template
  script:
    - |
      python -m familiar.eval \\
        --suite "${{MANUAL_SUITE:-$EVAL_SUITE}}" \\
        --model "${{MANUAL_MODEL:-$EVAL_MODEL}}" \\
        --output-dir ./eval-results \\
        --format all
  rules:
    - if: $CI_PIPELINE_SOURCE == "web"
      when: manual
  allow_failure: true

# Scheduled nightly evaluation
evaluate:nightly:
  extends: .eval_template
  script:
    - |
      python -m familiar.eval \\
        --suite "$EVAL_SUITE" \\
        --model "$EVAL_MODEL" \\
        --output-dir ./eval-results \\
        --format all \\
        --verbose
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
'''

    return config


# ============================================================
# CONFIGURATION GENERATOR
# ============================================================


def generate_eval_config(
    suite_path: str,
    output_dir: str = "./eval-results",
    model: str = "gpt-4",
    thresholds: Optional[EvalThresholds] = None,
) -> str:
    """Generate evaluation configuration file."""
    if thresholds is None:
        thresholds = EvalThresholds()

    config = {
        "suite_path": suite_path,
        "output_dir": output_dir,
        "model": model,
        "api_key_env": "OPENAI_API_KEY",
        "thresholds": thresholds.to_dict(),
        "output_formats": ["json", "junit", "markdown"],
        "parallel": False,
        "max_workers": 4,
        "timeout_seconds": 300,
    }

    return json.dumps(config, indent=2)


# ============================================================
# CLI RUNNER
# ============================================================


@dataclass
class EvalRunResult:
    """Result of a CI evaluation run."""

    success: bool
    exit_code: int
    report_dict: dict
    threshold_result: ThresholdResult
    output_files: Dict[str, Path]
    ci_metadata: dict

    def to_dict(self) -> dict:
        return {
            "success": self.success,
            "exit_code": self.exit_code,
            "report": self.report_dict,
            "thresholds": self.threshold_result.to_dict(),
            "output_files": {k: str(v) for k, v in self.output_files.items()},
            "ci_metadata": self.ci_metadata,
        }


class CIEvalRunner:
    """
    Runner for CI/CD evaluation pipelines.

    Handles evaluation execution, output generation, and threshold checking.
    """

    def __init__(self, config: CIConfig):
        self.config = config
        self.output_dir = Path(config.output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def run(self, evaluator_factory: Optional[Callable] = None) -> EvalRunResult:
        """
        Run evaluation and generate outputs.

        Args:
            evaluator_factory: Optional factory to create evaluator
                              If None, uses default Evaluator

        Returns:
            EvalRunResult with all outputs and status
        """
        ci_metadata = get_ci_metadata()
        output_files = {}

        # Load test suite
        suite_path = Path(self.config.suite_path)
        if not suite_path.exists():
            return EvalRunResult(
                success=False,
                exit_code=1,
                report_dict={"error": f"Suite not found: {suite_path}"},
                threshold_result=ThresholdResult(passed=False, violations=["Suite not found"]),
                output_files={},
                ci_metadata=ci_metadata,
            )

        # Import here to avoid circular imports
        try:
            from familiar.core.evaluation import load_test_suite
        except ImportError:
            # Fallback for standalone testing
            logger.warning("Could not import Evaluator, using mock")
            report_dict = self._run_mock_evaluation(suite_path)
        else:
            if self.config.dry_run:
                report_dict = self._run_mock_evaluation(suite_path)
            else:
                report_dict = self._run_real_evaluation(
                    suite_path, evaluator_factory, load_test_suite
                )

        # Check thresholds
        threshold_result = check_thresholds(report_dict, self.config.thresholds)

        # Generate outputs
        output_files = self._generate_outputs(report_dict, threshold_result, ci_metadata)

        # Print console output
        if (
            OutputFormat.CONSOLE in self.config.output_formats
            or OutputFormat.ALL in self.config.output_formats
        ):
            print(to_console_output(report_dict, threshold_result, self.config.verbose))

        return EvalRunResult(
            success=threshold_result.passed,
            exit_code=0 if threshold_result.passed else 1,
            report_dict=report_dict,
            threshold_result=threshold_result,
            output_files=output_files,
            ci_metadata=ci_metadata,
        )

    def _run_mock_evaluation(self, suite_path: Path) -> dict:
        """Run mock evaluation for dry-run mode."""
        logger.info(f"Dry run: would evaluate {suite_path}")
        return {
            "suite_name": suite_path.stem,
            "run_id": _utcnow().strftime("%Y%m%d_%H%M%S"),
            "pass_rate": 0.95,
            "total_tests": 10,
            "passed_tests": 9,
            "failed_tests": 1,
            "critical_failures": 0,
            "high_failures": 1,
            "avg_latency_ms": 1500,
            "total_cost_usd": 0.05,
            "total_tokens": 5000,
            "duration_seconds": 30,
            "started_at": _utcnow().isoformat(),
            "completed_at": _utcnow().isoformat(),
            "results": [
                {
                    "test_name": f"test_{i}",
                    "passed": i != 5,
                    "severity": "medium" if i != 5 else "high",
                    "failures": [] if i != 5 else ["Expected output not found"],
                    "latency_ms": 1500,
                }
                for i in range(10)
            ],
        }

    def _run_real_evaluation(
        self, suite_path: Path, evaluator_factory: Optional[Callable], load_test_suite: Callable
    ) -> dict:
        """Run actual evaluation using the Familiar Evaluator."""
        # Create evaluator
        if evaluator_factory:
            evaluator = evaluator_factory()
        else:
            from .evaluation import Evaluator

            evaluator = Evaluator(model=self.config.model)

        # Run evaluation - Evaluator.run_suite handles loading internally
        report = evaluator.run_suite(suite_path)

        return report.to_dict()

    def _generate_outputs(
        self, report_dict: dict, threshold_result: ThresholdResult, ci_metadata: dict
    ) -> Dict[str, Path]:
        """Generate all output files."""
        output_files = {}
        formats = self.config.output_formats

        # JSON report
        if OutputFormat.JSON in formats or OutputFormat.ALL in formats:
            json_path = self.output_dir / "report.json"
            full_report = {
                "report": report_dict,
                "thresholds": threshold_result.to_dict(),
                "ci_metadata": ci_metadata,
                "config": self.config.to_dict(),
            }
            json_path.write_text(json.dumps(full_report, indent=2))
            output_files["json"] = json_path
            logger.info(f"JSON report: {json_path}")

        # JUnit XML
        if OutputFormat.JUNIT in formats or OutputFormat.ALL in formats:
            junit_path = self.output_dir / "junit.xml"
            junit_path.write_text(to_junit_xml(report_dict, ci_metadata))
            output_files["junit"] = junit_path
            logger.info(f"JUnit XML: {junit_path}")

        # Markdown
        if OutputFormat.MARKDOWN in formats or OutputFormat.ALL in formats:
            md_path = self.output_dir / "report.md"
            md_path.write_text(to_markdown_report(report_dict, threshold_result, ci_metadata))
            output_files["markdown"] = md_path
            logger.info(f"Markdown report: {md_path}")

        return output_files


# ============================================================
# CLI ENTRYPOINT
# ============================================================


def create_cli_parser() -> argparse.ArgumentParser:
    """Create CLI argument parser."""
    parser = argparse.ArgumentParser(
        prog="familiar.eval", description="Familiar LLM Evaluation Runner for CI/CD"
    )

    subparsers = parser.add_subparsers(dest="command", help="Commands")

    # Run evaluation
    run_parser = subparsers.add_parser("run", help="Run evaluation suite")
    run_parser.add_argument("--suite", "-s", required=True, help="Path to evaluation suite JSON")
    run_parser.add_argument("--model", "-m", default="gpt-4", help="Model to evaluate")
    run_parser.add_argument("--output-dir", "-o", default="./eval-results", help="Output directory")
    run_parser.add_argument(
        "--format",
        "-f",
        nargs="+",
        default=["all"],
        choices=["json", "junit", "markdown", "console", "all"],
        help="Output formats",
    )
    run_parser.add_argument("--config", "-c", help="Config file path")
    run_parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
    run_parser.add_argument("--dry-run", action="store_true", help="Don't run actual evaluation")

    # Threshold options
    run_parser.add_argument("--min-pass-rate", type=float, help="Minimum pass rate (0-1)")
    run_parser.add_argument("--max-critical", type=int, help="Max critical failures")
    run_parser.add_argument("--max-latency", type=float, help="Max average latency (ms)")
    run_parser.add_argument("--max-cost", type=float, help="Max total cost (USD)")

    # Generate workflow
    gen_parser = subparsers.add_parser("generate", help="Generate CI configuration")
    gen_parser.add_argument(
        "--provider",
        "-p",
        required=True,
        choices=["github", "gitlab", "config"],
        help="CI provider or 'config' for eval config",
    )
    gen_parser.add_argument(
        "--suite", "-s", default="tests/eval_suite.json", help="Evaluation suite path"
    )
    gen_parser.add_argument("--model", "-m", default="gpt-4", help="Model to evaluate")
    gen_parser.add_argument("--output", "-o", help="Output file path")
    gen_parser.add_argument("--python-version", default="3.11", help="Python version")

    return parser


def cli_main(args: Optional[List[str]] = None) -> int:
    """CLI main entry point."""
    parser = create_cli_parser()
    parsed = parser.parse_args(args)

    if parsed.command == "run":
        # Build thresholds
        thresholds = EvalThresholds.from_env()
        if parsed.min_pass_rate is not None:
            thresholds.min_pass_rate = parsed.min_pass_rate
        if parsed.max_critical is not None:
            thresholds.max_critical_failures = parsed.max_critical
        if parsed.max_latency is not None:
            thresholds.max_avg_latency_ms = parsed.max_latency
        if parsed.max_cost is not None:
            thresholds.max_total_cost_usd = parsed.max_cost

        # Build config
        if parsed.config:
            config = CIConfig.from_file(parsed.config)
        else:
            config = CIConfig(
                suite_path=parsed.suite,
                thresholds=thresholds,
                output_dir=parsed.output_dir,
                output_formats=[OutputFormat(f) for f in parsed.format],
                model=parsed.model,
                verbose=parsed.verbose,
                dry_run=parsed.dry_run,
            )

        # Run
        runner = CIEvalRunner(config)
        result = runner.run()

        return result.exit_code

    elif parsed.command == "generate":
        thresholds = EvalThresholds()

        if parsed.provider == "github":
            content = generate_github_workflow(
                suite_path=parsed.suite,
                model=parsed.model,
                python_version=parsed.python_version,
                thresholds=thresholds,
            )
            default_output = ".github/workflows/evaluation.yml"
        elif parsed.provider == "gitlab":
            content = generate_gitlab_ci(
                suite_path=parsed.suite,
                model=parsed.model,
                python_version=parsed.python_version,
                thresholds=thresholds,
            )
            default_output = ".gitlab-ci-eval.yml"
        else:  # config
            content = generate_eval_config(
                suite_path=parsed.suite, model=parsed.model, thresholds=thresholds
            )
            default_output = "eval-config.json"

        output_path = parsed.output or default_output
        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
        Path(output_path).write_text(content)
        print(f"Generated: {output_path}")

        return 0

    else:
        parser.print_help()
        return 1


# ============================================================
# MODULE ENTRY POINT
# ============================================================

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    sys.exit(cli_main())


# ============================================================
# EXPORTS
# ============================================================

__all__ = [
    # Enums
    "OutputFormat",
    "CIProvider",
    # Config
    "EvalThresholds",
    "CIConfig",
    # Detection
    "detect_ci_provider",
    "is_ci_environment",
    "get_ci_metadata",
    # Threshold checking
    "ThresholdResult",
    "check_thresholds",
    # Output formatters
    "to_junit_xml",
    "to_markdown_report",
    "to_console_output",
    # Generators
    "generate_github_workflow",
    "generate_gitlab_ci",
    "generate_eval_config",
    # Runner
    "EvalRunResult",
    "CIEvalRunner",
    # CLI
    "create_cli_parser",
    "cli_main",
]
