"""
koci Pipeline Execution Engine

Orchestrates pipeline execution: stages, jobs, services, and artifacts.
"""

import os
import re
import signal
import sys
import time
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from typing import Optional, Callable
from datetime import datetime
from threading import Lock, Event

from ..core.models import Pipeline, Stage, Job, Step, Service, JobStatus
from ..core.expressions import ExecutionContext, evaluate_condition
from ..core.reports import ReportResult
from .container import ContainerRuntime, ContainerConfig, get_runtime
from .artifacts import ArtifactManager


# ANSI color codes
class Colors:
    """ANSI color codes for terminal output."""
    RESET = "\033[0m"
    BOLD = "\033[1m"
    DIM = "\033[2m"

    # Foreground colors
    RED = "\033[31m"
    GREEN = "\033[32m"
    YELLOW = "\033[33m"
    BLUE = "\033[34m"
    MAGENTA = "\033[35m"
    CYAN = "\033[36m"
    WHITE = "\033[37m"

    # Bright foreground
    BRIGHT_RED = "\033[91m"
    BRIGHT_GREEN = "\033[92m"
    BRIGHT_YELLOW = "\033[93m"
    BRIGHT_BLUE = "\033[94m"
    BRIGHT_CYAN = "\033[96m"

    @classmethod
    def disable(cls):
        """Disable colors (for non-TTY output)."""
        cls.RESET = ""
        cls.BOLD = ""
        cls.DIM = ""
        cls.RED = ""
        cls.GREEN = ""
        cls.YELLOW = ""
        cls.BLUE = ""
        cls.MAGENTA = ""
        cls.CYAN = ""
        cls.WHITE = ""
        cls.BRIGHT_RED = ""
        cls.BRIGHT_GREEN = ""
        cls.BRIGHT_YELLOW = ""
        cls.BRIGHT_BLUE = ""
        cls.BRIGHT_CYAN = ""


# Disable colors if not a TTY
if not sys.stdout.isatty():
    Colors.disable()


@dataclass
class StepResult:
    """Result of executing a single step."""
    step: Step
    success: bool
    exit_code: int
    output: str
    duration_seconds: float


@dataclass
class JobResult:
    """Result of executing a job."""
    job: Job
    status: JobStatus
    step_results: list[StepResult] = field(default_factory=list)
    start_time: Optional[datetime] = None
    end_time: Optional[datetime] = None
    error_message: Optional[str] = None
    report_result: Optional[ReportResult] = None

    @property
    def duration_seconds(self) -> float:
        if self.start_time and self.end_time:
            return (self.end_time - self.start_time).total_seconds()
        return 0.0

    @property
    def success(self) -> bool:
        return self.status == JobStatus.SUCCESS


@dataclass
class StageResult:
    """Result of executing a stage."""
    stage: Stage
    job_results: list[JobResult] = field(default_factory=list)
    start_time: Optional[datetime] = None
    end_time: Optional[datetime] = None

    @property
    def success(self) -> bool:
        # Stage is successful if no jobs FAILED (skipped jobs are OK)
        return all(
            jr.status != JobStatus.FAILED
            for jr in self.job_results
        )

    @property
    def duration_seconds(self) -> float:
        if self.start_time and self.end_time:
            return (self.end_time - self.start_time).total_seconds()
        return 0.0


@dataclass
class ExecutionResult:
    """Result of executing a full pipeline."""
    pipeline: Pipeline
    stage_results: list[StageResult] = field(default_factory=list)
    start_time: Optional[datetime] = None
    end_time: Optional[datetime] = None

    @property
    def success(self) -> bool:
        # Pipeline is successful if no stages have failed jobs
        if not self.stage_results:
            return True
        return all(sr.success for sr in self.stage_results)

    @property
    def duration_seconds(self) -> float:
        if self.start_time and self.end_time:
            return (self.end_time - self.start_time).total_seconds()
        return 0.0

    def get_job_result(self, job_name: str) -> Optional[JobResult]:
        """Get result for a specific job."""
        for sr in self.stage_results:
            for jr in sr.job_results:
                if jr.job.name == job_name:
                    return jr
        return None


class PipelineEngine:
    """
    Pipeline execution engine.

    Handles:
    - Stage execution in sequence
    - Job execution with dependency resolution
    - Service lifecycle management
    - Artifact handling
    - Condition evaluation
    """

    def __init__(
        self,
        runtime: Optional[ContainerRuntime] = None,
        workspace_path: Optional[str] = None,
        output_callback: Optional[Callable[[str], None]] = None,
        event_handler: Optional[object] = None,
        max_parallel: int = 4,
    ):
        """
        Initialize the engine.

        Args:
            runtime: Container runtime (auto-detected if not provided)
            workspace_path: Local workspace to mount (current dir if not provided)
            output_callback: Callback for streaming output
            event_handler: Optional object with event methods (stage_start, job_start, etc.)
            max_parallel: Maximum number of jobs to run in parallel (default 4, 1 = sequential)
        """
        self.runtime = runtime or get_runtime()
        self.workspace_path = workspace_path or os.getcwd()
        self.output_callback = output_callback or print
        self.event_handler = event_handler
        self.max_parallel = max_parallel

        self._artifact_manager: Optional[ArtifactManager] = None
        self._network_id: Optional[str] = None
        self._service_containers: list[str] = []
        self._job_containers: dict[str, str] = {}
        self._job_results: dict[str, JobStatus] = {}
        self._lock = Lock()  # Thread safety for shared state

        # Graceful shutdown support
        self._shutdown_event = Event()
        self._original_sigint = None
        self._original_sigterm = None

    def _emit(self, event: str, *args, **kwargs):
        """Emit an event to the event handler if one is registered."""
        if self.event_handler and hasattr(self.event_handler, event):
            getattr(self.event_handler, event)(*args, **kwargs)

    def _signal_handler(self, signum, frame):
        """Handle shutdown signals gracefully."""
        sig_name = signal.Signals(signum).name
        c = Colors

        if self._shutdown_event.is_set():
            # Second signal - force exit immediately (no cleanup)
            print(f"\n{c.RED}⚠ Force quit!{c.RESET}")
            os._exit(130)

        # First signal - graceful shutdown (with cleanup)
        self.output_callback(
            f"\n{c.YELLOW}⚠ Received {sig_name}, shutting down gracefully...{c.RESET}"
        )
        self.output_callback(
            f"{c.DIM}  (Press Ctrl+C again to force quit){c.RESET}"
        )
        self._shutdown_event.set()
        self._emit('shutdown', sig_name)

    def _setup_signal_handlers(self):
        """Install signal handlers for graceful shutdown."""
        self._original_sigint = signal.signal(signal.SIGINT, self._signal_handler)
        self._original_sigterm = signal.signal(signal.SIGTERM, self._signal_handler)

    def _restore_signal_handlers(self):
        """Restore original signal handlers."""
        if self._original_sigint is not None:
            signal.signal(signal.SIGINT, self._original_sigint)
        if self._original_sigterm is not None:
            signal.signal(signal.SIGTERM, self._original_sigterm)

    @property
    def is_shutting_down(self) -> bool:
        """Check if shutdown has been requested."""
        return self._shutdown_event.is_set()

    def run(
        self,
        pipeline: Pipeline,
        stage_filter: Optional[str] = None,
        context: Optional[ExecutionContext] = None,
    ) -> ExecutionResult:
        """
        Execute a pipeline.

        Args:
            pipeline: Pipeline to execute
            stage_filter: Optional stage name to run only that stage
            context: Execution context for conditions

        Returns:
            ExecutionResult with all results
        """
        result = ExecutionResult(pipeline=pipeline)
        result.start_time = datetime.now()

        # Setup
        run_id = str(uuid.uuid4())[:8]
        self._artifact_manager = ArtifactManager(pipeline.name, run_id)
        context = context or ExecutionContext()
        self._shutdown_event.clear()  # Reset shutdown state

        # Install signal handlers for graceful shutdown
        self._setup_signal_handlers()

        try:
            # Create network for this pipeline run
            network_name = f"koci-{pipeline.name}-{run_id}"
            self._network_id = self.runtime.create_network(network_name)

            # Determine stages to run
            stages_to_run = pipeline.stages
            if stage_filter:
                stages_to_run = [s for s in pipeline.stages if s.name == stage_filter]
                if not stages_to_run:
                    self.output_callback(f"Stage '{stage_filter}' not found")
                    return result

            # Execute stages sequentially
            for stage in stages_to_run:
                # Check for shutdown request
                if self.is_shutting_down:
                    c = Colors
                    self.output_callback(
                        f"{c.YELLOW}Skipping stage '{stage.name}' - shutdown requested{c.RESET}"
                    )
                    break

                stage_result = self._run_stage(stage, pipeline, context)
                result.stage_results.append(stage_result)

                # Stop if stage failed (unless all jobs have continue_on_error)
                if not stage_result.success:
                    all_continue = all(
                        jr.job.continue_on_error
                        for jr in stage_result.job_results
                        if not jr.success
                    )
                    if not all_continue:
                        c = Colors
                        self.output_callback(
                            f"{c.RED}Stage '{stage.name}' failed, stopping pipeline{c.RESET}"
                        )
                        break

        finally:
            # Cleanup resources
            c = Colors
            if self.is_shutting_down:
                self.output_callback(f"{c.YELLOW}Cleaning up resources...{c.RESET}")

            self._cleanup()

            # Restore original signal handlers
            self._restore_signal_handlers()

            if self.is_shutting_down:
                self.output_callback(f"{c.GREEN}✓ Cleanup complete{c.RESET}")

        result.end_time = datetime.now()
        return result

    def _run_stage(
        self,
        stage: Stage,
        pipeline: Pipeline,
        context: ExecutionContext,
    ) -> StageResult:
        """Execute a single stage."""
        result = StageResult(stage=stage)
        result.start_time = datetime.now()

        # Emit stage start event
        self._emit('stage_start', stage.name)

        c = Colors
        self.output_callback(f"\n{c.CYAN}{'='*60}{c.RESET}")
        self.output_callback(f"{c.BOLD}{c.CYAN}Stage: {stage.name}{c.RESET}")
        self.output_callback(f"{c.CYAN}{'='*60}{c.RESET}")

        # Start services for this stage
        self._start_services(stage.services)

        try:
            # Expand matrix jobs
            expanded_jobs = []
            for job in stage.jobs:
                expanded_jobs.extend(job.get_expanded_jobs())

            # Build dependency graph
            job_deps = {job.name: set(job.needs) for job in expanded_jobs}
            pending_jobs = set(job.name for job in expanded_jobs)
            # Include jobs completed in previous stages
            completed_jobs = set(self._job_results.keys())

            # Execute jobs respecting dependencies (with parallelism)
            stage_failed = False

            while pending_jobs and not stage_failed:
                # Check for shutdown request
                if self.is_shutting_down:
                    # Mark remaining jobs as cancelled
                    for remaining in pending_jobs:
                        remaining_job = next(j for j in expanded_jobs if j.name == remaining)
                        result.job_results.append(JobResult(
                            job=remaining_job,
                            status=JobStatus.SKIPPED,
                        ))
                        self._emit('job_end', remaining, 'cancelled', 0)
                    break

                # Find jobs ready to run (all dependencies satisfied)
                ready_jobs = [
                    job for job in expanded_jobs
                    if job.name in pending_jobs
                    and job_deps[job.name].issubset(completed_jobs)
                ]

                if not ready_jobs:
                    # Deadlock - remaining jobs have unsatisfied dependencies
                    c = Colors
                    self.output_callback(
                        f"{c.RED}Error: Circular or unsatisfied dependencies{c.RESET}"
                    )
                    break

                # Split ready jobs into parallel and sequential
                parallel_jobs = [j for j in ready_jobs if j.parallel]
                sequential_jobs = [j for j in ready_jobs if not j.parallel]

                # Run parallel jobs together (if any and parallelism enabled)
                if self.max_parallel > 1 and len(parallel_jobs) > 1:
                    batch_results = self._run_jobs_parallel(
                        parallel_jobs, pipeline, context
                    )
                    for job_result in batch_results:
                        result.job_results.append(job_result)
                        pending_jobs.discard(job_result.job.name)
                        completed_jobs.add(job_result.job.name)

                        with self._lock:
                            self._job_results[job_result.job.name] = job_result.status.value

                        # Check for failures
                        if not job_result.success and not job_result.job.continue_on_error:
                            stage_failed = True
                elif parallel_jobs:
                    # Single parallel job or parallelism disabled - run sequentially
                    sequential_jobs = parallel_jobs + sequential_jobs

                # Run sequential jobs one at a time
                for job in sequential_jobs:
                    if stage_failed:
                        break

                    job_result = self._run_job(job, pipeline, context)
                    result.job_results.append(job_result)

                    pending_jobs.remove(job.name)
                    completed_jobs.add(job.name)

                    # Update job results for condition evaluation
                    self._job_results[job.name] = job_result.status.value

                    # Stop stage if job failed and not continue_on_error
                    if not job_result.success and not job.continue_on_error:
                        stage_failed = True

            # Mark remaining jobs as skipped if stage failed
            if stage_failed and pending_jobs:
                for remaining in pending_jobs:
                    remaining_job = next(j for j in expanded_jobs if j.name == remaining)
                    result.job_results.append(JobResult(
                        job=remaining_job,
                        status=JobStatus.SKIPPED,
                    ))

        finally:
            # Stop services
            self._stop_services()

        result.end_time = datetime.now()

        # Emit stage end event
        self._emit('stage_end', stage.name, result.success, result.duration_seconds)

        return result

    def _run_jobs_parallel(
        self,
        jobs: list[Job],
        pipeline: Pipeline,
        context: ExecutionContext,
    ) -> list[JobResult]:
        """
        Execute multiple jobs in parallel.

        Args:
            jobs: Jobs to execute
            pipeline: Parent pipeline
            context: Execution context

        Returns:
            List of JobResults in completion order
        """
        results = []
        max_workers = min(self.max_parallel, len(jobs))

        c = Colors
        self.output_callback(
            f"\n  {c.CYAN}⚡ Running {len(jobs)} jobs in parallel "
            f"(max {max_workers} workers){c.RESET}"
        )

        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            # Submit all jobs
            future_to_job = {
                executor.submit(self._run_job, job, pipeline, context): job
                for job in jobs
            }

            # Collect results as they complete
            for future in as_completed(future_to_job):
                job = future_to_job[future]
                try:
                    job_result = future.result()
                    results.append(job_result)
                except Exception as e:
                    # Handle unexpected errors
                    results.append(JobResult(
                        job=job,
                        status=JobStatus.FAILED,
                        error_message=str(e),
                    ))

        return results

    def _run_job(
        self,
        job: Job,
        pipeline: Pipeline,
        context: ExecutionContext,
    ) -> JobResult:
        """Execute a single job."""
        result = JobResult(job=job, status=JobStatus.PENDING)
        result.start_time = datetime.now()

        # Check for shutdown before starting
        if self.is_shutting_down:
            result.status = JobStatus.SKIPPED
            result.end_time = datetime.now()
            self._emit('job_end', job.name, 'cancelled', 0)
            return result

        # Emit job start event
        self._emit('job_start', job.name)

        # Colorized job header
        c = Colors
        self.output_callback(f"\n  {c.BOLD}{c.WHITE}Job: {job.name}{c.RESET}")
        self.output_callback(f"  {c.DIM}Image: {job.image}{c.RESET}")
        self.output_callback(f"  {c.DIM}{'-'*50}{c.RESET}")

        # Evaluate conditions
        exec_context = ExecutionContext(
            branch=context.branch,
            tag=context.tag,
            event=context.event,
            env={**pipeline.env, **job.env, **context.env},
            variables=context.variables,
            job_results=self._job_results,
        )

        if not evaluate_condition(job.when, job.condition, exec_context):
            c = Colors
            self.output_callback(f"  {c.YELLOW}⊘ Skipped (condition not met){c.RESET}")
            result.status = JobStatus.SKIPPED
            result.end_time = datetime.now()
            self._emit('job_skipped', job.name)
            return result

        # Pull image if needed
        if not self.runtime.image_exists(job.image):
            c = Colors
            self.output_callback(f"  {c.CYAN}↓ Pulling image: {job.image}{c.RESET}")
            if not self.runtime.pull_image(job.image):
                result.status = JobStatus.FAILED
                result.error_message = f"Failed to pull image: {job.image}"
                result.end_time = datetime.now()
                return result

        # Create container
        container_id = None
        try:
            job_container_name = f"koci-{pipeline.name}-{job.name}-{uuid.uuid4().hex[:6]}"
            # Merge env: pipeline defaults < job overrides < context (secrets)
            merged_env = {**pipeline.env, **job.env, **context.env}

            # Expand ${VAR} references in env values
            merged_env = self._expand_env_vars(merged_env, context.env)
            config = ContainerConfig(
                image=job.image,
                name=job_container_name,
                working_dir=job.working_directory,
                env=merged_env,
                volumes={
                    self.workspace_path: {
                        'bind': job.working_directory,
                        'mode': 'rw',
                    }
                },
                network=self._network_id,
                labels={'koci': 'true', 'koci-job': job.name},
            )

            container_id = self.runtime.create_container(config)
            self._job_containers[job.name] = container_id
            self.runtime.start_container(container_id)

            # Inject artifacts from dependencies
            if self._artifact_manager and job.artifacts:
                self._artifact_manager.inject_artifacts(
                    job, self.runtime, container_id
                )

            # Execute steps
            result.status = JobStatus.RUNNING
            all_steps_success = True

            for step in job.steps:
                step_result = self._run_step(step, container_id, job)
                result.step_results.append(step_result)

                if not step_result.success and not step.continue_on_error:
                    all_steps_success = False
                    break

            # Collect and display test reports
            if job.reports and all_steps_success:
                report_result, reports_pass = self._collect_reports(
                    job, container_id
                )
                result.report_result = report_result
                if not reports_pass:
                    all_steps_success = False

            # Store artifacts
            if self._artifact_manager and job.artifacts and all_steps_success:
                self._artifact_manager.store_artifacts(
                    job, self.runtime, container_id
                )

            result.status = JobStatus.SUCCESS if all_steps_success else JobStatus.FAILED

        except Exception as e:
            result.status = JobStatus.FAILED
            result.error_message = str(e)
            c = Colors
            self.output_callback(f"  {c.RED}Error: {e}{c.RESET}")

        finally:
            # Cleanup container
            if container_id:
                self.runtime.stop_container(container_id)
                self.runtime.remove_container(container_id, force=True)

        result.end_time = datetime.now()

        c = Colors
        if result.success:
            status_icon = f"{c.BRIGHT_GREEN}✓{c.RESET}"
            status_text = f"{c.GREEN}{result.status.value}{c.RESET}"
        else:
            status_icon = f"{c.BRIGHT_RED}✗{c.RESET}"
            status_text = f"{c.RED}{result.status.value}{c.RESET}"

        self.output_callback(
            f"  {status_icon} {c.BOLD}Job {job.name}{c.RESET}: {status_text} "
            f"{c.DIM}({result.duration_seconds:.1f}s){c.RESET}"
        )

        # Emit report event and display summary if we collected reports
        if result.report_result:
            self._emit('job_report', job.name, result.report_result)
            self._display_report_summary(job, result.report_result)

        # Emit job end event
        self._emit('job_end', job.name, result.success, result.duration_seconds)

        return result

    def _run_step(
        self,
        step: Step,
        container_id: str,
        job: Job,
    ) -> StepResult:
        """Execute a single step."""
        start_time = time.time()

        # Check for shutdown before starting step
        if self.is_shutting_down:
            return StepResult(
                step=step,
                success=False,
                exit_code=-1,
                output="Cancelled - shutdown requested",
                duration_seconds=0,
            )

        # Emit step start event
        self._emit('step_start', job.name, step.name)

        c = Colors
        self.output_callback(f"    {c.CYAN}→{c.RESET} {step.name}")

        # Build command - wrap with exit code capture
        shell = step.shell or 'sh'
        working_dir = step.working_directory or job.working_directory
        # Only pass step-specific env overrides; job env is already in the container
        step_env = step.env if step.env else None

        try:
            # Execute command and capture exit code directly
            exec_result = self.runtime.exec_command(
                container_id,
                [shell, '-c', step.run],
                working_dir=working_dir,
                env=step_env,
            )

            # Output the result
            output = exec_result.output
            c = Colors
            for line in output.split('\n'):
                if line.strip():
                    self.output_callback(f"      {c.DIM}{line}{c.RESET}")

            exit_code = exec_result.exit_code
            success = exit_code == 0

        except Exception as e:
            output = str(e)
            exit_code = 1
            success = False
            c = Colors
            self.output_callback(f"      {c.RED}Error: {e}{c.RESET}")

        duration = time.time() - start_time

        result = StepResult(
            step=step,
            success=success,
            exit_code=exit_code,
            output=output,
            duration_seconds=duration,
        )

        c = Colors
        if success:
            status_icon = f"{c.BRIGHT_GREEN}✓{c.RESET}"
        else:
            status_icon = f"{c.BRIGHT_RED}✗{c.RESET}"
        self.output_callback(
            f"    {status_icon} {step.name} {c.DIM}({duration:.1f}s){c.RESET}"
        )

        # Emit step end event (include output for failed steps)
        self._emit('step_end', job.name, step.name, success, duration, output if not success else "")

        return result

    def _collect_reports(
        self,
        job: Job,
        container_id: str,
    ) -> tuple[ReportResult, bool]:
        """
        Collect test reports from the container.

        Args:
            job: Job with report configuration
            container_id: Container to collect reports from

        Returns:
            Tuple of (ReportResult, bool) where bool indicates if all thresholds passed
        """
        import tempfile

        reports = job.reports
        result = ReportResult()
        all_pass = True

        # Expand variables in report paths using job env
        def expand_path(path: str) -> str:
            if not path:
                return path
            for key, value in job.env.items():
                path = path.replace(f"${{{key}}}", str(value))
                path = path.replace(f"${key}", str(value))
            return path

        with tempfile.TemporaryDirectory() as tmpdir:
            # Collect JUnit reports
            if reports.junit:
                junit_path = expand_path(reports.junit)
                junit_result = self._collect_junit_reports(
                    container_id, junit_path, tmpdir, job
                )
                result.junit = junit_result
                if junit_result.tests > 0 and not junit_result.success:
                    all_pass = False

            # Collect coverage reports
            if reports.coverage:
                coverage_path = expand_path(reports.coverage.path)
                coverage_result = self._collect_coverage_report(
                    container_id, coverage_path,
                    reports.coverage.format, tmpdir, job
                )
                result.coverage = coverage_result

                if coverage_result.lines_total > 0:
                    threshold = reports.coverage.threshold
                    if threshold is not None and not coverage_result.meets_threshold(threshold):
                        all_pass = False

        return result, all_pass

    def _display_report_summary(self, job: Job, report_result: ReportResult):
        """
        Display a compact report summary after job completion.

        Shows a single informative line, with failure details only when needed.
        """
        c = Colors
        parts = []

        # JUnit summary
        if report_result.junit and report_result.junit.tests > 0:
            junit = report_result.junit
            if junit.success:
                parts.append(f"{c.GREEN}{junit.passed} passed{c.RESET}")
            else:
                parts.append(f"{c.GREEN}{junit.passed} passed{c.RESET}")
                if junit.failures > 0:
                    parts.append(f"{c.RED}{junit.failures} failed{c.RESET}")
                if junit.errors > 0:
                    parts.append(f"{c.RED}{junit.errors} errors{c.RESET}")
            if junit.skipped > 0:
                parts.append(f"{c.YELLOW}{junit.skipped} skipped{c.RESET}")

        # Coverage summary
        if report_result.coverage and report_result.coverage.lines_total > 0:
            cov = report_result.coverage
            threshold = job.reports.coverage.threshold if job.reports.coverage else None

            cov_color = c.GREEN
            if threshold and not cov.meets_threshold(threshold):
                cov_color = c.RED
            elif cov.line_percent < 50:
                cov_color = c.YELLOW

            cov_text = f"{cov_color}{cov.line_percent:.0f}% coverage{c.RESET}"
            if threshold:
                cov_text += f" {c.DIM}(min {threshold}%){c.RESET}"
            parts.append(cov_text)

        if parts:
            self.output_callback(f"    {c.DIM}├─{c.RESET} {' · '.join(parts)}")

        # Show failed tests only if there are failures
        if report_result.junit and not report_result.junit.success:
            failed_tests = report_result.junit.get_failed_tests()
            for tc in failed_tests[:3]:
                short_name = tc.name[:50] + "..." if len(tc.name) > 50 else tc.name
                self.output_callback(f"    {c.DIM}│{c.RESET}  {c.RED}✗ {short_name}{c.RESET}")
            if len(failed_tests) > 3:
                remaining = len(failed_tests) - 3
                self.output_callback(f"    {c.DIM}│  ... +{remaining} more{c.RESET}")

    def _collect_junit_reports(
        self,
        container_id: str,
        junit_path: str,
        tmpdir: str,
        job: Job,
    ):
        """Collect JUnit XML reports from container."""
        from ..core.reports import JUnitParser, JUnitResult
        import os

        parser = JUnitParser()

        # Get list of files matching pattern from container
        shell = 'sh'
        working_dir = job.working_directory

        # Use find command to list matching files
        if '*' in junit_path:
            # Glob pattern - use find
            base_dir = os.path.dirname(junit_path) or '.'
            pattern = os.path.basename(junit_path)
            find_cmd = f"find {base_dir} -name '{pattern}' -type f 2>/dev/null"
        else:
            find_cmd = f"test -f {junit_path} && echo {junit_path}"

        try:
            exec_result = self.runtime.exec_command(
                container_id,
                [shell, '-c', find_cmd],
                working_dir=working_dir,
            )
            files = [f.strip() for f in exec_result.output.split('\n') if f.strip()]
        except Exception:
            files = []

        if not files:
            return JUnitResult()

        # Copy each file and parse
        aggregated = JUnitResult()

        for remote_path in files:
            # Resolve path relative to working_dir if needed
            if not remote_path.startswith('/'):
                remote_path = f"{working_dir}/{remote_path}"

            filename = os.path.basename(remote_path)
            try:
                # copy_from_container extracts tar to tmpdir
                self.runtime.copy_from_container(
                    container_id,
                    remote_path,
                    tmpdir,
                )
                # File is now at tmpdir/filename
                local_path = os.path.join(tmpdir, filename)

                if os.path.exists(local_path):
                    file_result = parser.parse(local_path)
                    # Aggregate results
                    aggregated.tests += file_result.tests
                    aggregated.passed += file_result.passed
                    aggregated.failures += file_result.failures
                    aggregated.errors += file_result.errors
                    aggregated.skipped += file_result.skipped
                    aggregated.time += file_result.time
                    aggregated.test_cases.extend(file_result.test_cases)
            except Exception:
                continue

        return aggregated

    def _collect_coverage_report(
        self,
        container_id: str,
        coverage_path: str,
        coverage_format: str,
        tmpdir: str,
        job: Job,
    ):
        """Collect coverage report from container."""
        from ..core.reports import CoverageParser, CoverageResult
        import os

        parser = CoverageParser()
        working_dir = job.working_directory

        # Resolve path
        remote_path = coverage_path if coverage_path.startswith('/') else f"{working_dir}/{coverage_path}"

        # Check if file exists
        shell = 'sh'
        check_cmd = f"test -f {remote_path} && echo exists"

        try:
            exec_result = self.runtime.exec_command(
                container_id,
                [shell, '-c', check_cmd],
                working_dir=working_dir,
            )
            if 'exists' not in exec_result.output:
                return CoverageResult()
        except Exception:
            return CoverageResult()

        # Copy file - copy_from_container extracts tar to tmpdir
        filename = os.path.basename(coverage_path)
        try:
            self.runtime.copy_from_container(
                container_id,
                remote_path,
                tmpdir,
            )
            # File is now at tmpdir/filename
            local_path = os.path.join(tmpdir, filename)
            return parser.parse(local_path, coverage_format)
        except Exception:
            return CoverageResult()

    def _start_services(self, services: list[Service]):
        """Start service containers."""
        c = Colors
        for service in services:
            self.output_callback(
                f"  {c.MAGENTA}◆{c.RESET} Starting service: "
                f"{c.BOLD}{service.name}{c.RESET} {c.DIM}({service.image}){c.RESET}"
            )

            # Pull image if needed
            if not self.runtime.image_exists(service.image):
                self.runtime.pull_image(service.image, quiet=True)

            # Build port mappings
            ports = {}
            for port_spec in service.ports:
                if ':' in port_spec:
                    host, container = port_spec.split(':')
                    ports[container] = host
                else:
                    ports[port_spec] = port_spec

            config = ContainerConfig(
                image=service.image,
                name=f"koci-svc-{service.name}-{uuid.uuid4().hex[:6]}",
                env=service.env,
                ports=ports,
                network=self._network_id,
                labels={'koci': 'true', 'koci-service': service.name},
            )

            container_id = self.runtime.create_container(config)
            self.runtime.start_container(container_id)
            self._service_containers.append(container_id)

            # Wait a moment for service to start
            time.sleep(2)

    def _stop_services(self):
        """Stop all service containers."""
        for container_id in self._service_containers:
            self.runtime.stop_container(container_id)
            self.runtime.remove_container(container_id, force=True)
        self._service_containers.clear()

    def _expand_env_vars(self, env: dict, secrets: dict) -> dict:
        """
        Expand ${VAR} references in environment variable values.

        Args:
            env: Environment variables dict (may contain ${VAR} references)
            secrets: Secrets dict to use for lookups

        Returns:
            New dict with expanded values
        """
        result = {}
        # Pattern matches ${VAR_NAME}
        var_pattern = re.compile(r'\$\{([A-Za-z_][A-Za-z0-9_]*)\}')

        # Create a lookup dict: secrets take priority over env
        lookup = {**env, **secrets}

        for key, value in env.items():
            if isinstance(value, str) and '${' in value:
                # Find all ${VAR} references and expand them
                expanded = value
                for match in var_pattern.finditer(value):
                    var_name = match.group(1)
                    if var_name in lookup:
                        expanded = expanded.replace(match.group(0), str(lookup[var_name]))
                result[key] = expanded
            else:
                result[key] = value

        return result

    def _cleanup(self):
        """Clean up all resources (containers, networks, volumes) except images."""
        # Stop services
        self._stop_services()

        # Remove job containers
        for container_id in self._job_containers.values():
            self.runtime.stop_container(container_id)
            self.runtime.remove_container(container_id, force=True)
        self._job_containers.clear()

        # Clean up all koci-labeled resources (containers, networks, volumes)
        if hasattr(self.runtime, 'cleanup_koci_resources'):
            self.runtime.cleanup_koci_resources()

        self._network_id = None

        # Clean up old artifact runs
        if self._artifact_manager:
            self._artifact_manager.cleanup_old_runs(keep_runs=5)
