# Copyright(C) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
# SPDX-License-Identifier: MIT

import argparse
import json
import time
from datetime import datetime
from enum import Enum
from pathlib import Path

from gaia.eval.claude import ClaudeClient
from gaia.eval.config import DEFAULT_CLAUDE_MODEL
from gaia.logger import get_logger


class UseCase(Enum):
    """Supported use cases for ground truth generation."""

    RAG = "rag"
    SUMMARIZATION = "summarization"
    QA = "qa"
    EMAIL = "email"


class GroundTruthGenerator:
    """Generates ground truth data for various evaluation use cases using Claude."""

    @staticmethod
    def get_default_prompt(use_case=UseCase.RAG, num_samples=5):
        """Generate default prompt based on use case."""
        if use_case == UseCase.RAG:
            return f"""
    Given this document, generate exactly {num_samples} short queries a user may ask about the document
    and produce a set of ground truth answers to be used in validating a RAG system.
    Include a summary of the document in the queries. Return a json formatted list of
    query-response pairs formatted as follows:
    {{
        'source': 'path/to/document',
        'summary': 'summarized document',
        'qa_pairs': [
            {{'query': 'query1', 'response': 'response1'}},
            {{'query': 'query2', 'response': 'response2'}},
            ...
        ]
    }}

    Generate exactly {num_samples} qa_pairs - no more, no less.
    
    IMPORTANT: Return ONLY the JSON object with no additional text, explanations, or formatting.
    """
        elif use_case == UseCase.SUMMARIZATION:
            return f"""
    Given this transcript, generate comprehensive ground truth summaries and metadata for evaluation.
    Analyze the content and provide different types of summaries with evaluation criteria.
    Return a json formatted response as follows:
    {{
        'source': 'path/to/transcript',
        'transcript_metadata': {{
            'estimated_duration': 'estimated meeting duration',
            'participant_count': 'estimated number of participants',
            'meeting_type': 'type of meeting (e.g., standup, planning, review)'
        }},
        'summaries': {{
            'executive_summary': 'high-level overview for executives',
            'detailed_summary': 'comprehensive summary with key details',
            'action_items': ['list', 'of', 'action', 'items'],
            'key_decisions': ['list', 'of', 'key', 'decisions'],
            'participants': ['list', 'of', 'identified', 'participants'],
            'topics_discussed': ['list', 'of', 'main', 'topics']
        }},
        'evaluation_criteria': {{
            'summary_completeness': 'how complete should a good summary be',
            'summary_accuracy': 'what constitutes accurate information extraction for summaries',
            'relevance': 'what information is most relevant to include',
            'structure': 'how should the summary be structured'
        }}
    }}

    Focus on generating comprehensive summaries and metadata for transcript summarization evaluation.
    
    IMPORTANT: Return ONLY the JSON object with no additional text, explanations, or formatting.
    """
        elif use_case == UseCase.QA:
            return f"""
    Given this transcript, generate exactly {num_samples} relevant questions with accurate answers for evaluation.
    Focus on questions that would commonly be asked about this type of meeting or conversation.
    Return a json formatted response as follows:
    {{
        'source': 'path/to/transcript',
        'transcript_metadata': {{
            'estimated_duration': 'estimated meeting duration',
            'participant_count': 'estimated number of participants',
            'meeting_type': 'type of meeting (e.g., standup, planning, review)'
        }},
        'qa_pairs': [
            {{'query': 'What were the main topics discussed in this meeting?', 'response': 'detailed answer based on transcript content'}},
            {{'query': 'What action items were assigned and to whom?', 'response': 'specific action items and assignees'}},
            {{'query': 'What decisions were made during this meeting?', 'response': 'key decisions and rationale'}},
            {{'query': 'Who participated in this meeting and what were their roles?', 'response': 'participant list and contributions'}},
            {{'query': 'What are the next steps or follow-up items?', 'response': 'future actions and timelines'}}
        ],
        'evaluation_criteria': {{
            'qa_accuracy': 'what constitutes accurate answers to questions about the transcript',
            'relevance': 'what information is most relevant to include in answers',
            'completeness': 'how complete should answers be for different question types'
        }}
    }}

    Generate exactly {num_samples} qa_pairs - no more, no less. Focus on questions that would be commonly asked about this type of meeting transcript.
    
    IMPORTANT: Return ONLY the JSON object with no additional text, explanations, or formatting.
    """
        elif use_case == UseCase.EMAIL:
            return f"""
    Given this business email, generate comprehensive ground truth summaries and analysis for evaluation.
    Analyze the email content and provide structured summaries with evaluation criteria.
    Return a json formatted response as follows:
    {{
        'source': 'path/to/email',
        'email_metadata': {{
            'email_type': 'type of email (e.g., project_update, customer_support, sales_outreach)',
            'sender_role': 'estimated role of sender',
            'recipient_type': 'type of recipients',
            'urgency_level': 'low/medium/high priority assessment'
        }},
        'summaries': {{
            'executive_summary': 'high-level overview of email purpose and content',
            'detailed_summary': 'comprehensive summary with key details and context',
            'key_points': ['list', 'of', 'main', 'points'],
            'action_items': ['list', 'of', 'action', 'items', 'or', 'requests'],
            'decisions_mentioned': ['list', 'of', 'decisions', 'or', 'announcements'],
            'follow_up_required': 'whether follow-up is needed and what type'
        }},
        'qa_pairs': [
            {{'query': 'What is the main purpose of this email?', 'response': 'detailed answer based on email content'}},
            {{'query': 'What action items or requests are mentioned?', 'response': 'specific actions requested'}},
            {{'query': 'What key information or updates are shared?', 'response': 'main information conveyed'}},
            {{'query': 'Who is the intended audience and what is expected of them?', 'response': 'recipient expectations and required responses'}},
            {{'query': 'What is the timeline or urgency level?', 'response': 'timing and priority information'}}
        ],
        'evaluation_criteria': {{
            'summary_completeness': 'how complete should a good email summary be',
            'summary_accuracy': 'what constitutes accurate information extraction for emails',
            'relevance': 'what information is most relevant to include',
            'context_understanding': 'how well should the business context be captured',
            'action_identification': 'how effectively should action items be identified'
        }}
    }}

    Focus on generating comprehensive summaries and analysis for business email evaluation. Always include exactly 5 qa_pairs.
    
    IMPORTANT: Return ONLY the JSON object with no additional text, explanations, or formatting.
    """
        else:
            raise ValueError(f"Unsupported use case: {use_case}")

    def __init__(self, model=None, max_tokens=4096):
        self.log = get_logger(__name__)
        if model is None:
            model = DEFAULT_CLAUDE_MODEL
        self.claude = ClaudeClient(model=model, max_tokens=max_tokens)

    def generate(
        self,
        file_path,
        use_case=UseCase.RAG,
        prompt=None,
        save_text=True,
        output_dir=None,
        num_samples=5,
        save_file=True,
    ):
        """
        Generate ground truth data for a given document based on use case.

        Args:
            file_path (str): Path to the input document
            use_case (UseCase): The evaluation use case (RAG or transcript summarization)
            prompt (str, optional): Custom prompt for Claude. If None, uses default prompt for use case
            save_text (bool): Whether to save extracted text for HTML files
            output_dir (str, optional): Directory to save output files. If None, uses same directory as input
            num_samples (int): Number of Q&A pairs to generate (for RAG use case only, default: 5)

        Returns:
            dict: Generated ground truth data with metadata
        """
        self.log.info(
            f"Generating ground truth data for: {file_path} (use case: {use_case.value})"
        )

        file_path = Path(file_path)
        if not file_path.exists():
            raise FileNotFoundError(f"Input file not found: {file_path}")

        # Use appropriate prompt based on use case
        if prompt is None:
            prompt = self.get_default_prompt(use_case, num_samples)

        try:
            # Generate analysis using Claude with usage tracking
            response = self.claude.analyze_file_with_usage(
                str(file_path), prompt, save_text=save_text
            )
            analysis = response["content"]
            usage = response["usage"]
            cost = response["cost"]

            # Debug: Log the raw analysis response
            self.log.debug(
                f"Raw Claude response length: {len(analysis) if analysis else 0}"
            )
            self.log.debug(
                f"Raw Claude response preview: {analysis[:500] if analysis else 'None'}"
            )

            # Check if analysis is valid
            if not analysis or not analysis.strip():
                raise ValueError(
                    "Claude returned an empty response. This may be due to token limits or API issues."
                )

            # Try to parse the JSON response
            try:
                parsed_analysis = json.loads(analysis)
            except json.JSONDecodeError as je:
                # Try to extract JSON from the response if it's wrapped in text
                self.log.debug(
                    "Initial JSON parsing failed, attempting to extract JSON from response"
                )

                # Look for JSON content within the response
                start_idx = analysis.find("{")
                end_idx = analysis.rfind("}") + 1

                if start_idx >= 0 and end_idx > start_idx:
                    json_content = analysis[start_idx:end_idx]
                    try:
                        parsed_analysis = json.loads(json_content)
                        self.log.info("Successfully extracted JSON from response")
                    except json.JSONDecodeError:
                        self.log.error(
                            f"Failed to parse extracted JSON. Response content: {analysis}"
                        )
                        raise ValueError(
                            f"Claude response is not valid JSON: {str(je)}"
                        )
                else:
                    self.log.error(f"No JSON content found in response: {analysis}")
                    raise ValueError(
                        f"No valid JSON found in Claude response: {str(je)}"
                    )

            # Prepare output data with metadata including usage and cost
            output_data = {
                "metadata": {
                    "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                    "model": self.claude.model,
                    "source_file": str(file_path),
                    "use_case": use_case.value,
                    "prompt": prompt,
                    "num_samples_requested": (
                        num_samples if use_case == UseCase.RAG else None
                    ),
                    "usage": usage,
                    "cost": cost,
                },
                "analysis": parsed_analysis,
            }

            # Save to file if save_file is True
            if save_file:
                # Default output directory to ./groundtruth if not specified
                if output_dir is None:
                    output_dir = "./groundtruth"

                output_dir = Path(output_dir)
                output_dir.mkdir(parents=True, exist_ok=True)

                # If file_path is relative and has parent directories, preserve them
                if file_path.is_relative_to(Path.cwd()) and file_path.parent != Path(
                    "."
                ):
                    # Try to preserve relative directory structure
                    try:
                        relative_path = file_path.relative_to(Path.cwd())
                        relative_dir = relative_path.parent
                        output_subdir = output_dir / relative_dir
                        output_subdir.mkdir(parents=True, exist_ok=True)
                        output_path = (
                            output_subdir
                            / f"{file_path.stem}.{use_case.value}.groundtruth.json"
                        )
                    except ValueError:
                        # Fall back to flat structure if relative path calculation fails
                        output_path = (
                            output_dir
                            / f"{file_path.stem}.{use_case.value}.groundtruth.json"
                        )
                else:
                    output_path = (
                        output_dir
                        / f"{file_path.stem}.{use_case.value}.groundtruth.json"
                    )

                with open(output_path, "w", encoding="utf-8") as f:
                    json.dump(output_data, f, indent=2)
                self.log.info(f"Ground truth data saved to: {output_path}")

            return output_data

        except Exception as e:
            self.log.error(f"Error generating ground truth data: {e}")
            raise

    def generate_batch(
        self, input_dir, file_pattern="*", use_case=UseCase.RAG, force=False, **kwargs
    ):
        """
        Generate ground truth data for multiple documents in a directory.
        All results are automatically consolidated into a single JSON file.
        Supports resuming from interrupted runs by skipping files that already have ground truth.

        Args:
            input_dir (str): Directory containing input documents
            file_pattern (str): Glob pattern to match input files
            use_case (UseCase): The evaluation use case
            force (bool): If True, regenerate all ground truth files even if they exist (default: False)
            **kwargs: Additional arguments passed to generate()

        Returns:
            dict: Consolidated ground truth data for all documents
        """
        input_dir = Path(input_dir)
        if not input_dir.is_dir():
            raise NotADirectoryError(f"Input directory not found: {input_dir}")

        # Get output directory from kwargs, with fallback
        output_dir = kwargs.get("output_dir", "./output/groundtruth")
        output_dir = Path(output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)

        # Remove output_dir and force from kwargs to avoid individual file saves in the generate method
        generate_kwargs = {
            k: v for k, v in kwargs.items() if k not in ["output_dir", "force"]
        }

        results = []
        individual_files = []

        # Collect all matching files recursively
        matching_files = list(input_dir.rglob(file_pattern))
        self.log.info(
            f"Found {len(matching_files)} files matching pattern '{file_pattern}' in {input_dir}"
        )

        if not matching_files:
            self.log.warning(
                f"No files found matching pattern '{file_pattern}' in {input_dir}"
            )
            # List directory contents for debugging
            try:
                all_files = list(input_dir.rglob("*"))
                self.log.info(
                    f"All files in directory: {[str(f) for f in all_files[:10]]}"
                )  # Show first 10
            except Exception as e:
                self.log.error(f"Error listing directory contents: {e}")

        for match in matching_files:
            self.log.info(f"  Found: {match}")

        # Filter out metadata files
        filtered_files = []
        for f in matching_files:
            # Skip directories (rglob returns both files and directories)
            if f.is_dir():
                self.log.info(f"Skipping directory: {f.name}")
                continue
            # Skip metadata files
            if f.name in [
                "transcript_metadata.json",
                "metadata.json",
                "experiment_metadata.json",
            ]:
                self.log.info(f"Skipping metadata file: {f.name}")
                continue
            # Skip any file with 'metadata' in the name and .json extension
            if "metadata" in f.name.lower() and f.suffix == ".json":
                self.log.info(f"Skipping metadata file: {f.name}")
                continue
            filtered_files.append(f)

        if not filtered_files:
            self.log.warning(
                f"No valid files to process after filtering out metadata files"
            )
            return None

        self.log.info(
            f"Processing {len(filtered_files)} files (excluded {len(matching_files) - len(filtered_files)} metadata files)"
        )

        # Create progress tracking directory
        progress_dir = output_dir / f"groundtruth_{use_case.value}_progress"
        progress_dir.mkdir(parents=True, exist_ok=True)

        # Track statistics
        skipped_count = 0
        processed_count = 0
        error_count = 0

        # Track costs for this run (only newly generated files)
        run_usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
        run_cost = {"input_cost": 0.0, "output_cost": 0.0, "total_cost": 0.0}

        for i, file_path in enumerate(filtered_files):
            # Calculate the expected output path for this file
            relative_path = file_path.relative_to(input_dir)
            relative_dir = relative_path.parent
            output_subdir = output_dir / relative_dir
            individual_file_path = (
                output_subdir / f"{file_path.stem}.{use_case.value}.groundtruth.json"
            )

            # Check if ground truth already exists and skip if not forced
            if not force and individual_file_path.exists():
                self.log.info(
                    f"Skipping file {i+1}/{len(filtered_files)} (already exists): {file_path}"
                )
                self.log.debug(f"  Existing ground truth: {individual_file_path}")
                skipped_count += 1

                # Load existing result for consolidation
                try:
                    with open(individual_file_path, "r", encoding="utf-8") as f:
                        existing_result = json.load(f)
                        results.append(existing_result)
                        individual_files.append(individual_file_path)
                    # Successfully loaded existing file, skip to next iteration
                    continue
                except Exception as e:
                    self.log.error(
                        f"Error loading existing ground truth {individual_file_path}: {e}"
                    )
                    self.log.warning(
                        f"Will regenerate ground truth for {file_path} due to load error"
                    )
                    # Decrement skip count since we're actually processing this file
                    skipped_count -= 1
                    # Fall through to regeneration below

            self.log.info(f"Processing file {i+1}/{len(filtered_files)}: {file_path}")
            file_start_time = time.time()

            try:
                # Generate without saving individual files by passing save_file=False
                result = self.generate(
                    file_path, use_case=use_case, save_file=False, **generate_kwargs
                )
                results.append(result)
                processed_count += 1

                # Accumulate costs for this run
                usage = result.get("metadata", {}).get("usage", {})
                cost = result.get("metadata", {}).get("cost", {})
                run_usage["input_tokens"] += usage.get("input_tokens", 0)
                run_usage["output_tokens"] += usage.get("output_tokens", 0)
                run_usage["total_tokens"] += usage.get("total_tokens", 0)
                run_cost["input_cost"] += cost.get("input_cost", 0.0)
                run_cost["output_cost"] += cost.get("output_cost", 0.0)
                run_cost["total_cost"] += cost.get("total_cost", 0.0)

                # Create output directory structure
                output_subdir.mkdir(parents=True, exist_ok=True)

                # Save individual file (this provides immediate incremental progress)
                with open(individual_file_path, "w", encoding="utf-8") as f:
                    json.dump(result, f, indent=2)
                individual_files.append(individual_file_path)

                # Write progress tracking information
                file_time = time.time() - file_start_time
                progress_file = progress_dir / f"file_{i+1:04d}_progress.json"
                progress_data = {
                    "file_index": i,
                    "file_path": str(file_path),
                    "individual_output_path": str(individual_file_path),
                    "processing_time_seconds": round(file_time, 3),
                    "usage": result.get("metadata", {}).get("usage", {}),
                    "cost": result.get("metadata", {}).get("cost", {}),
                    "timestamp": datetime.now().isoformat(),
                    "status": "completed",
                }

                with open(progress_file, "w", encoding="utf-8") as f:
                    json.dump(progress_data, f, indent=2)

                # Update overall progress
                overall_progress_file = progress_dir / "overall_progress.json"
                overall_progress_data = {
                    "total_files": len(filtered_files),
                    "completed_files": i + 1,
                    "progress_percent": round((i + 1) / len(filtered_files) * 100, 1),
                    "use_case": use_case.value,
                    "last_updated": datetime.now().isoformat(),
                }

                with open(overall_progress_file, "w", encoding="utf-8") as f:
                    json.dump(overall_progress_data, f, indent=2)

                self.log.info(
                    f"Ground truth progress: {i+1}/{len(filtered_files)} files completed ({overall_progress_data['progress_percent']}%)"
                )

            except Exception as e:
                self.log.error(f"Error processing {file_path}: {e}")
                error_count += 1

                # Write error progress information
                progress_file = progress_dir / f"file_{i+1:04d}_progress.json"
                progress_data = {
                    "file_index": i,
                    "file_path": str(file_path),
                    "error": str(e),
                    "timestamp": datetime.now().isoformat(),
                    "status": "error",
                }

                with open(progress_file, "w", encoding="utf-8") as f:
                    json.dump(progress_data, f, indent=2)

                continue

        # Log summary statistics
        self.log.info("=" * 60)
        self.log.info("Ground Truth Generation Summary:")
        self.log.info(f"  Total files found: {len(filtered_files)}")
        self.log.info(f"  Processed (new): {processed_count}")
        self.log.info(f"  Skipped (existing): {skipped_count}")
        self.log.info(f"  Errors: {error_count}")
        self.log.info(f"  Total results available: {len(results)}")
        if processed_count > 0:
            self.log.info(
                f"  This run cost: ${run_cost['total_cost']:.4f} "
                f"({run_usage['total_tokens']:,} tokens)"
            )
        self.log.info("=" * 60)

        if not results:
            self.log.warning("No files were processed successfully")
            return None

        # Consolidate all results into a single file
        try:
            self.log.info(
                f"Consolidating {len(results)} ground truth files into single JSON"
            )

            # Use the consolidate method to create the final consolidated file
            consolidated_file_pattern = f"*.{use_case.value}.groundtruth.json"
            consolidated_output_path = (
                output_dir / f"consolidated_{use_case.value}_groundtruth.json"
            )

            consolidated_data = self.consolidate_groundtruth(
                input_dir=output_dir,
                output_path=consolidated_output_path,
                file_pattern=consolidated_file_pattern,
            )

            # Keep individual files - don't clean them up
            self.log.info(f"Individual ground truth files saved in: {output_dir}")
            self.log.info(
                f"Consolidated ground truth file saved as: {consolidated_output_path}"
            )

            self.log.info(
                f"Consolidated ground truth data saved to: {consolidated_output_path}"
            )
            self.log.info(f"Total files processed: {len(results)}")

            # Add this run's statistics to consolidated metadata
            consolidated_data["metadata"]["run_stats"] = {
                "processed_count": processed_count,
                "skipped_count": skipped_count,
                "error_count": error_count,
                "run_usage": run_usage,
                "run_cost": run_cost,
            }

            # Return the consolidated data instead of individual results
            return consolidated_data

        except Exception as e:
            self.log.error(f"Error during consolidation: {e}")
            # If consolidation fails, return individual results
            self.log.warning("Falling back to individual results")
            return results
        finally:
            # Clean up progress directory after successful completion
            try:
                import shutil

                if progress_dir.exists():
                    shutil.rmtree(progress_dir)
                    self.log.info(
                        f"Cleaned up progress tracking files from: {progress_dir}"
                    )
            except Exception as e:
                self.log.warning(
                    f"Failed to clean up progress directory {progress_dir}: {e}"
                )

    def consolidate_groundtruth(
        self,
        input_dir,
        output_path=None,
        file_pattern="*.summarization.groundtruth.json",
    ):
        """
        Consolidate multiple ground truth files into a single JSON file for easier evaluation.

        Args:
            input_dir (str): Directory containing ground truth files
            output_path (str, optional): Path for consolidated output file. If None, creates in input_dir
            file_pattern (str): Glob pattern to match ground truth files

        Returns:
            dict: Consolidated ground truth data
        """
        input_dir = Path(input_dir)
        if not input_dir.is_dir():
            raise NotADirectoryError(f"Input directory not found: {input_dir}")

        # Find all matching ground truth files
        gt_files = list(input_dir.rglob(file_pattern))
        if not gt_files:
            raise FileNotFoundError(
                f"No ground truth files found matching pattern: {file_pattern}"
            )

        self.log.info(f"Consolidating {len(gt_files)} ground truth files")

        consolidated_data = {
            "metadata": {
                "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                "consolidated_from": len(gt_files),
                "source_files": [],
                "total_usage": {
                    "input_tokens": 0,
                    "output_tokens": 0,
                    "total_tokens": 0,
                },
                "total_cost": {
                    "input_cost": 0.0,
                    "output_cost": 0.0,
                    "total_cost": 0.0,
                },
            },
            "analysis": {
                "summaries": {},
                "transcript_metadata": {},
                "qa_pairs": {},
                "evaluation_criteria": {},
            },
        }

        # Process each ground truth file
        for gt_file in gt_files:
            try:
                with open(gt_file, "r", encoding="utf-8") as f:
                    gt_data = json.load(f)

                # Extract transcript identifier from filename (handle all use cases)
                transcript_id = gt_file.stem
                for use_case in UseCase:
                    transcript_id = transcript_id.replace(
                        f".{use_case.value}.groundtruth", ""
                    )

                # Store source file info
                consolidated_data["metadata"]["source_files"].append(
                    {
                        "transcript_id": transcript_id,
                        "file_path": str(gt_file),
                        "source_file": gt_data["metadata"].get("source_file", ""),
                        "timestamp": gt_data["metadata"].get("timestamp", ""),
                    }
                )

                # Aggregate usage and cost data
                usage = gt_data["metadata"].get("usage", {})
                cost = gt_data["metadata"].get("cost", {})

                consolidated_data["metadata"]["total_usage"][
                    "input_tokens"
                ] += usage.get("input_tokens", 0)
                consolidated_data["metadata"]["total_usage"][
                    "output_tokens"
                ] += usage.get("output_tokens", 0)
                consolidated_data["metadata"]["total_usage"][
                    "total_tokens"
                ] += usage.get("total_tokens", 0)

                consolidated_data["metadata"]["total_cost"]["input_cost"] += cost.get(
                    "input_cost", 0.0
                )
                consolidated_data["metadata"]["total_cost"]["output_cost"] += cost.get(
                    "output_cost", 0.0
                )
                consolidated_data["metadata"]["total_cost"]["total_cost"] += cost.get(
                    "total_cost", 0.0
                )

                # Store analysis data with transcript ID as key
                analysis = gt_data.get("analysis", {})
                consolidated_data["analysis"]["summaries"][transcript_id] = (
                    analysis.get("summaries", {})
                )
                consolidated_data["analysis"]["transcript_metadata"][transcript_id] = (
                    analysis.get("transcript_metadata", {})
                )

                # Store qa_pairs if present (for QA use case)
                if "qa_pairs" in analysis:
                    consolidated_data["analysis"]["qa_pairs"][transcript_id] = (
                        analysis.get("qa_pairs", [])
                    )

                # Store evaluation criteria (should be similar across transcripts, but keep first one)
                if not consolidated_data["analysis"]["evaluation_criteria"]:
                    consolidated_data["analysis"]["evaluation_criteria"] = analysis.get(
                        "evaluation_criteria", {}
                    )

                self.log.debug(f"Processed ground truth file: {gt_file}")

            except Exception as e:
                self.log.error(f"Error processing {gt_file}: {e}")
                continue

        # Set output path if not provided
        if output_path is None:
            output_path = input_dir / "consolidated_groundtruth.json"
        else:
            output_path = Path(output_path)

        # Save consolidated data
        with open(output_path, "w", encoding="utf-8") as f:
            json.dump(consolidated_data, f, indent=2, ensure_ascii=False)

        self.log.info(f"Consolidated ground truth saved to: {output_path}")
        self.log.info(f"Total files: {len(gt_files)}")
        self.log.info(
            f"Total cost: ${consolidated_data['metadata']['total_cost']['total_cost']:.4f}"
        )

        return consolidated_data


def main():
    """Command line interface for ground truth generation."""
    parser = argparse.ArgumentParser(
        description="Generate ground truth data for various evaluation use cases",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  # Process a single file for RAG evaluation (default)
  python -m gaia.eval.groundtruth -f ./data/html/blender/introduction.html

  # Process a transcript for summary generation
  python -m gaia.eval.groundtruth -f ./data/transcripts/meeting.txt --use-case summarization

  # Process a transcript for Q&A generation
  python -m gaia.eval.groundtruth -f ./data/transcripts/meeting.txt --use-case qa

  # Process all HTML files in a directory for RAG (creates consolidated file)
  python -m gaia.eval.groundtruth -d ./data/html/blender

  # Process transcript files for summarization (creates consolidated file)
  python -m gaia.eval.groundtruth -d ./data/transcripts -p "*.txt" --use-case summarization

  # Process transcript files for Q&A generation (creates consolidated file)
  python -m gaia.eval.groundtruth -d ./data/transcripts -p "*.txt" --use-case qa

  # Process with custom output directory
  python -m gaia.eval.groundtruth -f ./data/html/intro.html -o ./output/gt

  # Use custom Claude model
  python -m gaia.eval.groundtruth -f ./data/doc.html -m claude-3-opus-20240229

  # Generate 10 Q&A pairs per document (RAG only)
  python -m gaia.eval.groundtruth -d ./data/html/blender --num-samples 10

  # Consolidate multiple ground truth files into one
  python -m gaia.eval.groundtruth -d ./output/groundtruth --consolidate

  # Resume generation (skip files that already have ground truth)
  python -m gaia.eval.groundtruth -d ./data/transcripts -p "*.txt" --use-case summarization

  # Force regeneration of all files (even if ground truth exists)
  python -m gaia.eval.groundtruth -d ./data/transcripts -p "*.txt" --use-case summarization --force
        """,
    )

    # Input source (mutually exclusive)
    input_group = parser.add_mutually_exclusive_group(required=True)
    input_group.add_argument(
        "-f", "--file", type=str, help="Path to a single document file to process"
    )
    input_group.add_argument(
        "-d", "--directory", type=str, help="Directory containing documents to process"
    )

    # Optional arguments
    parser.add_argument(
        "-o",
        "--output-dir",
        type=str,
        default="./output/groundtruth",
        help="Output directory for generated ground truth files (default: ./output/groundtruth)",
    )
    parser.add_argument(
        "-p",
        "--pattern",
        type=str,
        default="*.html",
        help="File pattern to match when processing directory (default: *.html)",
    )
    parser.add_argument(
        "-u",
        "--use-case",
        type=str,
        choices=[uc.value for uc in UseCase],
        default=UseCase.RAG.value,
        help=f"Use case for ground truth generation (default: {UseCase.RAG.value})",
    )
    parser.add_argument(
        "-m",
        "--model",
        type=str,
        default=None,
        help=f"Claude model to use (default: {DEFAULT_CLAUDE_MODEL})",
    )
    parser.add_argument(
        "--max-tokens",
        type=int,
        default=4096,
        help="Maximum tokens for Claude responses (default: 4096)",
    )
    parser.add_argument(
        "--no-save-text",
        action="store_true",
        help="Don't save extracted text for HTML files",
    )
    parser.add_argument(
        "--custom-prompt",
        type=str,
        help="Path to a file containing a custom prompt for Claude",
    )
    parser.add_argument(
        "--num-samples",
        type=int,
        default=5,
        help="Number of Q&A pairs to generate per document (RAG use case only, default: 5)",
    )
    parser.add_argument(
        "--consolidate",
        action="store_true",
        help="Consolidate multiple ground truth files into a single file",
    )
    parser.add_argument(
        "--force",
        action="store_true",
        help="Force regeneration of all ground truth files, even if they already exist",
    )

    args = parser.parse_args()

    # Convert use case string to enum
    use_case = UseCase(args.use_case)

    # Initialize generator
    try:
        generator = GroundTruthGenerator(model=args.model, max_tokens=args.max_tokens)
    except Exception as e:
        print(f"Error initializing generator: {e}")
        return 1

    # Load custom prompt if provided
    custom_prompt = None
    if args.custom_prompt:
        try:
            with open(args.custom_prompt, "r", encoding="utf-8") as f:
                custom_prompt = f.read().strip()
            print(f"Using custom prompt from: {args.custom_prompt}")
        except Exception as e:
            print(f"Error loading custom prompt: {e}")
            return 1

    save_text = not args.no_save_text

    try:
        if args.consolidate:
            # Consolidate mode - directory is required
            if not args.directory:
                print("Error: --consolidate requires --directory (-d) to be specified")
                return 1

            # Use the specified use case for the consolidation pattern
            consolidate_pattern = f"*.{use_case.value}.groundtruth.json"
            print(f"Consolidating ground truth files from: {args.directory}")
            print(f"Pattern: {consolidate_pattern}")
            result = generator.consolidate_groundtruth(
                input_dir=args.directory,
                output_path=Path(args.output_dir)
                / f"consolidated_{use_case.value}_groundtruth.json",
                file_pattern=consolidate_pattern,
            )
            print(
                f"✅ Successfully consolidated {result['metadata']['consolidated_from']} files"
            )
            print(
                f"  Output: {Path(args.output_dir) / f'consolidated_{use_case.value}_groundtruth.json'}"
            )
            print(
                f"  Total cost: ${result['metadata']['total_cost']['total_cost']:.4f}"
            )
            print(
                f"  Total tokens: {result['metadata']['total_usage']['total_tokens']:,}"
            )

        elif args.file:
            # Process single file
            print(f"Processing single file: {args.file} (use case: {use_case.value})")
            result = generator.generate(
                file_path=args.file,
                use_case=use_case,
                prompt=custom_prompt,
                save_text=save_text,
                output_dir=args.output_dir,
                num_samples=args.num_samples,
            )
            print(f"✅ Successfully generated ground truth data")
            print(f"  Output: {args.output_dir}")
            usage = result["metadata"]["usage"]
            cost = result["metadata"]["cost"]
            print(
                f"  Token usage: {usage['input_tokens']:,} input + {usage['output_tokens']:,} output = {usage['total_tokens']:,} total"
            )
            print(
                f"  Cost: ${cost['input_cost']:.4f} input + ${cost['output_cost']:.4f} output = ${cost['total_cost']:.4f} total"
            )

            # Different output based on use case
            if use_case == UseCase.RAG:
                qa_pairs_count = len(result["analysis"]["qa_pairs"])
                print(
                    f"  Q&A pairs: {qa_pairs_count} (${cost['total_cost']/qa_pairs_count:.4f} per pair)"
                )
            elif use_case == UseCase.SUMMARIZATION:
                print(
                    f"  Summary types generated: {len(result['analysis']['summaries'])} different formats"
                )
                print(
                    f"  Evaluation criteria: {len(result['analysis']['evaluation_criteria'])} categories"
                )
            elif use_case == UseCase.QA:
                print(f"  Q&A pairs: {len(result['analysis']['qa_pairs'])}")
                print(
                    f"  Evaluation criteria: {len(result['analysis']['evaluation_criteria'])} categories"
                )

        elif args.directory:
            # Process directory
            print(
                f"Processing directory: {args.directory} (use case: {use_case.value})"
            )
            print(f"File pattern: {args.pattern}")
            if args.force:
                print("Force mode: Regenerating all ground truth files")
            else:
                print(
                    "Resume mode: Skipping files with existing ground truth (use --force to regenerate all)"
                )
            results = generator.generate_batch(
                input_dir=args.directory,
                file_pattern=args.pattern,
                use_case=use_case,
                force=args.force,
                prompt=custom_prompt,
                save_text=save_text,
                output_dir=args.output_dir,
                num_samples=args.num_samples,
            )

            if results:
                # Handle consolidated dict vs list of results
                if isinstance(results, dict):
                    # Consolidated results from generate_batch
                    # Cumulative totals (all files including existing ones)
                    cumulative_usage = results["metadata"]["total_usage"]
                    cumulative_cost = results["metadata"]["total_cost"]
                    num_files = results["metadata"]["consolidated_from"]
                    print(f"✅ Successfully processed batch")
                    print(f"  Output: {args.output_dir}")
                    print(
                        f"  Total files: {num_files} ({results['metadata'].get('source_files', []).__len__()} in consolidated output)"
                    )
                else:
                    # Fallback: list of individual results
                    total_usage = {
                        "input_tokens": sum(
                            r["metadata"]["usage"]["input_tokens"] for r in results
                        ),
                        "output_tokens": sum(
                            r["metadata"]["usage"]["output_tokens"] for r in results
                        ),
                        "total_tokens": sum(
                            r["metadata"]["usage"]["total_tokens"] for r in results
                        ),
                    }
                    total_cost = {
                        "input_cost": sum(
                            r["metadata"]["cost"]["input_cost"] for r in results
                        ),
                        "output_cost": sum(
                            r["metadata"]["cost"]["output_cost"] for r in results
                        ),
                        "total_cost": sum(
                            r["metadata"]["cost"]["total_cost"] for r in results
                        ),
                    }
                    num_files = len(results)
                    print(f"✅ Successfully processed {num_files} files")
                    print(f"  Output: {args.output_dir}")

                # Print this run's cost (if available)
                if isinstance(results, dict) and "run_stats" in results.get(
                    "metadata", {}
                ):
                    run_stats = results["metadata"]["run_stats"]
                    run_cost_data = run_stats["run_cost"]
                    run_usage_data = run_stats["run_usage"]
                    processed = run_stats["processed_count"]

                    if processed > 0:
                        print(
                            f"\n  This run (generated {processed} new file{'s' if processed != 1 else ''}):"
                        )
                        print(
                            f"    Cost: ${run_cost_data['total_cost']:.4f} "
                            f"({run_usage_data['total_tokens']:,} tokens)"
                        )

                # Print cumulative totals for dict, or totals for list
                label = "Cumulative total" if isinstance(results, dict) else "Total"
                usage_data = (
                    cumulative_usage if isinstance(results, dict) else total_usage
                )
                cost_data = cumulative_cost if isinstance(results, dict) else total_cost

                print(f"\n  {label} (all files):")
                print(
                    f"    Token usage: {usage_data['input_tokens']:,} input + "
                    f"{usage_data['output_tokens']:,} output = {usage_data['total_tokens']:,} total"
                )
                print(
                    f"    Cost: ${cost_data['input_cost']:.4f} input + "
                    f"${cost_data['output_cost']:.4f} output = ${cost_data['total_cost']:.4f} total"
                )
                if num_files > 0:
                    print(
                        f"    Average per file: ${cost_data['total_cost']/num_files:.4f}"
                    )

                # Different summary stats based on use case
                if isinstance(results, dict):
                    # Consolidated results
                    if use_case == UseCase.RAG:
                        # Count total Q&A pairs across all files
                        total_pairs = sum(
                            len(qa_pairs)
                            for qa_pairs in results["analysis"]
                            .get("qa_pairs", {})
                            .values()
                        )
                        print(f"  Total Q&A pairs: {total_pairs}")
                        if total_pairs > 0:
                            print(
                                f"  Average cost per Q&A pair: ${cumulative_cost['total_cost']/total_pairs:.4f}"
                            )
                    elif use_case == UseCase.SUMMARIZATION:
                        num_summaries = len(results["analysis"].get("summaries", {}))
                        print(
                            f"  Generated {num_summaries} comprehensive transcript summaries"
                        )
                    elif use_case == UseCase.QA:
                        num_qa = len(results["analysis"].get("qa_pairs", {}))
                        print(f"  Generated Q&A for {num_qa} transcripts")
                else:
                    # List results
                    if use_case == UseCase.RAG:
                        total_pairs = sum(
                            len(r["analysis"]["qa_pairs"]) for r in results
                        )
                        print(f"  Total Q&A pairs: {total_pairs}")
                        if total_pairs > 0:
                            print(
                                f"  Average cost per Q&A pair: ${total_cost['total_cost']/total_pairs:.4f}"
                            )
                    elif use_case == UseCase.SUMMARIZATION:
                        print(
                            f"  Generated {len(results)} comprehensive transcript summaries"
                        )
                    elif use_case == UseCase.QA:
                        print(f"  Generated {len(results)} Q&A pairs")
            else:
                print("No files were processed successfully")
                return 1

    except Exception as e:
        print(f"Error during processing: {e}")
        return 1

    return 0


if __name__ == "__main__":
    exit(main())
