"""
Spark Simplicity - JSON File Reader
===================================

Intelligent JSON file reader with automatic format detection and comprehensive error recovery.
This module provides enterprise-grade JSON ingestion capabilities that automatically detect
and handle multiple JSON formats including JSONL, JSON arrays, and Spark-generated formats.
Optimized for production environments with robust error handling and format compatibility.

Key Features:
    - **Automatic Format Detection**: Intelligent detection of JSONL, JSON arrays, and Spark formats
    - **Multi-Strategy Loading**: Cascading fallback through different parsing approaches
    - **Production Safety**: Comprehensive error handling and data validation
    - **Cluster Compatibility**: Integrated path validation for distributed processing
    - **Format Flexibility**: Support for various JSON formatting styles and structures
    - **Performance Optimization**: Efficient loading strategies based on format characteristics

Supported JSON Formats:
    **JSON Lines (JSONL)**:
    - One JSON object per line for streaming compatibility
    - Optimal for large datasets and ETL processing
    - Memory-efficient processing with line-by-line parsing
    - Default format for Spark JSON operations

    **JSON Arrays**:
    - Standard JSON array format [{}, {}, {}] for web APIs
    - Pretty-formatted with indentation for human readability
    - Complete arrays loaded into memory for processing

    **Spark Pretty Format**:
    - Multi-line JSON objects separated by whitespace
    - Generated by Spark with pretty printing enabled
    - Specialized parsing for Spark-specific formatting

Advanced Features:
    **Intelligent Format Detection**:
    - Automatic detection of optimal parsing strategy
    - Cascading fallback through multiple format attempts
    - Error recovery with detailed format-specific diagnostics
    - Performance optimization based on detected format

    **Data Quality Validation**:
    - Automatic validation of loaded DataFrame structure
    - Schema consistency checking across records
    - Data integrity verification for production workflows
    - Error reporting with actionable failure information

Enterprise Integration:
    - **API Data Processing**: Seamless handling of REST API responses
    - **Log Analysis**: Efficient processing of JSON-formatted logs
    - **Data Lake Integration**: Compatible with various JSON storage patterns
    - **ETL Pipeline Support**: Robust ingestion for data transformation workflows
    - **Streaming Compatibility**: Optimized for real-time data processing

Usage:
    This module provides the primary interface for JSON data ingestion in
    Spark Simplicity, with automatic format detection eliminating the need
    for manual format specification.

    from spark_simplicity.io.readers.json_reader import load_json
"""

from pathlib import Path
from typing import Optional, Union

from pyspark.sql import DataFrame, SparkSession

from ...logger import get_logger
from ..validation.path_utils import configure_spark_path
from .base_reader import _validate_json_dataframe

# Logger for JSON reader
_json_logger = get_logger("spark_simplicity.io.readers.json")


def _try_load_json_with_options(
    spark: SparkSession, spark_path: str, reader_options: dict, user_options: dict
) -> tuple[Optional[DataFrame], bool]:
    """
    Attempt JSON loading with specific reader configuration and comprehensive error handling.

    Provides controlled JSON loading attempts with specific Spark reader options while
    maintaining error isolation between different format strategies. This function enables
    the cascading fallback approach where multiple JSON formats are attempted sequentially
    until a successful parse is achieved.

    Args:
        spark: Active SparkSession instance for DataFrame creation and JSON processing
        spark_path: Properly configured Spark path (with appropriate URI scheme)
        reader_options: Dictionary of Spark DataFrameReader options for format-specific parsing:
                       - 'multiLine': Boolean for multi-line JSON object handling
                       - 'allowComments': Boolean for JavaScript-style comment support
                       - 'mode': Parsing mode ('PERMISSIVE', 'DROPMALFORMED', 'FAILFAST')
        user_options: User-provided additional options merged with reader_options

    Returns:
        Tuple containing (DataFrame_or_None, success_boolean):
        - DataFrame: Successfully loaded DataFrame if parsing succeeded
        - None: If parsing failed or data validation failed
        - Boolean: True if loading and validation both succeeded, False otherwise

    Error Handling Strategy:
        - Catches I/O errors, runtime errors, and parsing failures
        - Returns None/False tuple instead of propagating exceptions
        - Enables clean fallback to alternative parsing strategies
        - Preserves error isolation between different format attempts
    """
    try:
        reader = spark.read

        # Apply reader-specific options
        for key, value in reader_options.items():
            reader = reader.option(key, value)

        # Add user-provided options
        for key, value in user_options.items():
            reader = reader.option(key, value)

        df = reader.json(spark_path)

        # Validate DataFrame quality safely
        is_valid = _validate_json_dataframe(df)
        return df, is_valid

    except (OSError, IOError, RuntimeError):
        return None, False


def load_json(
    spark: SparkSession,
    file_path: Union[str, Path],
    shared_mount: bool = False,
    **options,
) -> DataFrame:
    """
    Load JSON files with intelligent format detection and enterprise-grade error recovery.

    Provides comprehensive JSON data ingestion with automatic format detection that eliminates
    the need for manual format specification. This function implements a sophisticated cascading
    strategy that attempts multiple JSON parsing approaches until successful data loading is
    achieved, making it robust for handling diverse JSON sources in production environments.

    The function prioritizes efficiency by attempting the most performant formats first (JSONL)
    before falling back to more resource-intensive approaches (multi-line JSON arrays), ensuring
    optimal performance while maintaining comprehensive format compatibility.

    Args:
        spark: Active SparkSession instance configured for distributed JSON processing.
              Must have appropriate executor resources for the expected data volume.
              Used for DataFrame creation, cluster validation, and distributed parsing.
        file_path: Path to JSON file requiring automatic format detection. Can be string
                  or Path object with .json extension (recommended but not enforced).
                  Supports absolute and relative paths with automatic resolution.
                  Compatible with local storage, network mounts, and cloud storage.
        shared_mount: Boolean indicating whether the file resides on shared storage
                     accessible by all cluster nodes. When True, triggers cluster-wide
                     validation to ensure all executors can access the JSON file.
                     When False, uses local file URI scheme for single-node access.
        **options: Additional Spark DataFrameReader options for fine-tuning JSON parsing:
                  - 'timestampFormat': Custom timestamp parsing pattern
                  - 'dateFormat': Custom date parsing pattern
                  - 'allowBackslashEscapingAnyCharacter': Boolean for escape handling
                  - 'allowUnquotedFieldNames': Boolean for non-standard JSON
                  - 'columnNameOfCorruptRecord': Column name for malformed records
                  - 'dropFieldIfAllNull': Boolean to drop all-null columns

    Returns:
        Spark DataFrame containing the loaded JSON data:
        - Schema automatically inferred from JSON structure
        - Complex nested structures preserved as struct/array types
        - Data types optimized based on JSON content analysis
        - Column names derived from JSON field names
        - Malformed records handled according to parsing mode

    Raises:
        FileNotFoundError: If the specified JSON file does not exist at the given path.
                          Error message includes full path for troubleshooting.
        RuntimeError: If all parsing strategies fail, indicating incompatible JSON format,
                     file corruption, permission issues, or cluster validation failures.
                     Error includes details of attempted strategies for diagnostics.

    Format Detection Strategy:
        The function attempts parsing in the following optimized order:

        **1. JSONL Format (Preferred)**:
        - Configuration: multiLine=False
        - Target: Line-delimited JSON objects
        - Performance: Highest (streaming processing)
        - Use Case: ETL pipelines, log processing, streaming data

        **2. Spark Pretty Format**:
        - Configuration: multiLine=True, allowComments=False, mode=PERMISSIVE
        - Target: Spark-generated multi-line JSON objects
        - Performance: Medium (specialized parsing)
        - Use Case: Spark output files with pretty formatting

        **3. JSON Array Format (Fallback)**:
        - Configuration: multiLine=True
        - Target: Standard JSON arrays with objects
        - Performance: Lower (full file parsing)
        - Use Case: API responses, web application data

    Examples:
        Automatic format detection for diverse JSON sources:

         # JSONL format (most common for data processing)
         jsonl_df = load_json(spark, "streaming_data.json")
         # Automatically detects and uses optimal JSONL parsing

         # JSON array format (common for API responses)
         api_df = load_json(spark, "api_response.json")
         # Falls back to JSON array parsing if needed

         # Shared storage with cluster validation
         shared_df = load_json(
        ...     spark,
        ...     "/nfs/data/json_dataset.json",
        ...     shared_mount=True
        ... )
         # Includes comprehensive cluster accessibility validation

        Advanced parsing with custom options:

         custom_df = load_json(
        ...     spark,
        ...     "complex_data.json",
        ...     timestampFormat="yyyy-MM-dd HH:mm:ss",
        ...     dateFormat="yyyy-MM-dd",
        ...     allowUnquotedFieldNames=True,  # Handle non-standard JSON
        ...     columnNameOfCorruptRecord="_corrupt_record"
        ... )
         # Handles complex JSON with custom parsing rules

        Batch processing with error handling:

         json_files = Path("json_data").glob("*.json")
         successful_loads = []

         for json_file in json_files:
        ...     try:
        ...         df = load_json(spark, json_file)
        ...         successful_loads.append((json_file.name, df.count()))
        ...         print(f"✓ {json_file.name}: {df.count()} records")
        ...     except RuntimeError as e:
        ...         print(f"✗ {json_file.name}: {e}")

    Performance Characteristics:
        **JSONL Format**: Optimal performance with streaming processing
        **JSON Arrays**: Good performance for moderate file sizes
        **Complex Nested**: Performance depends on nesting depth and structure
        **Large Files**: Distributed processing provides linear scaling

    Data Type Inference:
        - **Strings**: Automatic detection with UTF-8 support
        - **Numbers**: Intelligent integer/double discrimination
        - **Booleans**: Standard JSON boolean handling
        - **Dates/Timestamps**: Configurable parsing patterns
        - **Arrays**: Preserved as Spark array types
        - **Objects**: Converted to Spark struct types
        - **Nulls**: Proper null value handling across all types

    Common JSON Sources:
        **API Integration**:
        - REST API response processing
        - Webhook payload analysis
        - Third-party service data ingestion

        **Log Processing**:
        - Application log analysis
        - System event processing
        - Audit trail examination

        **Data Exchange**:
        - Inter-system data transfer
        - Data lake ingestion
        - ETL pipeline integration

    Error Recovery and Diagnostics:
        - **Format Identification**: Detailed logging of attempted formats
        - **Validation Results**: Data quality assessment for each attempt
        - **Fallback Tracking**: Progress through parsing strategies
        - **Error Aggregation**: Comprehensive error reporting for troubleshooting

    Production Considerations:
        - **Memory Usage**: JSONL format recommended for large files
        - **Cluster Resources**: Ensure adequate executor memory for JSON arrays
        - **Schema Evolution**: Monitor schema changes in evolving JSON sources
        - **Error Monitoring**: Implement alerting for parsing failures

    See Also:
        - ``_try_load_json_with_options()``: Individual format attempt logic
        - ``_validate_json_dataframe()``: Data quality validation implementation
        - CSV readers: ``load_csv()`` for structured tabular data
        - Parquet readers: ``load_parquet()`` for columnar analytics data

    Note:
        This function is designed for production JSON processing with emphasis on
        reliability and format compatibility. The automatic detection eliminates
        the need for format specification while ensuring optimal performance
        through intelligent strategy selection.
    """
    file_path = Path(file_path)

    if not file_path.exists():
        raise FileNotFoundError(
            f"File not found — please check that the path is correct and the "
            f"file exists: {file_path}"
        )

    # Configure Spark path with validation
    spark_path = configure_spark_path(file_path, shared_mount, spark)

    # Define loading strategies in order of preference
    load_strategies = [
        ({"multiLine": False}, "JSONL format"),
        (
            {"multiLine": True, "allowComments": False, "mode": "PERMISSIVE"},
            "Spark pretty format",
        ),
        ({"multiLine": True}, "pretty JSON array format"),
    ]

    try:
        # Try each loading strategy
        for reader_options, format_name in load_strategies:
            df, success = _try_load_json_with_options(
                spark, spark_path, reader_options, options
            )

            if success:
                _json_logger.info(
                    "JSON loaded successfully (%s): %s", format_name, file_path.name
                )
                return df

            _json_logger.debug("%s failed for %s", format_name, file_path.name)

        # All strategies failed
        raise RuntimeError(
            f"Unable to parse JSON file with any supported format: {file_path}\n"
            f"Tried: JSONL, Spark pretty format, and JSON array formats"
        )

    except FileNotFoundError:
        raise
    except RuntimeError:
        raise
    except Exception as e:
        raise RuntimeError(
            f"Could not load the file (please check file format and accessibility) : "
            f"JSON {file_path}: {str(e)}"
        ) from e
