"""
PAMOLA.CORE - Privacy-Preserving AI Data Processors
Module:        Base Anonymization Operation
Package:       pamola_core.anonymization
Version:       3.0.0
Status:        stable
Author:        PAMOLA Core Team
Created:       Mar 2025
Updated:       2025-06-15
License:       BSD 3-Clause
Description:
  This module provides the base class for all anonymization operations,
  defining common functionality, interface, and behavior with enhanced
  support for conditional processing, profiling integration, memory-efficient
  operations, and Dask-based distributed processing for large datasets.

Key Features:
  - Standardized operation lifecycle with validation, execution, and result handling
  - Support for both in-place (REPLACE) and new field creation (ENRICH) modes
  - Configurable null value handling strategies (PRESERVE, EXCLUDE, ERROR)
  - Memory-efficient chunk-based processing for large datasets
  - Comprehensive metrics collection and visualization generation
  - Robust caching mechanism for operation results
  - Progress tracking and reporting throughout the operation
  - Secure output generation with optional encryption
  - Conditional processing based on field values and risk scores
  - Integration with k-anonymity profiling results
  - Enhanced memory management with automatic optimization
  - Dask integration for distributed processing of large datasets

Framework:
  Implementation follows the PAMOLA.CORE operation framework with standardized interfaces
  for input/output, progress tracking, and result reporting.

Changelog:
  v3.0.0 (2025-01-24):
      - Added full Dask support with automatic engine switching
      - Added pandas/Dask DataFrame conversion utilities
      - Added engine parameter (pandas/dask/auto)
      - Added dask_partition_size parameter for partition control
      - Implemented process_batch_dask() base method
      - Enhanced DataWriter integration for Dask DataFrames
      - Added distributed processing metrics collection
  v2.0.0 (2025-12-16):
      - Integrated with pamola_core/utils/ops/data_processing.py for memory optimization
      - Integrated with pamola_core/utils/ops/field_utils.py for field management
      - Added conditional processing support with complex conditions
      - Added k-anonymity risk-based processing
      - Enhanced memory management with automatic dtype optimization
      - Improved progress tracking with hierarchical trackers
      - Standardized artifact generation through framework utilities
"""

import time
import dask.dataframe as dd
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union

import pandas as pd

# Import anonymization-specific utilities
from pamola_core.anonymization.commons.data_utils import (
    handle_vulnerable_records,
    process_nulls,
)
from pamola_core.errors.codes import ErrorCode
from pamola_core.errors.error_handler import ErrorHandler
from pamola_core.errors.exceptions import (
    FeatureNotImplementedError,
    TypeValidationError,
    ValidationError,
)
from pamola_core.anonymization.commons.metric_utils import (
    calculate_anonymization_effectiveness,
)
from pamola_core.anonymization.commons.processing_utils import (
    process_dataframe_using_chunk,
    process_dataframe_using_dask,
    process_dataframe_using_joblib,
)
from pamola_core.anonymization.commons.visualization_utils import (
    create_metric_visualization,
    create_comparison_visualization,
    sample_large_dataset,
)
from pamola_core.common.constants import Constants
from pamola_core.common.helpers.data_helper import DataHelper
from pamola_core.utils.io import (
    load_data_operation,
    load_settings_operation,
)
from pamola_core.utils.io_helpers.crypto_utils import get_encryption_mode
from pamola_core.utils.ops.op_base import FieldOperation

# Import framework utilities
from pamola_core.utils.ops.op_cache import OperationCache
from pamola_core.utils.ops.op_data_processing import (
    optimize_dataframe_dtypes,
    get_memory_usage,
)
from pamola_core.utils.ops.op_data_source import DataSource
from pamola_core.utils.ops.op_data_writer import DataWriter
from pamola_core.utils.ops.op_field_utils import (
    apply_condition_operator,
    create_field_mask,
    create_multi_field_mask,
)
from pamola_core.utils.ops.op_result import OperationResult, OperationStatus
from pamola_core.utils.progress import HierarchicalProgressTracker
from pamola_core.utils.helpers import (
    build_base_cache,
    filter_used_kwargs,
    get_cache_result,
)
import pamola_core.utils.helpers as helpers


class AnonymizationOperation(FieldOperation):
    """
    Base class for all anonymization operations with Dask support.

    This class provides common functionality for all anonymization operations,
    including data source handling, result processing, metric generation,
    integration with profiling results for risk-based processing, and
    automatic switching to Dask for large dataset processing.
    """

    def __init__(
        self,
        field_name: str,
        # Conditional parameters
        condition_field: Optional[str] = None,
        condition_values: Optional[List[Any]] = None,
        condition_operator: str = "in",
        multi_conditions: Optional[List[Dict[str, Any]]] = None,
        condition_logic: str = "AND",
        # Risk-based anonymization
        ka_risk_field: Optional[str] = None,
        risk_threshold: float = 5.0,
        vulnerable_record_strategy: str = "suppress",
        **kwargs,
    ):
        """
        Initialize an anonymization operation.

        Parameters
        ----------
        field_name : str
            Field to anonymize.
        condition_field : str, optional
            Field to apply conditional anonymization.
        condition_values : list, optional
            Values to filter condition_field by.
        condition_operator : str, optional
            Operator for condition matching ("in", "eq", "gt", etc.).
        multi_conditions : list of dict, optional
            List of multiple filter conditions.
        condition_logic : str, optional
            Logic to combine multiple conditions ("AND"/"OR").
        ka_risk_field : str, optional
            Field containing k-anonymity risk scores.
        risk_threshold : float, optional
            Threshold under which records are considered vulnerable.
        vulnerable_record_strategy : str, optional
            Strategy for vulnerable records (e.g., "suppress", "mask").
        **kwargs : dict
            Additional parameters passed to :class:`FieldOperation`.
        """
        # Set default metadata if not provided
        kwargs.setdefault("name", f"{field_name}_anonymization")
        kwargs.setdefault(
            "description", f"Anonymization operation applied to '{field_name}'"
        )

        # Initialize base FieldOperation (creates scope, sets output field name)
        super().__init__(field_name=field_name, **kwargs)

        # Conditional anonymization config
        self.condition_field = condition_field
        self.condition_values = condition_values
        self.condition_operator = condition_operator
        self.multi_conditions = multi_conditions
        self.condition_logic = condition_logic

        # Risk-based anonymization config
        self.ka_risk_field = ka_risk_field
        self.risk_threshold = risk_threshold
        self.vulnerable_record_strategy = vulnerable_record_strategy

        # Set up common variables & placeholder for filtering
        self.filter_mask = pd.Series(True, index=pd.Index([]))

    def execute(
        self,
        data_source: DataSource,
        task_dir: Path,
        reporter: Any,
        progress_tracker: Optional[HierarchicalProgressTracker] = None,
        **kwargs,
    ) -> OperationResult:
        """
        Execute the anonymization operation with enhanced features including Dask support.

        Parameters
        -----------
        data_source : DataSource
            Source of data for the operation
        task_dir : Path
            Directory where task artifacts should be saved
        reporter : Any
            Reporter object for tracking progress and artifacts
        progress_tracker : Optional[HierarchicalProgressTracker]
            Progress tracker for the operation
        **kwargs : dict
            Additional parameters for the operation

        Returns
        --------
        OperationResult
            Results of the operation
        """
        try:
            # Start timing
            self.start_time = time.time()
            self.logger = kwargs.get("logger", self.logger)
            self.logger.info(
                f"Starting: {self.operation_name} operation at {self.start_time}"
            )

            # Initialize result object
            result = OperationResult(status=OperationStatus.PENDING)

            # Initialize dataframe
            df = None

            # Extract dataset name from kwargs (default to "main")
            dataset_name = kwargs.get("dataset_name", "main")

            # Generate single timestamp for all artifacts
            operation_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")

            # Prepare directories for artifacts
            dirs = self._prepare_directories(task_dir)

            # Initialize operation cache
            self.operation_cache = OperationCache(
                cache_dir=dirs["cache"],
            )

            # Initialize error handler
            self.error_handler = ErrorHandler(
                logger=self.logger,
                operation_name=self.operation_name,
            )

            # Create DataWriter for consistent file operations
            writer = DataWriter(
                task_dir=task_dir, logger=self.logger, progress_tracker=progress_tracker
            )

            # Save operation configuration
            self.save_config(task_dir)

            self.logger.info(
                f"Visualization settings: theme={self.visualization_theme}, backend={self.visualization_backend}, strict={self.visualization_strict}, timeout={self.visualization_timeout}s"
            )

            # Load settings operation
            settings_operation = load_settings_operation(
                data_source, dataset_name, **kwargs
            )

            # Set up progress tracking with proper steps
            # Main steps: 1. Data Loading & Validation, 2. Cache check, 3. Prepare output field, 4. Processing, 5. Metrics, 6. Visualization, 7. Save output
            TOTAL_MAIN_STEPS = 6 + (
                1 if self.use_cache and not self.force_recalculation else 0
            )
            main_progress = progress_tracker
            current_steps = 0
            if main_progress:
                self.logger.info(
                    f"Setting up progress tracker with {TOTAL_MAIN_STEPS} main steps"
                )
                try:
                    main_progress.total = TOTAL_MAIN_STEPS
                    main_progress.update(
                        current_steps,
                        {
                            "step": f"Starting {self.operation_name}",
                            "field": self.field_name,
                        },
                    )
                except Exception as e:
                    self.logger.warning(f"Could not update progress tracker: {e}")

            # Step 1: Data Loading & Validation
            if main_progress:
                current_steps += 1
                main_progress.update(
                    current_steps, {"step": "Data Loading", "field": self.field_name}
                )

            # Validate and get dataframe
            try:
                self.logger.info("Loading data for")
                df = self._validate_and_get_dataframe(
                    data_source, dataset_name, **settings_operation
                )

            except Exception as e:
                return self.error_handler.handle_error(
                    error=e,
                    error_code=ErrorCode.DATA_LOAD_FAILED,
                    context={"dataset": dataset_name, "operation": self.operation_name},
                    message_kwargs={"source": dataset_name, "reason": str(e)},
                )

            # Step 2: Check Cache (if enabled and not forced to recalculate)
            if self.use_cache and not self.force_recalculation:
                if main_progress:
                    current_steps += 1
                    main_progress.update(
                        current_steps,
                        {"step": "Checking cache", "field": self.field_name},
                    )

                self.logger.info("Checking operation cache...")
                cache_result = self._check_cache(df, reporter)

                if cache_result:
                    self.logger.info(
                        f"Using cached result for {self.field_name} generalization"
                    )

                    # Update progress
                    if main_progress:
                        main_progress.update(
                            current_steps,
                            {"step": "Complete (cached)", "field": self.field_name},
                        )

                    # Report cache hit to reporter
                    if reporter:
                        reporter.add_operation(
                            f"Anonymization of {self.field_name} (from cache)",
                            details={"cached": True},
                        )

                    return cache_result

            # Step 3: Prepare output field
            if main_progress:
                current_steps += 1
                main_progress.update(
                    current_steps,
                    {"step": "Preparing output field", "field": self.field_name},
                )

            try:
                self.output_field_name = self._prepare_output_field(df)
                self.logger.info(f"Prepared output_field: '{self.output_field_name}'")
                self._report_operation_details(reporter, self.output_field_name)
            except Exception as e:
                return self.error_handler.handle_error(
                    error=e,
                    error_code=ErrorCode.PROCESSING_FAILED,
                    context={"step": "prepare_output_field", "field": self.field_name},
                    message_kwargs={
                        "field_name": self.field_name,
                        "operation": self.operation_name,
                        "reason": str(e),
                    },
                )

            # Step 4: Processing
            if main_progress:
                current_steps += 1
                main_progress.update(
                    current_steps, {"step": "Processing", "field": self.field_name}
                )

            try:
                # Normalize integer dtype if required
                df[self.field_name] = DataHelper.normalize_int_dtype_vectorized(
                    df[self.field_name], safe_mode=False
                )

                # Copy original data for processing
                original_data = df[self.field_name].copy(deep=True)

                # Create child progress tracker for Chunk processing
                data_tracker = None
                if main_progress and hasattr(main_progress, "create_subtask"):
                    try:
                        data_tracker = main_progress.create_subtask(
                            total=3,
                            description="Processing dataframe",
                            unit="steps",
                        )
                    except Exception as e:
                        self.logger.debug(
                            f"Could not create child progress tracker: {e}"
                        )

                # Apply conditional filtering
                self.filter_mask, filtered_df = self._apply_conditional_filtering(df)

                # Process the filtered data only if not empty
                if not filtered_df.empty:
                    # Process the filtered data
                    processed_df = self._process_data_with_config(
                        df=filtered_df,
                        progress_tracker=data_tracker,
                    )
                else:
                    self.logger.warning(
                        "Filtered DataFrame is empty. Skipping _process_data_with_config."
                    )
                    processed_df = df.copy(deep=True)
                    processed_df[self.output_field_name] = original_data

                # Handle vulnerable records if k-anonymity is enabled
                if self.ka_risk_field and self.ka_risk_field in df.columns:
                    processed_df = self._handle_vulnerable_records(
                        processed_df, self.output_field_name
                    )

                # Get the anonymized data
                anonymized_data = processed_df[self.output_field_name]

                # Close child progress tracker
                if data_tracker:
                    try:
                        data_tracker.close()
                    except:
                        pass
            except Exception as e:
                return self.error_handler.handle_error(
                    error=e,
                    error_code=ErrorCode.PROCESSING_FAILED,
                    context={"step": "processing", "field": self.field_name},
                    message_kwargs={
                        "field_name": self.field_name,
                        "operation": self.operation_name,
                        "reason": str(e),
                    },
                )

            # Record end time after processing metrics
            self.end_time = time.time()
            if self.end_time and self.start_time:
                self.execution_time = self.end_time - self.start_time

            # Step 5: Metrics Calculation
            if main_progress:
                current_steps += 1
                main_progress.update(
                    current_steps,
                    {"step": "Metrics Calculation", "field": self.field_name},
                )

            # Initialize metrics in scope
            metrics = {}

            try:
                metrics = self._collect_all_metrics(
                    original_data, anonymized_data, self.filter_mask
                )

                # Generate metrics file name
                metrics_file_name = f"{self.field_name}_{self.operation_name}_metrics_{operation_timestamp}"

                self._save_metrics(
                    metrics=metrics,
                    writer=writer,
                    result=result,
                    reporter=reporter,
                    progress_tracker=progress_tracker,
                    operation_timestamp=operation_timestamp,
                    file_name=metrics_file_name,
                )
            except Exception as e:
                error_message = f"Error calculating metrics: {str(e)}"
                self.logger.warning(error_message)
                # Continue execution - metrics failure is not critical

            # Step 6: Visualizations
            if main_progress:
                current_steps += 1
                main_progress.update(
                    current_steps,
                    {"step": "Generating Visualizations", "field": self.field_name},
                )

            # Generate visualizations if required
            # Initialize visualization paths dictionary
            if self.generate_visualization and self.visualization_backend is not None:
                try:
                    kwargs_encryption = {
                        "use_encryption": self.use_encryption,
                        "encryption_key": self.encryption_key,
                    }
                    self._handle_visualizations(
                        original_data=original_data,
                        anonymized_data=anonymized_data,
                        task_dir=task_dir,
                        result=result,
                        reporter=reporter,
                        progress_tracker=main_progress,
                        vis_theme=self.visualization_theme,
                        vis_backend=self.visualization_backend,
                        vis_strict=self.visualization_strict,
                        vis_timeout=self.visualization_timeout,
                        operation_timestamp=operation_timestamp,
                        **kwargs_encryption,
                    )
                except Exception as e:
                    error_message = f"Error generating visualizations: {str(e)}"
                    self.logger.warning(error_message)
                    # Continue execution - visualization failure is not critical
            else:
                self.logger.info(
                    "Skipping visualizations as generate_visualization is False or backend is not set"
                )

            # Step 7: Save Output Data
            if main_progress:
                current_steps += 1
                main_progress.update(
                    current_steps,
                    {"step": "Save Output Data", "field": self.field_name},
                )

            # Save output data if required
            if self.save_output:
                try:
                    safe_kwargs = filter_used_kwargs(kwargs, self._save_output_data)
                    self._save_output_data(
                        result_df=processed_df,
                        writer=writer,
                        result=result,
                        reporter=reporter,
                        progress_tracker=main_progress,
                        timestamp=operation_timestamp,
                        **safe_kwargs,
                    )
                except Exception as e:
                    return self.error_handler.handle_error(
                        error=e,
                        error_code=ErrorCode.ARTIFACT_WRITE_FAILED,
                        context={"step": "save_output", "field": self.field_name},
                        message_kwargs={
                            "path": str(task_dir / "output"),
                            "reason": str(e),
                        },
                    )

            # Cache the result if caching is enabled
            if self.use_cache:
                try:
                    self._save_to_cache(
                        original_data=original_data,
                        anonymized_data=anonymized_data,
                        result=result,
                        task_dir=task_dir,
                    )
                except Exception as e:
                    # Failure to cache is non-critical
                    self.logger.warning(f"Failed to cache results: {str(e)}")

            # Clean up memory AFTER all write operations are complete
            self.logger.info("Cleaning up memory after all file operations")
            self._cleanup_memory(
                processed_df=processed_df,
                original_data=original_data,
                anonymized_data=anonymized_data,
            )

            # Report completion
            if reporter:
                reporter.add_operation(
                    f"Anonymization of {self.field_name} completed",
                    details={
                        "records_processed": self.process_count,
                        "execution_time": self.execution_time,
                        "records_filtered": len(filtered_df),
                        "vulnerable_records_handled": metrics.get(
                            "vulnerable_records", 0
                        ),
                    },
                )

            # Set success status
            result.status = OperationStatus.SUCCESS
            result.execution_time = self.execution_time
            self.logger.info(
                f"Processing completed {self.operation_name} operation in {self.execution_time:.2f} seconds"
            )
            return result

        except Exception as e:
            self.logger.exception(f"Error in {self.operation_name}: {str(e)}")
            return self.error_handler.handle_error(
                error=e,
                error_code=ErrorCode.PROCESSING_FAILED,
                context={"operation": self.operation_name, "field": self.field_name},
                message_kwargs={
                    "field_name": self.field_name,
                    "operation": self.operation_name,
                    "reason": str(e),
                },
            )

    def _validate_and_get_dataframe(
        self, data_source: DataSource, dataset_name: str, **kwargs: Any
    ) -> pd.DataFrame:
        """
        Validate data source and retrieve the main dataframe.

        Parameters
        -----------
        data_source : DataSource
            The data source to validate
        dataset_name : str
            The name of the dataset to retrieve
        **kwargs : Any
            Additional keyword arguments to pass to the data loading function

        Returns
        --------
        pd.DataFrame
            The validated dataframe

        Raises
        -------
        ValueError
            If no valid dataframe is found or the field is missing
        """
        # Get DataFrame from the data source
        df = load_data_operation(data_source, dataset_name, **kwargs)
        if df is None:
            error_message = "Failed to load input data!"
            self.logger.error(error_message)
            raise ValidationError(error_message)

        if self.field_name:
            if self.field_name not in df.columns:
                error_msg = f"Field '{self.field_name}' not found in DataFrame columns"
                self.logger.error(error_msg)
                raise ValidationError(error_msg)

        # Apply data types from data source
        try:
            df = data_source.apply_data_types(df, dataset_name)
        except (ValidationError, ValueError) as e:
            error_msg = (
                f"Failed to apply data types for dataset '{dataset_name}': {str(e)}"
            )
            self.logger.error(error_msg)
            raise ValidationError(error_msg) from e

        except TypeError as e:
            error_msg = f"Invalid dataframe type for dataset '{dataset_name}': {str(e)}"
            self.logger.error(error_msg)
            raise TypeValidationError(error_msg) from e

        # Optimize memory usage if enabled
        df = self._optimize_data(df)

        return df

    def _optimize_data(self, df: pd.DataFrame) -> Optional[pd.DataFrame]:
        """
        Load data and optimize memory usage.

        Parameters
        -----------
        df : pd.DataFrame
            DataFrame to optimize

        Returns
        --------
        Optional[pd.DataFrame]
            Loaded and optimized DataFrame or None if error
        """
        # Optimize memory if enabled
        if self.optimize_memory and len(df) > 10000:
            self.logger.info("Optimizing DataFrame memory usage")
            initial_memory = get_memory_usage(df)

            df, optimization_info = optimize_dataframe_dtypes(
                df,
                categorical_threshold=0.5,
                downcast_integers=True,
                downcast_floats=True,
            )

            self.logger.info(
                f"Memory optimization: {initial_memory['total_mb']:.2f}MB -> "
                f"{optimization_info['memory_after_mb']:.2f}MB "
                f"(saved {optimization_info['memory_saved_percent']:.1f}%)"
            )

            # Adjust chunk size if adaptive
            if self.adaptive_chunk_size:
                self._adjust_chunk_size(df)

        return df

    def _adjust_chunk_size(self, df: pd.DataFrame) -> None:
        """
        Adjust chunk size based on available memory.

        Parameters
        -----------
        df : pd.DataFrame
            DataFrame to process
        """
        memory_info = get_memory_usage(df)
        row_memory = memory_info["per_row_bytes"]

        # Target to use at most 100MB per chunk
        target_memory_mb = 100
        target_rows = int((target_memory_mb * 1024 * 1024) / row_memory)

        # Adjust chunk size within reasonable bounds
        self.chunk_size = max(1000, min(target_rows, 100000))

        # Ensure chunk size doesn't exceed data size
        self.chunk_size = min(self.chunk_size, len(df))

        if self.chunk_size != self.original_chunk_size:
            self.logger.info(
                f"Adjusted chunk size from {self.original_chunk_size} "
                f"to {self.chunk_size} based on memory usage"
            )

    def _prepare_output_field(self, df: pd.DataFrame) -> str:
        """
        Validate and generate the output field name.

        Parameters
        -----------
        df : pd.DataFrame
            The dataframe to check field names against

        Returns
        --------
        str
            The validated output field name
        """
        # Determine output field name based on mode
        if self.mode == "REPLACE":
            return self.field_name
        else:  # ENRICH mode
            if self.output_field_name:
                output_field = self.output_field_name
            else:
                output_field = f"{self.column_prefix}{self.field_name}"

            # Check if output field already exists in DataFrame
            if output_field in df.columns:
                self.logger.warning(
                    f"Output field '{output_field}' already exists and will be overwritten"
                )
        return output_field

    def _report_operation_details(self, reporter: Any, output_field: str) -> None:
        """
        Report details of the operation to the reporter.

        Parameters
        -----------
        reporter : Any
            The reporter to log details to
        output_field : str
            The name of the output field
        """
        if reporter:
            reporter.add_operation(
                f"Anonymizing field: {self.field_name}",
                details={
                    "field_name": self.field_name,
                    "output_field": output_field,
                    "mode": self.mode,
                    "null_strategy": self.null_strategy,
                    "operation_type": self.operation_name,
                },
            )

    def _apply_conditional_filtering(
        self,
        df: pd.DataFrame,
    ) -> Tuple[pd.Series, pd.DataFrame]:
        """
        Apply conditional filtering based on conditions and profiling results.

        Parameters
        -----------
        df : pd.DataFrame
            DataFrame to filter

        Returns
        --------
        Tuple[pd.Series, pd.DataFrame]
            (mask, filtered_dataframe)
        """
        # Start with all records
        mask = pd.Series(True, index=df.index)

        # Apply simple condition if specified
        if self.condition_field and self.condition_values:
            field_mask = create_field_mask(
                df,
                self.field_name,
                self.condition_field,
                self.condition_values,
                self.condition_operator,
            )
            mask = mask & field_mask
            self.logger.info(
                f"Applied condition on '{self.condition_field}': "
                f"{mask.sum()} records match"
            )

        # Apply multi-field conditions if specified
        if self.multi_conditions:
            multi_mask = create_multi_field_mask(
                df, self.multi_conditions, self.condition_logic
            )
            mask = mask & multi_mask
            self.logger.info(
                f"Applied {len(self.multi_conditions)} conditions: "
                f"{mask.sum()} records match"
            )

        # Apply k-anonymity filtering if specified
        if self.ka_risk_field and self.ka_risk_field in df.columns:
            risk_mask = df[self.ka_risk_field] < self.risk_threshold
            mask = mask & risk_mask
            self.logger.info(
                f"Applied k-anonymity filter (k < {self.risk_threshold}): "
                f"{mask.sum()} vulnerable records"
            )

        # Create filtered DataFrame
        filtered_df = df[mask].copy()

        return mask, filtered_df

    def _process_data_with_config(
        self,
        df: pd.DataFrame,
        progress_tracker: Optional[HierarchicalProgressTracker] = None,
    ) -> pd.DataFrame:
        """
        Handle processing of the dataframe, including chunk-wise or full processing.

        Parameters
        -----------
        df : pd.DataFrame
            The dataframe to process
        progress_tracker : Optional[HierarchicalProgressTracker]
            Optional progress tracker

        Returns
        --------
        pd.DataFrame
            The processed dataframe
        """
        # Check if dataframe is empty
        if len(df) == 0:
            self.logger.warning("Empty DataFrame provided, returning as is")
            return df

        # Handle null values based on strategy
        if self.null_strategy != "PRESERVE":
            anonymize_value = None

            if self.null_strategy == "ANONYMIZE":
                # Determine anonymize value
                if getattr(self, "unknown_value", None) is not None:
                    anonymize_value = self.unknown_value
                else:
                    # fallback: get default
                    anonymize_value = "SUPPRESSED"

            df[self.field_name] = process_nulls(
                df[self.field_name],
                strategy=self.null_strategy.upper(),
                anonymize_value=anonymize_value,
            )

        processed_df = None
        flag_processed = False
        # Backup and clear operation cache during processing
        cache_backup = self.operation_cache
        self.operation_cache = None
        self.logger.info("Process with config")

        # For larger dataframes, check if we should use parallel processing
        if not flag_processed and self.use_dask:
            try:
                self.logger.info("Parallel Enabled")
                self.logger.info("Parallel Engine: Dask")
                self.logger.info(f"Parallel Workers: {self.npartitions}")
                self.logger.info(
                    f"Using dask processing with chunk size {self.chunk_size}"
                )
                if progress_tracker:
                    progress_tracker.update(0, {"step": "Setting up dask processing"})

                self.logger.info("Process using Dask")

                runtime_kwargs = {
                    "npartitions": self.npartitions,
                    "dask_partition_size": self.dask_partition_size,
                }

                # Process with Dask - delegate to subclass method
                processed_df, flag_processed = process_dataframe_using_dask(
                    df=df,
                    process_function=self.process_batch,
                    progress_tracker=progress_tracker,
                    task_logger=self.logger,
                    **runtime_kwargs,
                )

                if flag_processed:
                    self.logger.info("Completed using Dask")

            except Exception as e:
                self.logger.warning(
                    f"Error in dask processing: {e}, falling back to chunk processing"
                )

        if not flag_processed and self.use_vectorization:
            try:
                self.logger.info("Parallel Enabled")
                self.logger.info("Parallel Engine: Joblib")
                self.logger.info(f"Parallel Workers: {self.parallel_processes}")
                self.logger.info(
                    f"Using vectorized processing with chunk size {self.chunk_size}"
                )
                if progress_tracker:
                    progress_tracker.update(
                        0, {"step": "Setting up vectorized processing"}
                    )

                self.logger.info("Process using Joblib")

                runtime_kwargs = {
                    "parallel_processes": self.parallel_processes,
                    "chunk_size": self.chunk_size,
                }

                processed_df, flag_processed = process_dataframe_using_joblib(
                    df=df,
                    process_function=self.process_batch,
                    progress_tracker=progress_tracker,
                    task_logger=self.logger,
                    **runtime_kwargs,
                )

                if flag_processed:
                    self.logger.info("Completed using Joblib")

            except Exception as e:
                self.logger.warning(
                    f"Error in vectorized processing: {e}, falling back to chunk processing"
                )

        if not flag_processed and self.chunk_size > 1:
            try:
                # Regular chunk processing
                self.logger.info(
                    f"Processing in chunks with chunk size {self.chunk_size}"
                )
                total_chunks = (len(df) + self.chunk_size - 1) // self.chunk_size
                self.logger.info(f"Total chunks to process: {total_chunks}")
                if progress_tracker:
                    progress_tracker.update(
                        0,
                        {
                            "step": "Processing in chunks",
                            "total_chunks": total_chunks,
                        },
                    )
                self.logger.info("Process using chunk")

                runtime_kwargs = {
                    "chunk_size": self.chunk_size,
                }

                processed_df, flag_processed = process_dataframe_using_chunk(
                    df=df,
                    process_function=self.process_batch,
                    progress_tracker=progress_tracker,
                    task_logger=self.logger,
                    **runtime_kwargs,
                )
                if flag_processed:
                    self.logger.info("Completed using chunk")
            except Exception as e:
                self.logger.warning(f"Error in chunk processing: {e}")

        if not flag_processed:
            self.logger.info("Fallback process as usual")
            processed_df = self.process_batch(df)
            flag_processed = True

        # Update process count
        self.process_count += len(df)
        # Restore operation cache
        self.operation_cache = cache_backup
        return processed_df

    def _handle_vulnerable_records(
        self,
        df: pd.DataFrame,
        output_field: str,
    ) -> pd.DataFrame:
        """
        Handle records identified as vulnerable by k-anonymity analysis.

        Parameters
        -----------
        df : pd.DataFrame
            DataFrame with processed data
        output_field : str
            Output field name

        Returns
        --------
        pd.DataFrame
            DataFrame with vulnerable records handled
        """
        # Identify vulnerable records
        vulnerability_mask = df[self.ka_risk_field] < self.risk_threshold
        vulnerable_count = vulnerability_mask.sum()

        if vulnerable_count > 0:
            self.logger.info(
                f"Handling {vulnerable_count} vulnerable records "
                f"with strategy: {self.vulnerable_record_strategy}"
            )

            # Apply strategy using commons utility
            df = handle_vulnerable_records(
                df, output_field, vulnerability_mask, self.vulnerable_record_strategy
            )

        return df

    def _collect_all_metrics(
        self,
        original_data: pd.Series,
        anonymized_data: pd.Series,
        mask: Optional[pd.Series] = None,
    ) -> Dict[str, Any]:
        """
        Collect comprehensive metrics.

        Parameters
        -----------
        original_data : pd.Series
            Original data
        anonymized_data : pd.Series
            Anonymized data
        mask : pd.Series, optional
            Processing mask

        Returns
        --------
        Dict[str, Any]
            Collected metrics
        """
        # Basic anonymization metrics
        metrics: Dict[str, Any] = {}

        # Add effectiveness metrics
        effectiveness = calculate_anonymization_effectiveness(
            original_data, anonymized_data
        )
        metrics["effectiveness"] = effectiveness

        # Add performance metrics
        if self.start_time and self.end_time:
            duration = self.end_time - self.start_time
            metrics.update(
                {
                    "duration_seconds": round(duration, 2),
                    "records_processed": self.process_count,
                    "records_per_second": (
                        round(self.process_count / duration, 2) if duration > 0 else 0
                    ),
                    "chunk_count": (
                        int((self.process_count - 1) / self.chunk_size + 1)
                        if self.process_count > 0
                        else 0
                    ),
                }
            )

        # Add operation-specific metrics
        metrics.update(self._collect_specific_metrics(original_data, anonymized_data))
        # Calculate effective mask
        effective_mask = mask.sum() if mask is not None else len(anonymized_data)
        # Add filtering metrics
        metrics.update(
            {
                "total_records": len(original_data),
                "processed_records": effective_mask,
                "filtered_records": len(original_data) - effective_mask,
                "processing_rate": (
                    (effective_mask / len(original_data)) * 100
                    if len(original_data) > 0
                    else 0
                ),
            }
        )

        # Add memory metrics if optimization was performed
        if self.optimize_memory:
            metrics["chunk_size_used"] = self.chunk_size
            metrics["adaptive_chunk_size"] = self.adaptive_chunk_size

        return metrics

    def _handle_visualizations(
        self,
        original_data: pd.Series,
        anonymized_data: pd.Series,
        task_dir: Path,
        result: OperationResult,
        reporter: Any,
        progress_tracker: Optional[HierarchicalProgressTracker],
        vis_theme: Optional[str] = None,
        vis_backend: Optional[str] = None,
        vis_strict: bool = False,
        vis_timeout: int = 120,
        operation_timestamp: Optional[str] = None,
        **kwargs,
    ) -> Dict[str, Any]:
        """
        Generate and save visualizations with thread-safe context support.

        Parameters
        -----------
        original_data : pd.Series
            The original data before anonymization
        anonymized_data : pd.Series
            The anonymized data after processing
        task_dir : Path
            The task directory
        result : OperationResult
            The operation result to add artifacts to
        reporter : Any
            The reporter to log artifacts to
        progress_tracker : Optional[HierarchicalProgressTracker]
            Optional progress tracker
        vis_theme : str, optional
            Theme to use for visualizations
        vis_backend : str, optional
            Backend to use: "plotly" or "matplotlib"
        vis_strict : bool, optional
            If True, raise exceptions for configuration errors
        vis_timeout : int, optional
            Timeout for visualization generation (default: 120 seconds)
        operation_timestamp : str, optional
            Timestamp for the operation (default: current time)
        **kwargs : dict
            Additional parameters for the operation
        """
        self.logger.info(
            f"Generating visualizations with backend: {vis_backend}, timeout: {vis_timeout}s"
        )
        try:
            import threading
            import contextvars

            visualization_paths = {}
            visualization_error = None

            def generate_viz_with_diagnostics():
                nonlocal visualization_paths, visualization_error
                thread_id = threading.current_thread().ident
                thread_name = threading.current_thread().name

                self.logger.info(
                    f"[DIAG] Visualization thread started - Thread ID: {thread_id}, Name: {thread_name}"
                )
                self.logger.info(
                    f"[DIAG] Field: {self.field_name}, Strategy: {self.null_strategy}, Backend: {vis_backend}, Theme: {vis_theme}, Strict: {vis_strict}"
                )

                start_time = time.time()

                try:
                    # Log context variables
                    self.logger.info("[DIAG] Checking context variables...")
                    try:
                        current_context = contextvars.copy_context()
                        self.logger.info(
                            f"[DIAG] Context vars count: {len(list(current_context))}"
                        )
                    except Exception as ctx_e:
                        self.logger.warning(
                            f"[DIAG] Could not inspect context: {ctx_e}"
                        )

                    # Generate visualizations with visualization context parameters
                    self.logger.info("[DIAG] Calling _generate_visualizations...")
                    # Create child progress tracker for visualization if available
                    total_steps = 3  # prepare data, create viz, save
                    viz_progress = None
                    if progress_tracker and hasattr(progress_tracker, "create_subtask"):
                        try:
                            viz_progress = progress_tracker.create_subtask(
                                total=total_steps,
                                description="Generating visualizations",
                                unit="steps",
                            )
                        except Exception as e:
                            self.logger.debug(
                                f"Could not create child progress tracker: {e}"
                            )

                    # Generate visualizations with context parameters
                    visualization_paths = self._generate_visualizations(
                        original_data=original_data,
                        anonymized_data=anonymized_data,
                        task_dir=task_dir,
                        vis_theme=vis_theme,
                        vis_backend=vis_backend or "plotly",
                        vis_strict=vis_strict,
                        progress_tracker=viz_progress,
                        timestamp=operation_timestamp,  # Pass the same timestamp
                        **kwargs,
                    )

                    # Close visualization progress tracker
                    if viz_progress:
                        try:
                            viz_progress.close()
                        except:
                            pass

                    elapsed = time.time() - start_time
                    self.logger.info(
                        f"[DIAG] Visualization completed in {elapsed:.2f}s, generated {len(visualization_paths)} files"
                    )

                except Exception as e:
                    elapsed = time.time() - start_time
                    visualization_error = e
                    self.logger.error(
                        f"[DIAG] Visualization failed after {elapsed:.2f}s: {type(e).__name__}: {e}"
                    )
                    self.logger.error("[DIAG] Stack trace:", exc_info=True)

            # Copy context for the thread
            self.logger.info("[DIAG] Preparing to launch visualization thread...")
            ctx = contextvars.copy_context()

            # Create thread with context
            viz_thread = threading.Thread(
                target=ctx.run,
                args=(generate_viz_with_diagnostics,),
                name=f"VizThread-{self.field_name}",
                daemon=False,  # Changed from True to ensure proper cleanup
            )

            self.logger.info(
                f"[DIAG] Starting visualization thread with timeout={vis_timeout}s"
            )
            thread_start_time = time.time()
            viz_thread.start()

            # Log periodic status while waiting
            check_interval = 5  # seconds
            elapsed = 0
            while viz_thread.is_alive() and elapsed < vis_timeout:
                viz_thread.join(timeout=check_interval)
                elapsed = time.time() - thread_start_time
                if viz_thread.is_alive():
                    self.logger.info(
                        f"[DIAG] Visualization thread still running after {elapsed:.1f}s..."
                    )

            if viz_thread.is_alive():
                self.logger.error(
                    f"[DIAG] Visualization thread still alive after {vis_timeout}s timeout"
                )
                self.logger.error(
                    f"[DIAG] Thread state: alive={viz_thread.is_alive()}, daemon={viz_thread.daemon}"
                )
                visualization_paths = {}
            elif visualization_error:
                self.logger.error(
                    f"[DIAG] Visualization failed with error: {visualization_error}"
                )
                visualization_paths = {}
            else:
                total_time = time.time() - thread_start_time
                self.logger.info(
                    f"[DIAG] Visualization thread completed successfully in {total_time:.2f}s"
                )
                self.logger.info(
                    f"[DIAG] Generated visualizations: {list(visualization_paths.keys())}"
                )

        except Exception as e:
            self.logger.error(
                f"[DIAG] Error in visualization thread setup: {type(e).__name__}: {e}"
            )
            self.logger.error("[DIAG] Stack trace:", exc_info=True)
            visualization_paths = {}

        # Register visualization artifacts
        for viz_type, path in visualization_paths.items():
            # Add to result
            result.add_artifact(
                artifact_type="png",
                path=path,
                description=f"{self.field_name} {viz_type} visualization",
                category=Constants.Artifact_Category_Visualization,
            )

            # Report to reporter
            if reporter:
                reporter.add_operation(
                    f"{self.field_name} {viz_type} visualization",
                    details={"artifact_type": "png", "path": str(path)},
                )

        return visualization_paths

    def _generate_visualizations(
        self,
        original_data: pd.Series,
        anonymized_data: pd.Series,
        task_dir: Path,
        vis_theme: Optional[str] = None,
        vis_backend: Optional[str] = None,
        vis_strict: bool = False,
        progress_tracker: Optional[HierarchicalProgressTracker] = None,
        timestamp: Optional[str] = None,
        **kwargs,
    ) -> Dict[str, Path]:
        """
        Generate visualizations using the core visualization utilities with thread-safe context support.

        This is a base implementation that provides a basic distribution comparison.
        Subclasses should override to provide operation-specific visualizations.

        Parameters
        -----------
        original_data : pd.Series
            Original data before anonymization
        anonymized_data : pd.Series
            Anonymized data after processing
        task_dir : Path
            Task directory for saving visualizations
        result : OperationResult
            Operation result to add artifacts to
        reporter : Any
            Reporter object for tracking artifacts
        vis_theme : str, optional
            Theme to use for visualizations
        vis_backend : str, optional
            Backend to use: "plotly" or "matplotlib"
        vis_strict : bool, optional
            If True, raise exceptions for configuration errors
        **kwargs : dict
            Additional parameters for the operation

        Returns
        --------
        Dict[str, Path]
            Dictionary with visualization types and paths
        """
        visualization_paths = {}
        viz_dir = task_dir / "visualizations"
        viz_dir.mkdir(parents=True, exist_ok=True)

        # Use provided timestamp or generate new one
        if timestamp is None:
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")

        # Check if visualization should be skipped
        if vis_backend is None:
            self.logger.info(
                f"Skipping visualization for {self.field_name} (backend=None)"
            )
            return visualization_paths

        self.logger.info(
            f"[VIZ] Starting visualization generation for {self.field_name} using {self.null_strategy} strategy"
        )
        self.logger.debug(
            f"[VIZ] Backend: {vis_backend}, Theme: {vis_theme}, Strict: {vis_strict}"
        )

        try:
            # Step 1: Prepare data
            if progress_tracker:
                progress_tracker.update(1, {"step": "Preparing visualization data"})

            # Sample large datasets for visualization
            if len(original_data) > 10000:
                self.logger.info(
                    f"[VIZ] Sampling large dataset: {len(original_data)} -> 10000 samples"
                )
                original_for_viz = sample_large_dataset(
                    original_data, max_samples=10000
                )
                anonymized_for_viz = sample_large_dataset(
                    anonymized_data, max_samples=10000
                )
            else:
                original_for_viz = original_data
                anonymized_for_viz = anonymized_data

            self.logger.debug(
                f"[VIZ] Data prepared for visualization: {len(original_for_viz)} samples"
            )
            self.logger.debug(
                f"[VIZ] Original data type: {original_for_viz.dtype}, Anonymized data type: {anonymized_for_viz.dtype}"
            )

            # Step 2: Create visualization
            if progress_tracker:
                progress_tracker.update(2, {"step": "Creating visualization"})

            # Create comparison visualization
            comparison_path = create_comparison_visualization(
                original_data=original_for_viz,
                anonymized_data=anonymized_for_viz,
                task_dir=viz_dir,
                field_name=self.field_name,
                operation_name=self.operation_name,
                timestamp=timestamp,
                theme=vis_theme,
                backend=vis_backend,
                strict=vis_strict,
                **kwargs,
            )

            if comparison_path:
                visualization_paths["comparison"] = comparison_path

            # Create distribution visualization for anonymized data
            metric_data = anonymized_for_viz.value_counts().to_dict()
            dist_path = create_metric_visualization(
                metric_name="distribution",
                metric_data=metric_data,
                task_dir=viz_dir,
                field_name=self.field_name,
                operation_name=self.operation_name,
                timestamp=timestamp,
                theme=vis_theme,
                backend=vis_backend,
                strict=vis_strict,
                **kwargs,
            )

            if dist_path:
                visualization_paths["distribution"] = dist_path

            # Step 3: Finalize visualizations
            if progress_tracker:
                progress_tracker.update(3, {"step": "Finalizing visualizations"})

        except Exception as e:
            self.logger.warning(f"Error generating visualizations: {e}")

        return visualization_paths

    def _save_output_data(
        self,
        result_df: pd.DataFrame,
        writer: DataWriter,
        result: OperationResult,
        reporter: Any,
        progress_tracker: Optional[HierarchicalProgressTracker],
        timestamp: Optional[str] = None,
        file_name: str = None,
        **kwargs,
    ) -> str:
        """
        Save the processed output data.

        Parameters
        -----------
        result_df : pd.DataFrame
            The processed dataframe to save
        writer : DataWriter
            The writer to use for saving data
        result : OperationResult
            The operation result to add artifacts to
        reporter : Any
            The reporter to log artifacts to
        progress_tracker : Optional[HierarchicalProgressTracker]
            Optional progress tracker
        timestamp : Optional[str]
            Optional timestamp for the operation
        file_name : str
            File name for the name of output file
        **kwargs : dict
            Additional parameters for the operation
        """
        if progress_tracker:
            progress_tracker.update(0, {"step": "Saving output data"})

        # Generate standardized output filename with timestamp
        file_name = file_name or (
            f"{self.field_name}_{self.operation_name}_output_{timestamp}"
        )

        # Use the DataWriter to save the DataFrame
        safe_kwargs = filter_used_kwargs(kwargs, writer.write_dataframe)
        safe_kwargs["encryption_mode"] = get_encryption_mode(
            result_df, self.use_encryption
        )
        output_result = writer.write_dataframe(
            df=result_df,
            name=file_name,
            format=self.output_format,
            subdir="output",
            timestamp_in_name=False,
            encryption_key=self.encryption_key if self.use_encryption else None,
            overwrite=True,
            **safe_kwargs,
        )

        # Register output artifact with the result
        result.add_artifact(
            artifact_type=self.output_format,
            path=output_result.path,
            description=f"{self.field_name} {self.operation_name} anonymized data",
            category=Constants.Artifact_Category_Output,
        )

        # Report to reporter
        if reporter:
            reporter.add_operation(
                f"{self.field_name} anonymized data",
                details={
                    "artifact_type": self.output_format,
                    "path": str(output_result.path),
                },
            )

        # Save output data types
        self._save_dtypes_output(
            df=result_df,
            writer=writer,
            result=result,
            reporter=reporter,
            file_name=file_name,
        )

        return str(output_result.path)

    def _cleanup_memory(
        self,
        processed_df: Optional[pd.DataFrame] = None,
        original_data: Optional[pd.Series] = None,
        anonymized_data: Optional[pd.Series] = None,
    ) -> None:
        """
        Clean up memory after operation completes.

        For large datasets, explicitly free memory by deleting
        references and optionally calling garbage collection.

        Parameters
        -----------
        processed_df : pd.DataFrame, optional
            Processed DataFrame to clear from memory
        original_data : pd.Series, optional
            Original data to clear from memory
        anonymized_data : pd.Series, optional
            Anonymized data to clear from memory
        """
        # Delete references
        if processed_df is not None:
            del processed_df
        if original_data is not None:
            del original_data
        if anonymized_data is not None:
            del anonymized_data

        # cleanup memory from instance
        helpers.cleanup_memory(instance=self)

    def _should_process_record(self, record: pd.Series) -> bool:
        """
        Determine if a record should be processed based on conditions.

        Parameters
        -----------
        record : pd.Series
            Record to check

        Returns
        --------
        bool
            True if record should be processed
        """
        # No conditions means process all records
        if not self.condition_field:
            return True

        # Check simple condition
        if self.condition_field in record.index:
            value = record[self.condition_field]
            mask = apply_condition_operator(
                pd.Series([value]), self.condition_values, self.condition_operator
            )
            return mask.iloc[0]

        return True

    def process_batch(self, batch: pd.DataFrame) -> pd.DataFrame:
        """
        Process a batch of data. Must be implemented by subclasses.

        Parameters
        -----------
        batch : pd.DataFrame
            DataFrame batch to process

        Returns
        --------
        pd.DataFrame
            Processed DataFrame batch
        """
        raise FeatureNotImplementedError(
            "Subclasses must implement process_batch method"
        )

    def process_batch_dask(self, ddf: dd.DataFrame) -> dd.DataFrame:
        """
        Process Dask DataFrame. Should be overridden by subclasses for optimal performance.

        Parameters
        -----------
        ddf : dd.DataFrame
            Dask DataFrame to process

        Returns
        --------
        dd.DataFrame
            Processed Dask DataFrame
        """

        # Default implementation: process each partition with process_batch
        def process_partition(partition):
            return self.process_batch(partition)

        return ddf.map_partitions(process_partition)

    def process_value(self, value):
        """
        Process a single value. Must be implemented by subclasses.

        Parameters
        -----------
        value : Any
            Value to process

        Returns
        --------
        Any
            Processed value
        """
        raise FeatureNotImplementedError(
            "Subclasses must implement process_value method"
        )

    def _collect_specific_metrics(
        self, original_data: pd.Series, anonymized_data: pd.Series
    ) -> Dict[str, Any]:
        """
        Collect operation-specific metrics. Should be overridden by subclasses.

        Parameters
        -----------
        original_data : pd.Series
            Original data
        anonymized_data : pd.Series
            Anonymized data

        Returns
        --------
        Dict[str, Any]
            Operation-specific metrics
        """
        return {}

    def _get_base_parameters(self) -> Dict[str, str]:
        """Get the basic parameters for the cache key generation."""
        # Get basic operation parameters

        parameters = super()._get_base_parameters()

        # Add operation-specific parameters
        parameters.update(
            {
                "field_name": self.field_name,
                # Conditional parameters
                "condition_field": self.condition_field,
                "condition_values": self.condition_values,
                "condition_operator": self.condition_operator,
                "multi_conditions": self.multi_conditions,
                "condition_logic": self.condition_logic,
                # Risk-based anonymization
                "ka_risk_field": self.ka_risk_field,
                "risk_threshold": self.risk_threshold,
                "vulnerable_record_strategy": self.vulnerable_record_strategy,
            }
        )
        return parameters

    def _check_cache(
        self, df: pd.DataFrame, reporter: Any
    ) -> Optional[OperationResult]:
        """
        Check if a cached result exists for this operation.

        Parameters
        ----------
        df : pd.DataFrame
            DataFrame for the operation
        reporter : Any
            Reporter object for tracking progress and artifacts

        Returns
        -------
        Optional[OperationResult]
            Cached result if found, None otherwise
        """
        if not self.use_cache:
            return None

        try:
            if self.field_name not in df.columns:
                self.logger.warning(
                    f"Field '{self.field_name}' not found in DataFrame."
                )
                return None

            cache_key = self._generate_cache_key(df[self.field_name])
            self.logger.debug(f"Checking cache for key: {cache_key}")

            cached_result = self.operation_cache.get_cache(
                cache_key=cache_key, operation_type=self.operation_name
            )

            self.logger.info(
                f"Using cached result for {self.field_name} generalization"
            )

            if not cached_result:
                self.logger.info("No cached result found, proceeding with operation")
                return None

            result = get_cache_result(cached_result)

            if reporter:
                reporter.add_operation(
                    f"Generalization of {self.field_name} (cached)",
                    details={"null_strategy": self.null_strategy, "cached": True},
                )
            return result

        except Exception as e:
            self.logger.warning(f"Error checking cache: {str(e)}")
            return None

    def _save_to_cache(
        self,
        original_data: Union[pd.Series, pd.DataFrame],
        anonymized_data: Union[pd.Series, pd.DataFrame],
        result: OperationResult,
        task_dir: Path,
    ) -> bool:
        """
        Save operation results to cache.

        Parameters
        -----------
        original_data : pd.Series or pd.DataFrame
            Original input data
        anonymized_data : pd.Series or pd.DataFrame
            Anonymized output data
        result: OperationResult
            Result object OperationResult
        task_dir : Path
            Task directory

        Returns
        --------
        bool
            True if successfully saved to cache, False otherwise
        """
        if not self.use_cache:
            return False

        try:
            # Generate cache key
            cache_key = self._generate_cache_key(original_data)

            # Prepare cache data
            cache_data = build_base_cache(
                parameters=self._get_base_parameters(), result=result
            )

            # Add data info
            cache_data.update(
                {
                    "data_info": {
                        "original_length": len(original_data),
                        "anonymized_length": len(anonymized_data),
                        "original_null_count": int(
                            original_data.isna().sum().sum()
                            if isinstance(original_data, pd.DataFrame)
                            else original_data.isna().sum()
                        ),
                        "anonymized_null_count": int(
                            anonymized_data.isna().sum().sum()
                            if isinstance(anonymized_data, pd.DataFrame)
                            else anonymized_data.isna().sum()
                        ),
                    }
                }
            )

            # Save to cache
            self.logger.debug(f"Saving to cache with key: {cache_key}")

            success = self.operation_cache.save_cache(
                data=cache_data,
                cache_key=cache_key,
                operation_type=self.operation_name,
                metadata={"task_dir": str(task_dir)},
            )

            if success:
                self.logger.info(
                    f"Successfully saved {self.field_name} anonymization results to cache"
                )
            else:
                self.logger.warning(
                    f"Failed to save {self.field_name} anonymization results to cache"
                )

            return success

        except Exception as e:
            self.logger.warning(f"Error saving to cache: {str(e)}")
            return False

    def _save_dtypes_output(
        self,
        df: pd.DataFrame,
        writer: DataWriter,
        result: OperationResult,
        reporter: Any,
        file_name: str = None,
    ) -> bool:
        """
        Saves data types dataframe format to a JSON file.

        Returns
        -------
        Path or None
            Path to saved file if success, otherwise None
        """
        try:

            # Get the dtypes of the columns as a Series
            dtypes_series = df.dtypes

            # Convert the dtypes Series to a dictionary
            dtypes_dict = dtypes_series.astype(str).to_dict()

            # Generate standardized output filename with timestamp
            dtypes_filename = f"data_types_{file_name}"

            dtypes_result = writer.write_metrics(
                metrics=dtypes_dict,
                name=dtypes_filename,
                timestamp_in_name=False,
                encryption_key=self.encryption_key,
            )

            result.add_metric(dtypes_filename, dtypes_dict)

            result.add_artifact(
                artifact_type="json",
                path=dtypes_result.path,
                description=f"Data types of output {self.field_name} {self.operation_name}",
                category=Constants.Artifact_Category_Metrics,
            )

            if reporter:
                reporter.add_artifact(
                    artifact_type="json",
                    path=str(dtypes_result.path),
                    description=f"Data types of output {self.field_name} {self.operation_name}",
                )

            self.logger.info(f"Dtypes output saved to: {Path(dtypes_result.path).name}")
            return True

        except Exception as e:
            self.logger.warning(f"Failed to save dtypes format: {str(e)}")
            return False

    def _save_metrics(
        self,
        metrics: Dict[str, Any],
        writer: DataWriter,
        result: OperationResult,
        reporter: Any,
        progress_tracker: Optional[HierarchicalProgressTracker],
        file_name: str = None,
        operation_timestamp: Optional[str] = None,
    ) -> bool:
        """
        Save metrics.

        Parameters
        -----------
        metrics : dict
            The metrics of operation
        writer : DataWriter
            The writer to use for saving data
        result : OperationResult
            The operation result to add artifacts to
        reporter : Any
            The reporter to log artifacts to
        progress_tracker : Optional[HierarchicalProgressTracker]
            Optional progress tracker
        operation_timestamp : str, optional
            Timestamp of the operation, if any (for filename purposes)
        file_name : Str
            The file name

        Returns
        -------
        True or False

        """
        try:
            if progress_tracker:
                progress_tracker.update(0, {"step": "Saving metrics"})

            # Use the DataWriter to save
            metrics_filename = file_name or (
                f"{self.operation_name.lower()}_metrics_{operation_timestamp}"
            )

            metrics_result = writer.write_metrics(
                metrics=metrics,
                name=metrics_filename,
                timestamp_in_name=False,  # Already included in the filename
                encryption_key=self.encryption_key if self.use_encryption else None,
            )

            # Add metrics to result
            for key, value in metrics.items():
                if isinstance(value, (int, float, str, bool)):
                    result.add_metric(key, value)

            # Register metrics artifact
            result.add_artifact(
                artifact_type="json",
                path=metrics_result.path,
                description=f"{self.operation_name.lower()} {self.field_name} anonymization metrics",
                category=Constants.Artifact_Category_Metrics,
            )

            # Report artifact
            if reporter:
                reporter.add_artifact(
                    artifact_type="json",
                    path=str(metrics_result.path),
                    description=f"{self.operation_name.lower()} {self.field_name} anonymization metrics",
                )

            return True
        except Exception as e:
            self.logger.warning(f"Failed to save metrics: {str(e)}")
            return False
