# pylint: disable=too-many-lines
import os
from importlib import metadata
from datetime import timezone
from datetime import datetime
from dataclasses import dataclass
from uuid import uuid1
import ast
from typing import Dict, Optional, List, Tuple, Any
from pyspark.sql import DataFrame, SparkSession
from spark_expectations import _log
from spark_expectations.config.user_config import Constants as user_config
from spark_expectations.core.exceptions import SparkExpectationsMiscException


# TODO: Add exceptions to follow standardized naming conventions for _run_id, product_id and _dq_stats_table_name in
#       the future
@dataclass
class SparkExpectationsContext:
    """
    This class provides the context for SparkExpectations
    """

    product_id: str
    spark: SparkSession

    def __post_init__(self) -> None:
        self._run_id: str = f"{self.product_id}_{uuid1()}"
        self._enable_obs_dq_report_result: bool = False
        self._df_dq_obs_report_dataframe: DataFrame = None
        self._detailed_default_template: str
        self._dq_obs_rpt_gen_status_flag: bool = False
        self._dataframe: DataFrame = None
        self._report_table_name: DataFrame = None
        self._custom_dataframe: DataFrame = None
        self._se_dq_obs_alert_flag: bool = False
        self._run_date: str = self.set_run_date()
        self._basic_default_template: Optional[str] = None
        self._custom_default_template: Optional[str] = None
        self._dq_stats_table_name: Optional[str] = None
        self._dq_detailed_stats_table_name: Optional[str] = None
        self._final_table_name: Optional[str] = None
        self._error_table_name: Optional[str] = None
        self._error_table_name_user_specified: bool = False
        self._row_dq_rule_type_name: str = "row_dq"
        self._agg_dq_rule_type_name: str = "agg_dq"
        self._query_dq_rule_type_name: str = "query_dq"
        self._row_dq_status: str = "Skipped"
        self._source_agg_dq_status: str = "Skipped"
        self._final_agg_dq_status: str = "Skipped"
        self._source_query_dq_status: str = "Skipped"
        self._final_query_dq_status: str = "Skipped"
        self._dq_run_status: str = "Failed"
        self._dq_expectations: Optional[Dict[str, str]] = None
        self._se_enable_error_table: bool = True
        self._dq_rules_params: Dict[str, str] = {}
        # above configuration variable value has to be set to python
        self._dq_project_env_name = "spark_expectations"
        self._dq_config_file_name = "dq_spark_expectations_config.ini"
        self._dq_config_abs_path: Optional[str] = None

        self._min_priority_email: str
        self._min_priority_pagerduty: str
        self._min_priority_slack: str
        self._min_priority_teams: str
        self._min_priority_zoom: str

        self._enable_mail: bool = False
        self._enable_smtp_server_auth: bool = False
        self._enable_custom_email_body: bool = False
        self._enable_templated_basic_email_body: bool = False
        self._enable_templated_custom_email: bool = False
        self._to_mail: Optional[str] = None
        self._mail_subject: Optional[str] = None
        self._mail_from: Optional[str] = None
        self._mail_smtp_server: str
        self._mail_smtp_port: int
        self._mail_smtp_password: Optional[str] = None
        self._mail_smtp_user_name: Optional[str] = None
        self._smtp_creds_dict: Dict[str, str] = {}
        self._email_custom_body: Optional[str] = None

        self._enable_slack: bool = False
        self._slack_webhook_url: Optional[str] = None

        self._enable_teams: bool = False
        self._teams_webhook_url: Optional[str] = None

        self._enable_zoom: bool = False
        self._zoom_webhook_url: Optional[str] = None
        self._zoom_token: Optional[str] = None

        self._enable_pagerduty: bool = False
        self._pagerduty_webhook_url: Optional[str] = None
        self._pagerduty_integration_key: Optional[str] = None
        self._pagerduty_creds_dict: Dict[str, str] = {}

        self._table_name: Optional[str] = None
        self._input_count: int = 0
        self._error_count: int = 0
        self._output_count: int = 0

        self._env: Optional[str] = None

        self._source_agg_dq_result: Optional[List[Dict[str, str]]] = None
        self._final_agg_dq_result: Optional[List[Dict[str, str]]] = None
        self._source_query_dq_result: Optional[List[Dict[str, str]]] = None
        self._final_query_dq_result: Optional[List[Dict[str, str]]] = None
        self._job_metadata: Optional[str] = None
        self._se_job_metadata: Optional[Dict[str, Any]] = None

        self._source_agg_dq_detailed_stats: Optional[List[Tuple]] = None
        self._source_query_dq_detailed_stats: Optional[List[Tuple]] = None

        self._target_agg_dq_detailed_stats: Optional[List[Tuple]] = None
        self._target_query_dq_detailed_stats: Optional[List[Tuple]] = None

        self._notification_on_start: bool = False
        self._notification_on_completion: bool = False
        self._notification_on_fail: bool = False
        self._notification_on_error_drop_exceeds_threshold_breach: bool = False
        self._notifications_on_rules_action_if_failed_set_ignore: bool = False
        self._error_drop_threshold: int = 100

        self._cerberus_url: str = "your_cerberus_url"
        self._cerberus_cred_path: str = "your_cerberus_sdb_path"
        self._cerberus_token: Optional[str] = os.environ.get("DQ_SPARK_EXPECTATIONS_CERBERUS_TOKEN")

        self._se_streaming_stats_dict: Dict[str, str]
        self._enable_se_streaming: bool = False
        self._se_streaming_secret_env: str = ""
        self._se_streaming_stats_kafka_custom_config_enable: bool = False
        self._se_streaming_stats_topic_name: str = ""
        self._se_streaming_stats_kafka_bootstrap_server: str = ""

        self._debugger_mode: bool = False
        self._supported_df_query_dq: DataFrame = self.set_supported_df_query_dq()

        self._source_agg_dq_start_time: Optional[datetime] = None
        self._final_agg_dq_start_time: Optional[datetime] = None
        self._source_query_dq_start_time: Optional[datetime] = None
        self._final_query_dq_start_time: Optional[datetime] = None
        self._row_dq_start_time: Optional[datetime] = None
        self._dq_start_time: Optional[datetime] = None

        self._source_agg_dq_end_time: Optional[datetime] = None
        self._final_agg_dq_end_time: Optional[datetime] = None
        self._source_query_dq_end_time: Optional[datetime] = None
        self._final_query_dq_end_time: Optional[datetime] = None
        self._row_dq_end_time: Optional[datetime] = None
        self._dq_end_time: Optional[datetime] = None

        self._run_id_name = "meta_dq_run_id"
        self._run_date_name = "meta_dq_run_date"
        self._run_date_time_name = "meta_dq_run_datetime"

        self._num_row_dq_rules: int = 0
        self._num_agg_dq_rules: Dict[str, int] = {
            "num_agg_dq_rules": 0,
            "num_source_agg_dq_rules": 0,
            "num_final_agg_dq_rules": 0,
        }
        self._num_query_dq_rules: Dict[str, int] = {
            "num_query_dq_rules": 0,
            "num_source_query_dq_rules": 0,
            "num_final_query_dq_rules": 0,
        }
        self._num_dq_rules: int = 0
        self._summarized_row_dq_res: Optional[List[Dict[str, str]]] = None
        self._rules_error_per: Optional[List[dict]] = None

        # The below config is used to set the type of the writer for the target and error table and stats table
        self._target_and_error_table_writer_type: str = "batch"
        self._stats_table_writer_type: str = "batch"
        self._target_and_error_table_writer_config: dict = {}
        self._stats_table_writer_config: dict = {}

        # The below config is user config and will be enabled if detailed result is required for agg and query dq
        self._enable_agg_dq_detailed_result: bool = False
        self._enable_query_dq_detailed_result: bool = False

        self._rules_execution_settings_config: Dict[str, str]
        self._querydq_secondary_queries: dict

        self._source_query_dq_output: Optional[List[dict]] = None

        self._target_query_dq_output: Optional[List[dict]] = None

        self._query_dq_output_custom_table_name: str

        self._stats_dict: List[dict] = []

        # Kafka write status tracking
        self._kafka_write_status: str = "Disabled"
        self._kafka_write_error_message: str = ""

    @property
    def get_dbr_version(self) -> Optional[str]:
        """
        Returns the raw DATABRICKS_RUNTIME_VERSION environment variable value,
        or None when not running on Databricks.

        On standard (non-serverless) compute this is a numeric version string
        such as '13.3' or '14.2'.
        On Serverless compute is string value, e.g. client.1.13, client.4.9
        """
        return os.environ.get("DATABRICKS_RUNTIME_VERSION")

    @property
    def get_dbr_workspace_id(self) -> str:
        """
        This function returns the Databricks workspace ID.

        Returns:
            str: Returns the workspace ID if running in Databricks, "local" otherwise
        """
        try:
            workspace_id = os.environ.get("DATABRICKS_WORKSPACE_ID")
            if workspace_id:
                return workspace_id

            try:
                from dbruntime.databricks_repl_context import get_context  # type: ignore

                context = get_context()
                if context and hasattr(context, "workspaceId"):
                    return context.workspaceId
            except (ImportError, Exception) as e:
                # Expected when not running in Databricks; fall through to return "local"
                _log.info(f"Unable to retrieve Databricks workspace ID via dbruntime: {e}")

            return "local"
        except Exception as e:
            # Catch-all for any unexpected errors; return safe default
            _log.info(f"Failed to retrieve Databricks workspace ID: {e}")
            return "local"

    @property
    def get_dbr_workspace_url(self) -> str:
        """
        This function returns Databricks workspace hostname.

        Returns:
            str: Returns the workspace hostname if running in Databricks, "local" otherwise
        """
        try:
            workspace_url = os.environ.get("DATABRICKS_HOST")

            if not workspace_url:
                try:
                    from dbruntime.databricks_repl_context import get_context

                    context = get_context()
                    if context and hasattr(context, "browserHostName"):
                        workspace_url = context.browserHostName
                except (ImportError, Exception):
                    # Expected when not running in Databricks; fall through to spark.conf
                    pass

            if not workspace_url:
                try:
                    workspace_url = self.spark.conf.get("spark.databricks.workspaceUrl", None)
                except Exception:
                    # spark.conf may not be available; fall through to return "local"
                    pass

            if workspace_url:
                workspace_url = workspace_url.replace("https://", "").replace("http://", "")
                return workspace_url

            return "local"
        except Exception:
            # Catch-all for any unexpected errors; return safe default
            return "local"

    @property
    def get_spark_version(self) -> str:
        """
        This function returns the Spark version.

        Returns:
            str: Returns the Spark version if available, "unknown" otherwise
        """
        try:
            spark_version = getattr(self.spark, "version", None)
            return spark_version if spark_version else "unknown"
        except Exception as e:
            # Catch-all for any unexpected errors; return safe default
            _log.warning(f"Failed to retrieve Spark version: {e}")
            return "unknown"

    @property
    def get_spark_expectations_version(self) -> str:
        """
        This function returns the Spark Expectations package version.

        Returns:
            str: Returns the package version if available, "unknown" otherwise
        """
        try:
            return metadata.version("spark-expectations")
        except metadata.PackageNotFoundError:
            return "unknown"
        except Exception:
            # Catch-all for any unexpected errors; return safe default
            return "unknown"

    @property
    def get_se_job_metadata(self) -> dict:
        """
        This function returns Spark Expectations job metadata for local/Databricks runtimes.

        Returns:
            dict: Job metadata including versions and runtime environment details.
        """
        dbr_version = self.get_dbr_version
        workspace_url = self.get_dbr_workspace_url
        workspace_id = self.get_dbr_workspace_id

        is_databricks = (
            dbr_version is not None or workspace_url != "local" or workspace_id != "local"
        )

        runtime_env = {
            "host": "databricks" if is_databricks else "local",
            "info": {
                "workspace_url": "" if not is_databricks else workspace_url,
                "workspace_id": "" if not is_databricks else workspace_id,
                "dbr_version": "" if not is_databricks else str(dbr_version),
                "job_id": "" if not is_databricks else self.get_dbr_job_id,
            },
        }

        default_metadata = {
            "se_version": self.get_spark_expectations_version,
            "spark_version": self.get_spark_version,
            "runtime_env": runtime_env,
        }
        if self._se_job_metadata is not None:
            merged_metadata = dict(default_metadata)
            merged_metadata["user_metadata"] = self._se_job_metadata
            return merged_metadata

        return default_metadata

    def set_se_job_metadata(self, se_job_metadata: Optional[Any] = None) -> None:
        """
        This function sets Spark Expectations job metadata override.

        Returns:
            None
        """
        if se_job_metadata is None:
            self._se_job_metadata = None
            return

        if isinstance(se_job_metadata, dict):
            self._se_job_metadata = se_job_metadata
            return

        if isinstance(se_job_metadata, str):
            try:
                parsed_value = ast.literal_eval(se_job_metadata)
                if isinstance(parsed_value, dict):
                    self._se_job_metadata = parsed_value
                    return
            except (ValueError, SyntaxError) as e:
                _log.warning(f"Failed to parse se_job_metadata string as dict: {e}")

            self._se_job_metadata = {"job_metadata": se_job_metadata}
            return

        self._se_job_metadata = {"job_metadata": str(se_job_metadata)}

    @property
    def get_dbr_job_id(self) -> str:
        """
        This function returns the Databricks job ID if running in a job.

        Returns:
            str: Returns the job ID if available, "local" otherwise
        """
        try:
            from dbruntime.databricks_repl_context import get_context  # type: ignore

            context = get_context()
            if context and hasattr(context, "jobId"):
                job_id_value = context.jobId
                return "local" if job_id_value is None else str(job_id_value)
            return "local"
        except (ImportError, Exception):
            # Expected when not running in Databricks; return safe default
            return "local"

    @property
    def get_run_id(self) -> str:
        """
        Get run_id for the instance of spark-expectations class

        Returns:
            str: returns the run_id
        """
        return self._run_id

    @property
    def get_run_date(self) -> str:
        """
        Get run_date for the instance of the spark-expectations class

        Returns:
            str: returns the run_date

        """
        return self._run_date

    def set_dq_stats_table_name(self, dq_stats_table_name: str) -> None:
        """
        Sets the DQ stats table name where final statistics will be written.

        Args:
            dq_stats_table_name: The fully qualified table name for DQ statistics
        """
        self._dq_stats_table_name = dq_stats_table_name

    @property
    def get_dq_stats_table_name(self) -> str:
        """
        Get dq_stats_table_name to which the final stats of the dq job will be written into

        Returns:
            str: returns the dq_stats_table_name
        """
        if self._dq_stats_table_name:
            return self._dq_stats_table_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_dq_stats_table_name' before 
            accessing it"""
        )

    def set_dq_expectations(self, dq_expectations: dict) -> None:
        """
        Sets the data quality expectations/rules dictionary.

        Args:
            dq_expectations: Dictionary containing DQ rule definitions
        """
        self._dq_expectations = dq_expectations

    @property
    def get_dq_expectations(self) -> dict:
        """
        Get dq_expectations to which has rule information

        Returns:
            str: returns the rules_df
        """
        if self._dq_expectations:
            return self._dq_expectations
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_dq_expectations' before 
            accessing it"""
        )

    def set_final_table_name(self, final_table_name: str) -> None:
        """
        Sets the final/target table name where validated data will be written.

        Args:
            final_table_name: The fully qualified target table name
        """
        self._final_table_name = final_table_name

    @property
    def get_final_table_name(self) -> str:
        """
        Get dq_stats_table_name to which the final stats of the dq job will be written into

        Returns:
            str: returns the dq_stats_table_name
        """
        if self._final_table_name:
            return self._final_table_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_final_table_name' before 
            accessing it"""
        )

    def set_error_table_name(self, error_table_name: str, user_specified: bool = True) -> None:
        """
        Sets the error table name where failed records will be written.

        Args:
            error_table_name: The fully qualified error table name
            user_specified: Flag indicating if user explicitly provided this name
        """
        self._error_table_name = error_table_name
        self._error_table_name_user_specified = user_specified

    @property
    def get_error_table_name(self) -> str:
        """
        Get error_table_name to which the final stats of the dq job will be written into

        Returns:
            str: returns error_table_name attribute value
        """
        if self._error_table_name:
            return self._error_table_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_error_table_name' before 
            accessing it"""
        )

    @property
    def get_error_table_name_user_specified(self) -> bool:
        """
        Returns True if user override default error table name, otherwise False
        """
        return self._error_table_name_user_specified

    def set_min_priority_email(self, min_priority_email: str) -> None:
        """
        Sets the minimum priority level for email notifications.

        Args:
            min_priority_email: Minimum priority threshold for triggering email alerts
        """
        self._min_priority_email = min_priority_email

    @property
    def get_min_priority_email(self) -> str:
        """
        Returns the min priority for email notifications
        Returns:
            str: The minimum priority for email notifications
        """
        return self._min_priority_email

    def set_min_priority_pagerduty(self, min_priority_pagerduty: str) -> None:
        """
        Sets the minimum priority level for PagerDuty notifications.

        Args:
            min_priority_pagerduty: Minimum priority threshold for triggering PagerDuty alerts
        """
        self._min_priority_pagerduty = min_priority_pagerduty

    @property
    def get_min_priority_pagerduty(self) -> str:
        """
        Returns the min priority for pagerduty notifications
        Returns:
            str: The minimum priority for pagerduty notifications
        """
        return self._min_priority_pagerduty

    def set_min_priority_slack(self, min_priority_slack: str) -> None:
        """
        Sets the minimum priority level for Slack notifications.

        Args:
            min_priority_slack: Minimum priority threshold for triggering Slack alerts
        """
        self._min_priority_slack = min_priority_slack

    @property
    def get_min_priority_slack(self) -> str:
        """
        Returns the min priority for slack notifications
        Returns:
            str: The minimum priority for slack notifications
        """
        return self._min_priority_slack

    def set_min_priority_teams(self, min_priority_teams: str) -> None:
        """
        Sets the minimum priority level for Microsoft Teams notifications.

        Args:
            min_priority_teams: Minimum priority threshold for triggering Teams alerts
        """
        self._min_priority_teams = min_priority_teams

    @property
    def get_min_priority_teams(self) -> str:
        """
        Returns the min priority for teams notifications
        Returns:
            str: The minimum priority for teams notifications
        """
        return self._min_priority_teams

    def set_min_priority_zoom(self, min_priority_zoom: str) -> None:
        """
        Sets the minimum priority level for Zoom notifications.

        Args:
            min_priority_zoom: Minimum priority threshold for triggering Zoom alerts
        """
        self._min_priority_zoom = min_priority_zoom

    @property
    def get_min_priority_zoom(self) -> str:
        """
        Returns the min priority for zoom notifications
        Returns:
            str: The minimum priority for zoom notifications
        """
        return self._min_priority_zoom

    @staticmethod
    def set_run_date() -> str:
        """
        This function is used to generate the current datetime in UTC

        Returns:
            str: Returns the current utc datetime in the format - "%Y-%m-%d %H:%M:%S"

        """
        current_datetime: datetime = datetime.now(timezone.utc)
        return current_datetime.replace(tzinfo=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")

    @property
    def get_row_dq_rule_type_name(self) -> str:
        """
        This function is used to get row data quality rule type name

        Returns:
             str: Returns _row_dq_rule_type_name"

        """
        if self._row_dq_rule_type_name:
            return self._row_dq_rule_type_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_row_dq_rule_type_name' before 
            accessing it"""
        )

    @property
    def get_agg_dq_rule_type_name(self) -> str:
        """
        This function is used to get aggregation data quality rule type name

        Returns:
            str: Returns _agg_dq_rule_type_name"

        """
        if self._agg_dq_rule_type_name:
            return self._agg_dq_rule_type_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_agg_dq_rule_type_name' before 
            accessing it"""
        )

    @property
    def get_query_dq_rule_type_name(self) -> str:
        """
        This function is used to get query data quality rule type name

        Returns:
            str: Returns _query_dq_rule_type_name"

        """
        if self._query_dq_rule_type_name:
            return self._query_dq_rule_type_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_query_dq_rule_type_name' before 
            accessing it"""
        )

    def set_row_dq_status(self, row_dq_status: str = "Skipped") -> None:
        """
        Sets the row-level data quality check status.

        Args:
            row_dq_status: Status of row DQ execution (e.g., "Skipped", "Passed", "Failed")
        """
        self._row_dq_status = row_dq_status

    @property
    def get_row_dq_status(self) -> str:
        """
        This function is used to get row data quality status

        Returns:
            str: Returns _row_dq_status"

        """
        if self._row_dq_status:
            return self._row_dq_status
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_row_dq_status' before 
            accessing it"""
        )

    def set_source_agg_dq_status(self, source_agg_dq_status: str = "Skipped") -> None:
        """
        Sets the source aggregate data quality check status.

        Args:
            source_agg_dq_status: Status of source agg DQ execution (e.g., "Skipped", "Passed", "Failed")
        """
        self._source_agg_dq_status = source_agg_dq_status

    @property
    def get_source_agg_dq_status(self) -> str:
        """
        This function is used to get source aggregation data quality status

        Returns:
            str: Returns _source_agg_dq_status"

        """
        if self._source_agg_dq_status:
            return self._source_agg_dq_status
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_source_agg_dq_status' before 
            accessing it"""
        )

    def set_final_agg_dq_status(self, final_agg_dq_status: str = "Skipped") -> None:
        """
        Sets the final/target aggregate data quality check status.

        Args:
            final_agg_dq_status: Status of final agg DQ execution (e.g., "Skipped", "Passed", "Failed")
        """
        self._final_agg_dq_status = final_agg_dq_status

    @property
    def get_final_agg_dq_status(self) -> str:
        """
        This function is used to get final aggregation data quality status

        Returns:
            str: Returns _final_agg_dq_status"

        """
        if self._final_agg_dq_status:
            return self._final_agg_dq_status
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_final_agg_dq_status' before 
            accessing it"""
        )

    def set_source_query_dq_status(self, source_query_dq_status: str = "Skipped") -> None:
        """
        Sets the source query data quality check status.

        Args:
            source_query_dq_status: Status of source query DQ execution (e.g., "Skipped", "Passed", "Failed")
        """
        self._source_query_dq_status = source_query_dq_status

    @property
    def get_source_query_dq_status(self) -> str:
        """
        This function is used to get source query data quality status

        Returns:
            str: Returns _source_query_dq_status"

        """
        if self._source_query_dq_status:
            return self._source_query_dq_status
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_source_query_dq_status' before 
            accessing it"""
        )

    def set_final_query_dq_status(self, final_query_dq_status: str = "Skipped") -> None:
        """
        Sets the final/target query data quality check status.

        Args:
            final_query_dq_status: Status of final query DQ execution (e.g., "Skipped", "Passed", "Failed")
        """
        self._final_query_dq_status = final_query_dq_status

    @property
    def get_final_query_dq_status(self) -> str:
        """
        This function is used to get final query dq  data quality status

        Returns:
            str: Returns _final_query_dq_status"

        """
        if self._final_query_dq_status:
            return self._final_query_dq_status
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_final_query_dq_status' before 
            accessing it"""
        )

    def set_dq_run_status(self, dq_run_status: str = "Failed") -> None:
        """
        Sets the overall data quality pipeline run status.

        Args:
            dq_run_status: Overall DQ run status (e.g., "Failed", "Passed")
        """
        self._dq_run_status = dq_run_status

    @property
    def get_dq_run_status(self) -> str:
        """
        This function is used to get data quality pipeline status

        Returns:
            str: Returns _dq_status"

        """
        if self._dq_run_status:
            return self._dq_run_status
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_dq_run_status' before 
            accessing it"""
        )

    def set_kafka_write_status(self, kafka_write_status: str = "Disabled") -> None:
        """
        This function sets the Kafka write status
        Args:
            kafka_write_status: Status of Kafka write operation ("Success", "Failed", "Disabled")
        Returns: None
        """
        self._kafka_write_status = kafka_write_status

    @property
    def get_kafka_write_status(self) -> str:
        """
        This function returns the Kafka write status
        Returns:
            str: _kafka_write_status ("Success", "Failed", "Disabled")
        """
        return getattr(self, "_kafka_write_status", "Disabled")

    def set_kafka_write_error_message(self, error_message: str = "") -> None:
        """
        This function sets the Kafka write error message
        Args:
            error_message: Error message from Kafka write failure
        Returns: None
        """
        self._kafka_write_error_message = error_message

    @property
    def get_kafka_write_error_message(self) -> str:
        """
        This function returns the Kafka write error message
        Returns:
            str: Returns _kafka_write_error_message
        """
        return getattr(self, "_kafka_write_error_message", "")

    @property
    def get_config_file_path(self) -> str:
        """
        This function returns config file abs path
        Returns:
            str: Returns _config_file_path(str)

        """
        if self._dq_config_abs_path:
            return self._dq_config_abs_path

        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_dq_config_abs_path' before
            accessing it"""
        )

    def set_mail_smtp_server(self, mail_smtp_server: str) -> None:
        """
        Sets the SMTP server hostname for email notifications.

        Args:
            mail_smtp_server: SMTP server hostname or IP address
        """
        self._mail_smtp_server = mail_smtp_server

    @property
    def get_mail_smtp_server(self) -> str:
        """
        This functions returns smtp server host
        Returns:
            str: returns _mail_smtp_server

        """
        if self._mail_smtp_server:
            return self._mail_smtp_server

        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_mail_smtp_server' before 
            accessing it"""
        )

    def set_mail_smtp_port(self, mail_smtp_port: int) -> None:
        """
        Sets the SMTP server port for email notifications.

        Args:
            mail_smtp_port: SMTP server port number (e.g., 25, 465, 587)
        """
        self._mail_smtp_port = mail_smtp_port

    @property
    def get_mail_smtp_port(self) -> int:
        """
        This functions returns smtp port
        Returns:
            int: returns _mail_smtp_server port

        """
        if self._mail_smtp_port:
            return self._mail_smtp_port

        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_mail_smtp_port' before 
            accessing it"""
        )

    def set_mail_smtp_password(self, mail_smtp_password: str) -> None:
        """
        Sets the SMTP password for authentication.

        Args:
            mail_smtp_password: Password for SMTP authentication
        """
        self._mail_smtp_password = mail_smtp_password

    @property
    def get_mail_smtp_password(self) -> Optional[str]:
        """
        This functions returns smtp password
        Returns:
            str: returns _mail_smtp_server password or None if smtp password is not set

        """
        if self._mail_smtp_password:
            return self._mail_smtp_password

        else:
            return None

    def set_mail_smtp_user_name(self, mail_smtp_user_name: str) -> None:
        """
        Sets the SMTP username for authentication.
        Args:
            mail_smtp_user_name: The SMTP username to use for authentication
        """
        self._mail_smtp_user_name = mail_smtp_user_name

    @property
    def get_mail_smtp_user_name(self) -> Optional[str]:
        """
        Returns the SMTP username for authentication, if configured.
        Returns:
            str: _mail_smtp_user_name or None if not set
        """
        return self._mail_smtp_user_name

    def set_smtp_creds_dict(self, smtp_creds_dict: Dict[str, str]) -> None:
        """
        This function helps to set secret keys dict for smtp server authentication"""
        self._smtp_creds_dict = smtp_creds_dict

    @property
    def get_smtp_creds_dict(self) -> Dict[str, str]:
        """
        This function returns secret keys dict for smtp server authentication
        """
        return self._smtp_creds_dict

    def set_enable_mail(self, enable_mail: bool) -> None:
        """
        Enables or disables email notifications.

        Args:
            enable_mail: True to enable email notifications, False to disable
        """
        self._enable_mail = bool(enable_mail)

    @property
    def get_enable_mail(self) -> bool:
        """
        This function return whether mail notification to enable or not
        Returns:
            str: Returns  _enable_mail(bool)

        """
        return self._enable_mail

    def set_enable_smtp_server_auth(self, enable_smtp_server_auth: bool) -> None:
        """
        Enables or disables SMTP server authentication.

        Args:
            enable_smtp_server_auth: True to enable SMTP authentication, False to disable
        """
        self._enable_smtp_server_auth = bool(enable_smtp_server_auth)

    @property
    def get_enable_smtp_server_auth(self) -> bool:
        """
        This function return whether smtp server requires authentication or not
        Returns:
            str: Returns  _enable_smtp_server_auth(bool)

        """
        return self._enable_smtp_server_auth

    def set_to_mail(self, to_mail: str) -> None:
        """
        Sets the recipient email addresses for notifications.

        Args:
            to_mail: Comma-separated email addresses of recipients
        """
        self._to_mail = to_mail

    @property
    def get_to_mail(self) -> str:
        """
        This function returns list of mail id's
        Returns:
            str: Returns _mail_id(str)

        """
        if self._to_mail:
            return self._to_mail

        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_to_mail' before 
            accessing it"""
        )

    def set_enable_custom_email_body(self, enable_custom_email_body: bool) -> None:
        """
        Enables or disables custom email body content.

        Args:
            enable_custom_email_body: True to enable custom email body, False to use default
        """
        self._enable_custom_email_body = bool(enable_custom_email_body)

    @property
    def get_enable_custom_email_body(self) -> bool:
        """
        This function return whether to enable custom email body or not
        Returns:
            str: Returns  _enable_custom_email_body(bool)

        """
        return self._enable_custom_email_body

    def set_enable_templated_basic_email_body(self, enable_templated_basic_email_body: bool) -> None:
        """
        Enables or disables HTML templating for basic email notifications.

        Args:
            enable_templated_basic_email_body: True to enable HTML templating, False to disable
        """
        self._enable_templated_basic_email_body = bool(enable_templated_basic_email_body)

    @property
    def get_enable_templated_basic_email_body(self) -> bool:
        """
        This function return whether to enable html templating for basic emails or not
        Returns:
            str: Returns  _enable_templated_basic_email_body(bool)

        """
        return self._enable_templated_basic_email_body

    def set_enable_templated_custom_email(self, enable_templated_custom_email: bool) -> None:
        """
        Enables or disables HTML templating for custom email notifications.

        Args:
            enable_templated_custom_email: True to enable HTML templating for custom emails, False to disable
        """
        self._enable_templated_custom_email = bool(enable_templated_custom_email)

    @property
    def get_enable_templated_custom_email(self) -> bool:
        """
        This function return whether to enable html templating for custom emails or not
        Returns:
            str: Returns  _enable_templated_custom_email(bool)

        """
        return self._enable_templated_custom_email

    def set_mail_from(self, mail_from: str) -> None:
        """
        Sets the sender email address for notifications.

        Args:
            mail_from: Email address of the sender
        """
        self._mail_from = mail_from

    @property
    def get_mail_from(self) -> str:
        """
        This function returns mail id to send email
        Returns:

        """
        if self._mail_from:
            return self._mail_from

        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_mail_from' before 
            accessing it"""
        )

    def set_mail_subject(self, mail_subject: str) -> None:
        """
        Sets the email subject line for notifications.

        Args:
            mail_subject: Subject line for email notifications
        """
        self._mail_subject = mail_subject

    @property
    def get_mail_subject(self) -> str:
        """
        This function returns mail subject
        Returns:
            str: Returns _mail_subject(str)

        """
        if self._mail_subject:
            return self._mail_subject

        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_mail_subject' before 
            accessing it"""
        )

    def set_email_custom_body(self, email_custom_body: str) -> None:
        """
        Sets the custom email body content.

        Args:
            email_custom_body: Custom HTML or text content for email body
        """
        self._email_custom_body = email_custom_body

    @property
    def get_email_custom_body(self) -> str:
        """
        This function returns email custom body
        Returns:
            str: Returns _email_custom_body(str)

        """
        if self._email_custom_body:
            return self._email_custom_body
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_email_custom_body' before 
            accessing it"""
        )

    def set_enable_slack(self, enable_slack: bool) -> None:
        """

        Args:
            enable_slack:

        Returns:

        """
        self._enable_slack = enable_slack

    @property
    def get_enable_slack(self) -> bool:
        """
        This function returns whether to enable slack notification or not
        Returns: Returns _enable_slack(bool)

        """
        return self._enable_slack

    def set_slack_webhook_url(self, slack_webhook_url: str) -> None:
        """
        Sets the Slack webhook URL for notifications.

        Args:
            slack_webhook_url: Slack incoming webhook URL
        """
        self._slack_webhook_url = slack_webhook_url

    @property
    def get_slack_webhook_url(self) -> str:
        """
        This function returns sack webhook url
        Returns:
            str: Returns _webhook_url(str)

        """

        if self._slack_webhook_url:
            return self._slack_webhook_url
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_slack_webhook_url' before 
            accessing it"""
        )

    def set_enable_teams(self, enable_teams: bool) -> None:
        """

        Args:
            enable_teams:

        Returns:

        """
        self._enable_teams = enable_teams

    @property
    def get_enable_teams(self) -> bool:
        """
        This function returns whether to enable teams notification or not
        Returns: Returns _enable_teams(bool)

        """
        return self._enable_teams

    def set_teams_webhook_url(self, teams_webhook_url: str) -> None:
        """
        Sets the Microsoft Teams webhook URL for notifications.

        Args:
            teams_webhook_url: Teams incoming webhook URL
        """
        self._teams_webhook_url = teams_webhook_url

    @property
    def get_teams_webhook_url(self) -> str:
        """
        This function returns sack webhook url
        Returns:
            str: Returns _webhook_url(str)

        """

        if self._teams_webhook_url:
            return self._teams_webhook_url
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_teams_webhook_url' before 
            accessing it"""
        )

    # def set_enable_zoom(self, enable_zoom: bool, zoom_token: str) -> None:
    def set_enable_zoom(self, enable_zoom: bool) -> None:
        """
        Set whether to enable Zoom notification and its token.

        Args:
            enable_zoom (bool): Whether to enable Zoom notification or not.
        """
        self._enable_zoom = enable_zoom

    @property
    def get_enable_zoom(self) -> bool:
        """
        Get whether Zoom notification is enabled.

        Returns:
            bool: Whether Zoom notification is enabled or not.
        """
        return self._enable_zoom

    def set_zoom_webhook_url(self, zoom_webhook_url: str) -> None:
        """
        Set the Zoom webhook URL.

        Args:
            zoom_webhook_url (str): The webhook URL for Zoom notification.
        """
        self._zoom_webhook_url = zoom_webhook_url

    @property
    def get_zoom_webhook_url(self) -> str:
        """
        Get the Zoom webhook URL.

        Returns:
            str: The Zoom webhook URL.
        """
        if self._zoom_webhook_url:
            return self._zoom_webhook_url
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_zoom_webhook_url' before 
            accessing it"""
        )

    def set_zoom_token(self, zoom_token: str) -> None:
        """
        Set the Zoom webhook token.

        Args:
            zoom_token (str): The token for Zoom notification.
        """
        self._zoom_token = zoom_token

    @property
    def get_zoom_token(self) -> str:
        """
        Get the Zoom token.

        Returns:
            str: The Zoom token.
        """
        if self._zoom_token:
            return self._zoom_token
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_zoom_token' before 
            accessing it"""
        )

    def set_enable_pagerduty(self, enable_pagerduty: bool) -> None:
        """
        Set whether to enable pagerduty notification or not

        Args:
            enable_pagerduty (bool): Whether to enable pagerduty incidents or not
        """
        self._enable_pagerduty = enable_pagerduty

    @property
    def get_enable_pagerduty(self) -> bool:
        """
        This function returns if pagerduty notifications are enabled or not.

        Returns:
            bool: Whether to enable pagerduty incidents or not
        """
        return self._enable_pagerduty

    def set_pagerduty_integration_key(self, pagerduty_integration_key: str) -> None:
        """
        Set the pagerduty integration key manually.

        Args:
            pagerduty_integration_key (str): Integration key for PagerDuty when creating incidents.
        """
        self._pagerduty_integration_key = pagerduty_integration_key

    @property
    def get_pagerduty_integration_key(self) -> Optional[str]:
        """
        This function returns pagerduty integration key
        Returns:
            str: Returns _pagerduty_integration_key(str)

        """
        if self._pagerduty_integration_key:
            return self._pagerduty_integration_key
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_pagerduty_integration_key' before 
            accessing it"""
        )

    def set_pagerduty_webhook_url(self, pagerduty_webhook_url: str) -> None:
        """
        This function helps to set pagerduty webhook url
        Args:
            pagerduty_webhook_url (str): PagerDuty webhook url to create incidents
        """
        self._pagerduty_webhook_url = pagerduty_webhook_url

    @property
    def get_pagerduty_webhook_url(self) -> str:
        """
        This function returns pagerduty webhook url
        Returns:
            str: Returns _pagerduty_webhook_url(str)

        """
        if self._pagerduty_webhook_url:
            return self._pagerduty_webhook_url
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_pagerduty_webhook_url' before 
            accessing it"""
        )

    def set_pagerduty_creds_dict(self, pagerduty_creds_dict: Dict[str, str]) -> None:
        """
        This function helps to set secret keys dict for pagerduty authentication
        Args:
            pagerduty_creds_dict (Dict[str, str]): Dictionary containing secrets for PagerDuty authentication
        """
        self._pagerduty_creds_dict = pagerduty_creds_dict

    @property
    def get_pagerduty_creds_dict(self) -> Dict[str, str]:
        """
        This function returns secret keys dict for pagerduty authentication
        """
        return self._pagerduty_creds_dict

    def set_table_name(self, table_name: str) -> None:
        """
        Sets the table name being processed.

        Args:
            table_name: Name of the table being processed
        """
        self._table_name = table_name

    @property
    def get_table_name(self) -> str:
        """
        This function returns table name
        Returns:
            str: Returns _table_name(str)

        """
        if self._table_name:
            return self._table_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_table_name' before 
            accessing it"""
        )

    def set_input_count(self, input_count: int = 0) -> None:
        """
        Sets the input record count before DQ processing.

        Args:
            input_count: Number of input records
        """
        self._input_count = input_count

    @property
    def get_input_count(self) -> int:
        """
        This function return input count
        Returns:
            int: Returns _input_count(int)

        """
        return self._input_count

    def set_error_count(self, error_count: int = 0) -> None:
        """
        Sets the error record count after DQ processing.

        Args:
            error_count: Number of records that failed DQ rules
        """
        self._error_count = error_count

    @property
    def get_error_count(self) -> int:
        """
        This functions return error count
        Returns:
            int: Returns _error_count(int)

        """
        return self._error_count

    def set_output_count(self, output_count: int = 0) -> None:
        """
        Sets the output record count after DQ processing.

        Args:
            output_count: Number of records that passed DQ rules
        """
        self._output_count = output_count

    @property
    def get_output_count(self) -> int:
        """
        This function returns output count
        Returns:
            int: Returns _output(int)

        """
        return self._output_count

    def set_source_agg_dq_result(self, source_agg_dq_result: Optional[List[Dict[str, str]]] = None) -> None:
        """
        Sets the source aggregate data quality check results.

        Args:
            source_agg_dq_result: List of dictionaries containing source agg DQ results
        """
        self._source_agg_dq_result = source_agg_dq_result

    @property
    def get_source_agg_dq_result(self) -> Optional[List[Dict[str, str]]]:
        """
        This function return status of the source_agg_dq_result
        Returns:
            dict: Returns source_agg_dq_result which in list of dict with str(key) and str(value)

        """

        return self._source_agg_dq_result

    def set_final_agg_dq_result(self, final_agg_dq_result: Optional[List[Dict[str, str]]] = None) -> None:
        """
        Sets the final/target aggregate data quality check results.

        Args:
            final_agg_dq_result: List of dictionaries containing final agg DQ results
        """
        self._final_agg_dq_result = final_agg_dq_result

    @property
    def get_final_agg_dq_result(self) -> Optional[List[Dict[str, str]]]:
        """
        This function return status of the final_agg_dq_result
        Returns:
            dict: Returns final_agg_dq_result which in list of dict with str(key) and str(value)

        """
        return self._final_agg_dq_result

    def set_source_query_dq_result(self, source_query_dq_result: Optional[List[Dict[str, str]]] = None) -> None:
        """
        Sets the source query data quality check results.

        Args:
            source_query_dq_result: List of dictionaries containing source query DQ results
        """
        self._source_query_dq_result = source_query_dq_result

    @property
    def get_source_query_dq_result(self) -> Optional[List[Dict[str, str]]]:
        """
        This function return status of the source_query_dq_result
        Returns:
            dict: Returns source_query_dq_result which in list of dict with str(key) and str(value)

        """
        return self._source_query_dq_result

    def set_final_query_dq_result(self, final_query_dq_result: Optional[List[Dict[str, str]]] = None) -> None:
        """
        Sets the final/target query data quality check results.

        Args:
            final_query_dq_result: List of dictionaries containing final query DQ results
        """
        self._final_query_dq_result = final_query_dq_result

    @property
    def get_final_query_dq_result(self) -> Optional[List[Dict[str, str]]]:
        """
        This function return status of the final_query_dq_result
        Returns:
            dict: Returns final_query_dq_result which in list of dict with str(key) and str(value)

        """
        return self._final_query_dq_result

    def set_notification_on_start(self, notification_on_start: bool) -> None:
        """
        Enables or disables notifications when DQ processing starts.

        Args:
            notification_on_start: True to send notification on start, False to disable
        """
        self._notification_on_start = notification_on_start

    @property
    def get_notification_on_start(self) -> bool:
        """
        This function returns notification on start
        Returns:
            bool: Returns _notification_on_start

        """
        return self._notification_on_start

    def set_notification_on_completion(self, notification_on_completion: bool) -> None:
        """
        Enables or disables notifications when DQ processing completes.

        Args:
            notification_on_completion: True to send notification on completion, False to disable
        """
        self._notification_on_completion = notification_on_completion

    @property
    def get_notification_on_completion(self) -> bool:
        """
        This function returns notification on completion
        Returns:
            bool: Returns _notification_on_completion

        """
        return self._notification_on_completion

    def set_notification_on_fail(self, notification_on_fail: bool) -> None:
        """
        Enables or disables notifications when DQ processing fails.

        Args:
            notification_on_fail: True to send notification on failure, False to disable
        """
        self._notification_on_fail = notification_on_fail

    @property
    def get_notification_on_fail(self) -> bool:
        """
        This function returns notification on fail
        Returns:
            bool: Returns _notification_on_fail

        """
        return self._notification_on_fail
    
    def set_notification_on_error_drop_exceeds_threshold_breach(self, notification_on_error_drop_exceeds_threshold_breach: bool) -> None:
        """
        Enables or disables notifications when error drop exceeds threshold.

        Args:
            notification_on_error_drop_exceeds_threshold_breach: True to send notification on threshold breach, False to disable
        """
        self._notification_on_error_drop_exceeds_threshold_breach = notification_on_error_drop_exceeds_threshold_breach

    @property
    def get_notification_on_error_drop_exceeds_threshold_breach(self) -> bool:
        """
        Returns whether notification is enabled when error drop exceeds threshold.

        Returns:
            bool: True if notification on threshold breach is enabled, False otherwise
        """
        return self._notification_on_error_drop_exceeds_threshold_breach
    
    def set_notifications_on_rules_action_if_failed_set_ignore(self, notifications_on_rules_action_if_failed_set_ignore: bool) -> None:
        """
        Enables or disables notifications for rules with action_if_failed set to 'ignore'.

        Args:
            notifications_on_rules_action_if_failed_set_ignore: True to send notifications for ignored failures, False to disable
        """
        self._notifications_on_rules_action_if_failed_set_ignore = notifications_on_rules_action_if_failed_set_ignore

    @property
    def get_notifications_on_rules_action_if_failed_set_ignore(self) -> bool:
        """
        Returns whether notifications are enabled for rules with action_if_failed set to 'ignore'.

        Returns:
            bool: True if notifications for ignored failures are enabled, False otherwise
        """
        return self._notifications_on_rules_action_if_failed_set_ignore

    

    def set_env(self, env: Optional[str]) -> None:
        """
        Args:
            env: which accepts env type

        Returns:
            None

        """
        self._env = env

    @property
    def get_env(self) -> Optional[str]:
        """
        functions returns running environment type
        Returns:
               str: Returns _env
        """
        return self._env

    @property
    def get_error_percentage(self) -> float:
        """
        This function returns error percentage
        Returns:
               float: error percentage
        """
        if self._input_count > 0:
            return round((self.get_error_count / self.get_input_count) * 100, 2)
        return 0.0

    @property
    def get_output_percentage(self) -> float:
        """
        This function return output percentage
        Returns:
            float: output percentage

        """
        if self._input_count > 0:
            return round((self.get_output_count / self.get_input_count) * 100, 2)
        return 0.0

    @property
    def get_success_percentage(self) -> float:
        """
        This function returns success percentage
        Returns:
            float: success percentage

        """
        if self._input_count > 0:
            return round(
                ((self.get_input_count - self.get_error_count) / self.get_input_count) * 100,
                2,
            )
        return 0.0

    @property
    def get_error_drop_percentage(self) -> float:
        """
        This function returns error drop percentage percentage
        Returns:
            float: error drop percentage

        """
        if self._input_count > 0:
            return round(
                ((self.get_input_count - self.get_output_count) / self.get_input_count) * 100,
                2,
            )
        return 0.0

    def set_error_drop_threshold(self, error_drop_threshold: int) -> None:
        """
        Sets the error drop threshold percentage.

        Args:
            error_drop_threshold: Maximum allowed error percentage before alerting
        """
        self._error_drop_threshold = error_drop_threshold

    @property
    def get_error_drop_threshold(self) -> int:
        """
        This function return error threshold breach
        Returns:
               int: error threshold breach
        """
        if self._error_drop_threshold:
            return self._error_drop_threshold
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_error_drop_threshold'  before 
            accessing it"""
        )

    @property
    def get_cerberus_url(self) -> str:
        """
        This functions implemented to return cerberus url
        Returns:

        """
        if self._cerberus_url:
            return self._cerberus_url
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_cerberus_url'  before 
            accessing it"""
        )

    @property
    def get_cerberus_cred_path(self) -> str:
        """
        This functions implemented to return cerberus credentials path
        Returns:

        """
        if self._cerberus_cred_path:
            return self._cerberus_cred_path
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_cerberus_cred_path'  before 
            accessing it"""
        )

    @property
    def get_cerberus_token(self) -> str:
        """
        This functions implemented to return cerberus token
        Returns:

        """
        if self._cerberus_token:
            return self._cerberus_token
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_cerberus_token'  before 
            accessing it"""
        )

    def set_se_streaming_stats_dict(self, se_streaming_stats_dict: Dict[str, str]) -> None:
        """
        This function helps to set secret keys dict"""
        self._se_streaming_stats_dict = se_streaming_stats_dict

    @property
    def get_se_streaming_stats_dict(self) -> Dict[str, str]:
        """
        This function returns secret keys dict
        """
        return self._se_streaming_stats_dict

    @property
    def get_secret_type(self) -> Optional[str]:
        """
        This function helps in getting secret type
        Returns:
             secret type in Optional[str]

        """
        if self._se_streaming_stats_dict.get(user_config.secret_type):
            return self._se_streaming_stats_dict[user_config.secret_type].lower()
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign 
            'UserConfig.secret_type' before 
            accessing it"""
        )

    @property
    def get_server_url_key(self) -> Optional[str]:
        """
        This function helps in getting key / path for kafka server url
         Returns:
              kafka server url key / path in Optional[str]
        """
        _server_url_key: Optional[str] = (
            self._se_streaming_stats_dict.get(user_config.cbs_kafka_server_url)
            if self.get_secret_type == "cerberus"
            else self._se_streaming_stats_dict.get(user_config.dbx_kafka_server_url)
        )
        if _server_url_key:
            return _server_url_key
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign
            'UserConfig.cbs_kafka_server_url' before
            accessing it"""
        )

    @property
    def get_token_endpoint_url(self) -> Optional[str]:
        """
        This function helps in getting key / path for end point url
         Returns:
             end point url key / path in  Optional[str]
        """
        _token_endpoint_url: Optional[str] = (
            self._se_streaming_stats_dict.get(user_config.cbs_secret_token_url)
            if self.get_secret_type == "cerberus"
            else self._se_streaming_stats_dict.get(user_config.dbx_secret_token_url)
        )
        if _token_endpoint_url:
            return _token_endpoint_url
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign
            'UserConfig.cbs_secret_token_url' before
            accessing it"""
        )

    @property
    def get_token(self) -> Optional[str]:
        """
        This function helps in getting key / path for token
        Returns:
             token key / path in Optional[str]
        """
        _token: Optional[str] = (
            self._se_streaming_stats_dict.get(user_config.cbs_secret_token)
            if self.get_secret_type == "cerberus"
            else self._se_streaming_stats_dict.get(user_config.dbx_secret_token)
        )
        if _token:
            return _token
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign
            'UserConfig.cbs_secret_token' before
            accessing it"""
        )

    @property
    def get_client_id(self) -> Optional[str]:
        """
        This function helps in getting key / path for client id
        Returns:
            client id key / path in Optional[str]
        """
        _client_id: Optional[str] = (
            self._se_streaming_stats_dict.get(user_config.cbs_secret_app_name)
            if self.get_secret_type == "cerberus"
            else self._se_streaming_stats_dict.get(user_config.dbx_secret_app_name)
        )
        if _client_id:
            return _client_id
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign
            'UserConfig.cbs_secret_app_name' before
            accessing it"""
        )

    @property
    def get_topic_name(self) -> Optional[str]:
        """
        This function helps in getting key / path for topic name
        Returns:
            topic name key / path in Optional[str]
        """

        _topic_name: Optional[str] = (
            self._se_streaming_stats_dict.get(user_config.se_streaming_stats_topic_name)
            if self.get_se_streaming_stats_kafka_custom_config_enable or self.get_env == "local"
            else (
                self._se_streaming_stats_dict.get(user_config.cbs_topic_name)
                if self.get_secret_type == "cerberus"
                else self._se_streaming_stats_dict.get(user_config.dbx_topic_name)
            )
        )

        if _topic_name:
            return _topic_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign 
            'UserConfig.cbs_topic_name' before 
            accessing it"""
        )

    def set_se_streaming_stats_kafka_custom_config_enable(self, se_streaming_stats_kafka_config_enable: bool) -> None:
        self._se_streaming_stats_kafka_custom_config_enable = se_streaming_stats_kafka_config_enable

    @property
    def get_se_streaming_stats_kafka_custom_config_enable(self) -> bool:
        """
        This function returns whether it's enabled to use the custom kafka config
        Returns:
            bool: Returns _se_streaming_stats_kafka_custom_config_enable

        """
        if self._se_streaming_stats_kafka_custom_config_enable:
            return self._se_streaming_stats_kafka_custom_config_enable
        return False

    def set_se_streaming_stats_kafka_bootstrap_server(self, se_streaming_stats_kafka_server: str) -> None:
        self._se_streaming_stats_kafka_bootstrap_server = se_streaming_stats_kafka_server

    @property
    def get_se_streaming_stats_kafka_bootstrap_server(self) -> str:
        """
        This function returns kafka bootstrap server that was specified in custom config
        Returns:
            str: Returns _se_streaming_stats_kafka_bootstrap_server

        """
        if self._se_streaming_stats_kafka_bootstrap_server:
            return self._se_streaming_stats_kafka_bootstrap_server
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign 
            'se_streaming_stats_kafka_bootstrap_server' before 
            accessing it"""
        )

    def set_debugger_mode(self, debugger_mode: bool) -> None:
        """
        This function sets debugger mode
        Returns:

        """
        self._debugger_mode = debugger_mode

    @property
    def get_debugger_mode(self) -> bool:
        """
        This function returns a debugger
        Returns:
             bool: return debugger

        """
        return self._debugger_mode

    def print_dataframe_with_debugger(self, df: DataFrame) -> None:
        """
        This function has a debugger that can print out the DataFrame
        Returns:

        """
        if self.get_debugger_mode:
            df.show(truncate=False)

    def set_supported_df_query_dq(self) -> DataFrame:
        return self.spark.createDataFrame(
            [{"spark_expectations_query_check": "supported_place_holder_dataset_to_run_query_check"}]
        )

    @property
    def get_supported_df_query_dq(self) -> DataFrame:
        """
        This function returns the place holder dataframe for query check
        Returns:
            DataFrame: returns dataframe for query dq

        """

        if self._supported_df_query_dq:
            return self._supported_df_query_dq
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_supported_df_query_dq'  before 
            accessing it"""
        )

    def set_source_agg_dq_start_time(self) -> None:
        """
        This function sets start time source agg dq computation
        Returns:
             None

        """
        self._source_agg_dq_start_time = datetime.now()

    def set_source_agg_dq_end_time(self) -> None:
        """
        This function sets end time source agg dq computation
        Returns:
            None

        """
        self._source_agg_dq_end_time = datetime.now()

    def set_final_agg_dq_start_time(self) -> None:
        """
        This function sets start time final agg dq computation
        Returns:
        None

        """
        self._final_agg_dq_start_time = datetime.now()

    def set_final_agg_dq_end_time(self) -> None:
        """
        This function sets end time final agg dq computation
        Returns:
            None
        """
        self._final_agg_dq_end_time = datetime.now()

    def set_source_query_dq_start_time(self) -> None:
        """
        This function sets start time source query dq computation
        Returns:
            None
        """
        self._source_query_dq_start_time = datetime.now()

    def set_source_query_dq_end_time(self) -> None:
        """
        This function sets end time source query dq computation
        Returns:
            None
        """
        self._source_query_dq_end_time = datetime.now()

    def set_final_query_dq_start_time(self) -> None:
        """
        This function sets start time final query dq computation
        Returns:
            None
        """
        self._final_query_dq_start_time = datetime.now()

    def set_final_query_dq_end_time(self) -> None:
        """
        This function sets end time final query dq computation
        Returns:
            None
        """
        self._final_query_dq_end_time = datetime.now()

    def set_row_dq_start_time(self) -> None:
        """
        This function sets start time row dq computation
        Returns:
            None
        """
        self._row_dq_start_time = datetime.now()

    @property
    def get_row_dq_start_time(self) -> datetime:
        """
        This function sets start time row dq computation
        Returns:
            None
        """
        if self._row_dq_start_time:
            return self._row_dq_start_time
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, 
            please assign '_row_dq_start_time'  before 
            accessing it"""
        )

    def set_row_dq_end_time(self) -> None:
        """
        This function sets end time row dq computation
        Returns:
            None
        """
        self._row_dq_end_time = datetime.now()

    @property
    def get_row_dq_end_time(self) -> datetime:
        """
        This function sets end time row dq computation
        Returns:
            None
        """
        if self._row_dq_end_time:
            return self._row_dq_end_time
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_row_dq_end_time'  before 
            accessing it"""
        )

    def set_dq_start_time(self) -> None:
        """
        This function sets start time dq computation
        Returns:
            None
        """
        self._dq_start_time = datetime.now()

    def set_dq_end_time(self) -> None:
        """
        This function sets end time dq computation
        Returns:
            None
        """
        self._dq_end_time = datetime.now()

    def set_end_time_when_dq_job_fails(self) -> None:
        """
        function used to set end time when job fails in any one of the stages by using start time
        Returns:
        """
        if self._source_agg_dq_start_time and self._source_agg_dq_end_time is None:
            self.set_source_agg_dq_end_time()
        elif self._source_query_dq_start_time and self._source_query_dq_end_time is None:
            self.set_source_query_dq_end_time()
        elif self._row_dq_start_time and self._row_dq_end_time is None:
            self.set_row_dq_end_time()
        elif self._final_agg_dq_start_time and self._final_agg_dq_end_time is None:
            self.set_final_agg_dq_end_time()
        elif self._final_query_dq_start_time and self._final_query_dq_end_time is None:
            self.set_final_query_dq_end_time()

    def get_time_diff(self, start_time: Optional[datetime], end_time: Optional[datetime]) -> float:
        """
        This function implements time diff
        Args:
            start_time:
            end_time:

        Returns:

        """
        if start_time and end_time:
            time_diff = end_time - start_time

            return round(float(time_diff.total_seconds()), 1)
        else:
            return 0.0

    @property
    def get_source_agg_dq_run_time(self) -> float:
        """
        This function implements time diff for source agg dq run
        Returns:
             float: time in float

        """
        return self.get_time_diff(self._source_agg_dq_start_time, self._source_agg_dq_end_time)

    @property
    def get_final_agg_dq_run_time(self) -> float:
        """
        This function implements time diff for final agg dq run
        Returns:
             float: time in float

        """
        return self.get_time_diff(self._final_agg_dq_start_time, self._final_agg_dq_end_time)

    @property
    def get_source_query_dq_run_time(self) -> float:
        """
        This function implements time diff for source query dq run
        Returns:
            float: time in float
        """
        return self.get_time_diff(self._source_query_dq_start_time, self._source_query_dq_end_time)

    @property
    def get_final_query_dq_run_time(self) -> float:
        """
        This function implements time diff for final query dq run
        Returns:
            float: time in float
        """
        return self.get_time_diff(self._final_query_dq_start_time, self._final_query_dq_end_time)

    @property
    def get_row_dq_run_time(self) -> float:
        """
        This function implements time diff for row dq run
        Returns:
            float: time in float
        """
        return self.get_time_diff(self._row_dq_start_time, self._row_dq_end_time)

    @property
    def get_dq_run_time(self) -> float:
        """
        This function implements time diff for dq run
        Returns:
            float: time in float
        """
        return self.get_time_diff(self._dq_start_time, self._dq_end_time)

    @property
    def get_run_id_name(self) -> str:
        """
        This function returns name for the run_id column
        Returns:
            str: name of run_id in str

        """
        if self._run_id_name:
            return self._run_id_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_run_id_name'  before 
            accessing it"""
        )

    @property
    def get_run_date_name(self) -> str:
        """
        This function returns name for the run_date column
        Returns:
            str: name of run_date in str

        """
        if self._run_date_name:
            return self._run_date_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_run_date_name'  before 
            accessing it"""
        )

    @property
    def get_run_date_time_name(self) -> str:
        """
        This function returns name for the run_date_time column
        Returns:
            str: name of run_date_time in str

        """
        if self._run_date_time_name:
            return self._run_date_time_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_run_date_time_name'  before 
            accessing it"""
        )

    def reset_num_row_dq_rules(self) -> None:
        """
        This function used to reset the _num_row_dq_rules
        Returns:
            None

        """

        self._num_row_dq_rules = 0  # pragma: no cover

    def reset_num_agg_dq_rules(self) -> None:
        """
        This function used to reset the_num_agg_dq_rules
        Returns:
            None

        """
        self._num_agg_dq_rules = {
            "num_agg_dq_rules": 0,
            "num_source_agg_dq_rules": 0,
            "num_final_agg_dq_rules": 0,
        }

    def reset_num_query_dq_rules(self) -> None:
        """
        This function used to rest the _num_query_dq_rules
        Returns:
            None

        """
        self._num_query_dq_rules = {
            "num_query_dq_rules": 0,
            "num_source_query_dq_rules": 0,
            "num_final_query_dq_rules": 0,
        }

    def reset_num_dq_rules(self) -> None:
        """
        This function used to reset the _num_dq_rules
        Returns:
            None

        """
        self._num_dq_rules = 0

    def set_num_row_dq_rules(self) -> None:
        """
        This function sets number of applied row dq rules for batch run
        Returns:
            None

        """
        self._num_row_dq_rules += 1
        self._num_dq_rules += 1

    def set_num_agg_dq_rules(self, source_agg_enabled: bool = False, final_agg_enabled: bool = False) -> None:
        """
        This function sets number of applied agg dq rules for batch run
        source_agg_enabled: Marked True when agg rules set for source, by default False
        final_agg_enabled: Marked True when agg rules set for final, by default False
        Returns:
            None
        """

        self._num_agg_dq_rules["num_agg_dq_rules"] += 1
        self._num_dq_rules += 1

        if source_agg_enabled:
            self._num_agg_dq_rules["num_source_agg_dq_rules"] += 1
        if final_agg_enabled:
            self._num_agg_dq_rules["num_final_agg_dq_rules"] += 1

    def set_num_query_dq_rules(self, source_query_enabled: bool = False, final_query_enabled: bool = False) -> None:
        """
        This function sets number of applied query dq rules for batch run
        source_query_enabled: Marked True when query rules set for source, by default False
        final_query_enabled: Marked True when query rules set for final, by default False
        Returns:
            None
        """

        self._num_query_dq_rules["num_query_dq_rules"] += 1
        self._num_dq_rules += 1

        if source_query_enabled:
            self._num_query_dq_rules["num_source_query_dq_rules"] += 1
        if final_query_enabled:
            self._num_query_dq_rules["num_final_query_dq_rules"] += 1

    @property
    def get_num_row_dq_rules(self) -> int:
        """
        This function returns number row dq rules applied for batch run
        Returns:
            int: number of rules in int

        """

        if isinstance(self._num_row_dq_rules, int):
            return self._num_row_dq_rules
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_num_row_dq_rules'  before 
            accessing it"""
        )

    @property
    def get_num_agg_dq_rules(self) -> dict:
        """
        This function returns number agg dq rules applied for batch run
        Returns:
            int: number of rules in int

        """
        if isinstance(self._num_agg_dq_rules, dict):
            return self._num_agg_dq_rules
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_num_agg_dq_rules'  before 
            accessing it"""
        )

    @property
    def get_num_query_dq_rules(self) -> dict:
        """
        This function returns number query dq rules applied for batch run
        Returns:
            int: number of rules in int

        """
        if isinstance(self._num_query_dq_rules, dict):
            return self._num_query_dq_rules
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_num_query_dq_rules'  before 
            accessing it"""
        )

    @property
    def get_num_dq_rules(self) -> int:
        """
        This function returns number dq rules applied for batch run
        Returns:
            int: number of rules in int
        """
        if isinstance(self._num_dq_rules, int):
            return self._num_dq_rules
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign '_num_dq_rules'  before 
            accessing it"""
        )

    def set_summarized_row_dq_res(self, summarized_row_dq_res: Optional[List[Dict[str, str]]] = None) -> None:
        """
        This function implements or supports to set row dq summarized res
        Args:
            summarized_row_dq_res: list(dict)
        Returns: None

        """
        self._summarized_row_dq_res = summarized_row_dq_res

    @property
    def get_summarized_row_dq_res(self) -> Optional[List[Dict[str, str]]]:
        """
        This function returns row dq summarized res
        Returns:
            list(dict): Returns summarized_row_dq_res which in list of dict with str(key) and
            str(value) of rule meta data

        """
        return self._summarized_row_dq_res

    def set_rules_exceeds_threshold(self, rules: Optional[List[dict]] = None) -> None:
        """
        This function implements error percentage for each rule type
        """
        self._rules_error_per = rules

    @property
    def get_rules_exceeds_threshold(self) -> Optional[List[dict]]:
        """
        This function returns error percentage for each rule
        """
        return self._rules_error_per

    def set_target_and_error_table_writer_type(self, writer_type: str) -> None:
        """
        This function returns target and error table writer type
        Args:
            writer_type: str
        Returns:
            str: Returns target_and_error_table_writer_type which in str
        """
        self._target_and_error_table_writer_type = writer_type

    @property
    def get_target_and_error_table_writer_type(self) -> str:
        """
        This function returns target and error table writer type
        Returns:
            str: Returns target_and_error_table_writer_type which in str
        """
        return self._target_and_error_table_writer_type

    def set_stats_table_writer_type(self, writer_type: str) -> None:
        """
        This function returns stats table writer type
        Args:
            writer_type: str
        Returns:
            str: Returns stats_table_writer_type which in str
        """
        self._stats_table_writer_type = writer_type

    @property
    def get_stats_table_writer_type(self) -> str:
        """
        This function returns stats table writer type
        Returns:
            str: Returns stats_table_writer_type which in str
        """
        return self._stats_table_writer_type

    def set_target_and_error_table_writer_config(self, config: dict) -> None:
        """
        This function sets target and error table writer config
        Args:
            config: dict
        Returns: None

        """
        self._target_and_error_table_writer_config = config

    @property
    def get_target_and_error_table_writer_config(self) -> dict:
        """
        This function returns target and error table writer config
        Returns:
            dict: Returns target_and_error_table_writer_config which in dict

        """
        return self._target_and_error_table_writer_config

    def set_stats_table_writer_config(self, config: dict) -> None:
        """
        This function sets stats table writer config
        Args:
            config: dict
        Returns: None
        """
        self._stats_table_writer_config = config

    @property
    def get_stats_table_writer_config(self) -> dict:
        """
        This function returns stats table writer config
        Returns:
            dict: Returns stats_table_writer_config which in dict
        """
        return self._stats_table_writer_config

    def set_agg_dq_detailed_stats_status(self, agg_dq_detailed_result_status: bool) -> None:
        """
        Args:
            _enable_agg_dq_detailed_result:
        Returns:
        """
        self._enable_agg_dq_detailed_result = bool(agg_dq_detailed_result_status)

    @property
    def get_agg_dq_detailed_stats_status(self) -> bool:
        """
        This function returns whether to enable detailed result for Agg and Query dq is enabled or not
        Returns: Returns _enable_agg_dq_detailed_result(bool)
        """

        return self._enable_agg_dq_detailed_result

    def set_query_dq_detailed_stats_status(self, query_dq_detailed_result_status: bool) -> None:
        """
        Args:
            _enable_query_dq_detailed_result:
        Returns:
        """
        self._enable_query_dq_detailed_result = bool(query_dq_detailed_result_status)

    @property
    def get_query_dq_detailed_stats_status(self) -> bool:
        """
        This function returns whether to enable detailed result for Agg and Query dq is enabled or not
        Returns: Returns _enable_query_dq_detailed_result(bool)
        """

        return self._enable_query_dq_detailed_result

    def set_source_agg_dq_detailed_stats(self, source_agg_dq_detailed_stats: Optional[List[Tuple]] = None) -> None:
        """
        Args:
            _source_agg_dq_detailed_stats:
        Returns:
        """
        self._source_agg_dq_detailed_stats = source_agg_dq_detailed_stats

    @property
    def get_source_agg_dq_detailed_stats(self) -> Optional[List[Tuple]]:
        """
        This function returns the detailed result for Agg and Query dq
        Returns: Returns _source_agg_dq_detailed_stats
        """

        return self._source_agg_dq_detailed_stats

    def set_source_query_dq_detailed_stats(self, source_query_dq_detailed_stats: Optional[List[Tuple]] = None) -> None:
        """
        Args:
            _source_query_dq_detailed_stats:
        Returns:
        """
        self._source_query_dq_detailed_stats = source_query_dq_detailed_stats

    @property
    def get_source_query_dq_detailed_stats(self) -> Optional[List[Tuple]]:
        """
        This function returns the detailed result for Agg and Query dq
        Returns: Returns _source_query_dq_detailed_stats
        """

        return self._source_query_dq_detailed_stats

    def set_target_agg_dq_detailed_stats(self, target_agg_dq_detailed_stats: Optional[List[Tuple]] = None) -> None:
        """
        Args:
            _target_agg_dq_detailed_stats:
        Returns:
        """
        self._target_agg_dq_detailed_stats = target_agg_dq_detailed_stats

    @property
    def get_target_agg_dq_detailed_stats(self) -> Optional[List[Tuple]]:
        """
        This function returns the detailed result for Agg and Query dq
        Returns: Returns _target_agg_dq_detailed_stats
        """

        return self._target_agg_dq_detailed_stats

    def set_target_query_dq_detailed_stats(self, target_query_dq_detailed_stats: Optional[List[Tuple]] = None) -> None:
        """
        Args:
            _target_query_dq_detailed_stats:
        Returns:
        """
        self._target_query_dq_detailed_stats = target_query_dq_detailed_stats

    @property
    def get_target_query_dq_detailed_stats(self) -> Optional[List[Tuple]]:
        """
        This function returns the detailed result for Agg and Query dq
        Returns: Returns _target_query_dq_detailed_stats
        """

        return self._target_query_dq_detailed_stats

    def set_dq_detailed_stats_table_name(self, dq_detailed_stats_table_name: str) -> None:
        """
        Sets the table name for detailed DQ statistics.

        Args:
            dq_detailed_stats_table_name: Fully qualified table name for detailed stats
        """
        self._dq_detailed_stats_table_name = dq_detailed_stats_table_name

    @property
    def get_dq_detailed_stats_table_name(self) -> str:
        """
        Get dq_stats_table_name to which the final stats of the dq job will be written into

        Returns:
            str: returns the dq_stats_table_name
        """
        if (
            self.get_agg_dq_detailed_stats_status or self.get_query_dq_detailed_stats_status
        ) and self._dq_detailed_stats_table_name:
            return self._dq_detailed_stats_table_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign 
            '_dq_detailed_stats_table_name' before 
            accessing it"""
        )

    def set_query_dq_output_custom_table_name(self, query_dq_output_custom_table_name: str) -> None:
        """
        Sets the custom table name for query DQ output results.

        Args:
            query_dq_output_custom_table_name: Fully qualified table name for query DQ output
        """
        self._query_dq_output_custom_table_name = query_dq_output_custom_table_name

    @property
    def get_query_dq_output_custom_table_name(self) -> str:
        """
        Get query_dq_detailed_stats_status to which the final output of the query of the querydq  will be written into

        Returns:
            str: returns the query_dq_output_custom_table_name
        """
        if self.get_query_dq_detailed_stats_status and self._dq_detailed_stats_table_name:
            return self._query_dq_output_custom_table_name
        raise SparkExpectationsMiscException(
            """The spark expectations context is not set completely, please assign 
            '_dq_detailed_stats_table_name,query_dq_detailed_stats_status' before 
            accessing it"""
        )

    def set_detailed_stats_table_writer_config(self, config: dict) -> None:
        """
        This function sets stats table writer config
        Args:
            config: dict
        Returns: None
        """
        self._stats_table_writer_config = config

    @property
    def get_detailed_stats_table_writer_config(self) -> dict:
        """
        This function returns stats table writer config
        Returns:
            dict: Returns detailed_stats_table_writer_config which in dict
        """
        return self._stats_table_writer_config

    def set_rules_execution_settings_config(self, config: dict) -> None:
        """
        This function sets stats table writer config
        Args:
            config: dict
        Returns: None
        """
        self._rules_execution_settings_config = config

    @property
    def get_rules_execution_settings_config(self) -> dict:
        """
        This function returns stats table writer config
        Returns:
            dict: Returns detailed_stats_table_writer_config which in dict
        """
        return self._rules_execution_settings_config

    def set_querydq_secondary_queries(self, querydq_secondary_queries: dict) -> None:
        """
        This function sets row dq secondary queries
        Args:
            querydq_secondary_queries: dict
        Returns: None
        """
        self._querydq_secondary_queries = querydq_secondary_queries

    @property
    def get_querydq_secondary_queries(self) -> dict:
        """
        This function gets row dq secondary queries
        Returns:
            dict: Returns querydq_secondary_queries
        """
        return self._querydq_secondary_queries

    def set_source_query_dq_output(self, source_query_dq_output: Optional[List[dict]] = None) -> None:
        """
        This function sets row dq secondary queries
        Args:
            source_query_dq_output: List[dict]
        Returns: None
        """
        self._source_query_dq_output = source_query_dq_output

    @property
    def get_source_query_dq_output(self) -> Optional[List[dict]]:
        """
        This function gets row dq secondary queries
        Returns:
            dict: Returns source_query_dq_output
        """
        return self._source_query_dq_output

    def set_target_query_dq_output(self, target_query_dq_output: Optional[List[dict]] = None) -> None:
        """
        This function sets row dq secondary queries
        Args:
            target_query_dq_output: List[dict]
        Returns: None
        """
        self._target_query_dq_output = target_query_dq_output

    @property
    def get_target_query_dq_output(self) -> Optional[List[dict]]:
        """
        This function gets row dq secondary queries
        Returns:
            dict: Returns target_query_dq_output
        """
        return self._target_query_dq_output

    def set_se_enable_error_table(self, _enable_error_table: bool) -> None:
        """

        Args:
            _se_enable_error_table:

        Returns:

        """
        self._se_enable_error_table = _enable_error_table

    @property
    def get_se_enable_error_table(self) -> bool:
        """
        This function returns whether to enable relational table or not
        Returns: Returns _se_enable_error_table(bool)

        """
        return self._se_enable_error_table

    def set_dq_rules_params(self, _dq_rules_params: dict) -> None:
        """
        This function set params for dq rules
        Args:
            _se_dq_rules_params:

        Returns:

        """
        self._dq_rules_params = _dq_rules_params

    @property
    def get_dq_rules_params(self) -> dict:
        """
        This function returns params which are mapping in dq rules
        Returns: _dq_rules_params(dict)

        """
        return self._dq_rules_params

    def set_job_metadata(self, job_metadata: Optional[str] = None) -> None:
        """
        This function is used to set the job_metadata

        Returns:
            None

        """
        self._job_metadata = job_metadata

    @property
    def get_job_metadata(self) -> Optional[str]:
        """
        This function is used to get row data quality rule type name

        Returns:
             str: Returns _row_dq_rule_type_name"

        """
        if self._job_metadata is not None:
            return str(self._job_metadata)
        return None

    def set_stats_dict(self, df: DataFrame) -> None:
        """
        This function is used to set the stats_dict

        Returns:
            dictionary of statistics

        """
        self._stats_dict = [row.asDict() for row in df.collect()]

    @property
    def get_stats_dict(self) -> Optional[List[Dict[str, Any]]]:
        """
        This function is used to get the stats_dict

        Returns:
            Optional[List[Dict[str, Any]]]: Returns the stats_dict if it exists, otherwise None
        """
        return self._stats_dict if hasattr(self, "_stats_dict") else None

    def set_enable_obs_dq_report_result(self, enable_obs_dq_report_result: bool) -> None:
        """
        This function is used to set the enable_obs_dq_report_result

        Returns:
            None

        """
        self._enable_obs_dq_report_result = enable_obs_dq_report_result

    @property
    def get_enable_obs_dq_report_result(self) -> bool:
        """
        This function is used to get the enable_obs_dq_report_result

        Returns:
            bool: Returns the enable_obs_dq_report_result
        """
        return self._enable_obs_dq_report_result

    def set_se_dq_obs_alert_flag(self, se_dq_obs_alert_flag: bool) -> None:
        """
        This function is used to set the se_dq_obs_alert_flag

        Returns:
            None

        """
        self._se_dq_obs_alert_flag = se_dq_obs_alert_flag

    @property
    def get_se_dq_obs_alert_flag(self) -> bool:
        """
        This function is used to get the se_dq_obs_alert_flag

        Returns:
            bool: Returns the se_dq_obs_alert_flag
        """
        return self._se_dq_obs_alert_flag

    def set_detailed_default_template(self, detailed_default_template: str) -> None:
        """
        This function is used to set the detailed_default_template

        Returns:
            None

        """
        self._detailed_default_template = detailed_default_template

    @property
    def get_detailed_default_template(self) -> str:
        """
        This function is used to get the detailed_default_template

        Returns:
            str: Returns the default_template
        """
        return self._detailed_default_template

    def set_basic_default_template(self, basic_default_template: str) -> None:
        """
        This function is used to set the basic_default_template

        Returns:
            None

        """
        self._basic_default_template = basic_default_template

    @property
    def get_basic_default_template(self) -> str:
        """
        This function is used to get the basic_default_template

        Returns:
            str: Returns the default_template
        """
        return self._basic_default_template

    def set_custom_default_template(self, custom_default_template: str) -> None:
        """
        This function is used to set the custom_default_template

        Returns:
            None

        """
        self._custom_default_template = custom_default_template

    @property
    def get_custom_default_template(self) -> str:
        """
        This function is used to get the custom_default_template

        Returns:
            str: Returns the custom_template
        """
        return self._custom_default_template

    def set_stats_detailed_dataframe(self, dataframe: DataFrame) -> None:
        """
        Sets the DataFrame containing detailed statistics.

        Args:
            dataframe: DataFrame with detailed DQ statistics
        """
        self._dataframe = dataframe

    @property
    def get_stats_detailed_dataframe(self) -> DataFrame:
        """
        Returns the DataFrame containing detailed statistics.

        Returns:
            DataFrame: DataFrame with detailed DQ statistics
        """
        return self._dataframe

    def set_custom_detailed_dataframe(self, dataframe: DataFrame) -> None:
        """
        Sets the DataFrame containing custom detailed statistics.

        Args:
            dataframe: DataFrame with custom detailed DQ statistics
        """
        self._custom_dataframe = dataframe

    @property
    def get_custom_detailed_dataframe(self) -> DataFrame:
        """
        Returns the DataFrame containing custom detailed statistics.

        Returns:
            DataFrame: DataFrame with custom detailed DQ statistics
        """
        return self._custom_dataframe

    def set_report_table_name(self, report_table_name: str) -> None:
        """
        Sets the report table name for DQ observability reports.

        Args:
            report_table_name: Fully qualified table name for DQ reports
        """
        self._report_table_name = report_table_name

    @property
    def get_report_table_name(self) -> str:
        """
        Returns the report table name for DQ observability reports.

        Returns:
            str: Fully qualified table name for DQ reports
        """
        return self._report_table_name

    def set_dq_obs_rpt_gen_status_flag(self, dq_obs_rpt_gen_status_flag: bool) -> None:
        """
        This function is used to set the dq_obs_rpt_gen_status_flag

        Returns:
            None

        """
        self._dq_obs_rpt_gen_status_flag = dq_obs_rpt_gen_status_flag

    @property
    def get_dq_obs_rpt_gen_status_flag(self) -> bool:
        """
        This function is used to get the dq_obs_rpt_gen_status_flag

        Returns:
            bool: Returns the dq_obs_rpt_gen_status_flag
        """
        return self._dq_obs_rpt_gen_status_flag

    def set_df_dq_obs_report_dataframe(self, dataframe: DataFrame) -> None:
        """
        Sets the DataFrame for DQ observability report.

        Args:
            dataframe: DataFrame containing DQ observability report data
        """
        self._df_dq_obs_report_dataframe = dataframe

    @property
    def get_df_dq_obs_report_dataframe(self) -> DataFrame:
        """
        Returns the DataFrame for DQ observability report.

        Returns:
            DataFrame: DataFrame containing DQ observability report data
        """
        return self._df_dq_obs_report_dataframe
