"""
Contains helper functions when working with sklearn (scikit-learn) objects in particular, for
evaluating models.
"""

from __future__ import annotations
import math
import warnings
from re import match
import re

import numpy as np
import pandas as pd
import plotly
import scipy.stats as st
import seaborn as sns
from plotly.graph_objs import _figure
import plotly.express as px
import yaml
from matplotlib import pyplot as plt
from pandas.io.formats.style import Styler
from sklearn.dummy import DummyClassifier, DummyRegressor
from sklearn.metrics import confusion_matrix, roc_auc_score, r2_score, average_precision_score
from sklearn.model_selection._search import BaseSearchCV
from sklearn.preprocessing import MinMaxScaler

import helpsk.color as hcolor
import helpsk.pandas_style as hstyle
import helpsk.string as hstring
from helpsk.pandas import get_numeric_columns, get_non_numeric_columns
from helpsk.exceptions import HelpskParamValueError
from helpsk.plot import STANDARD_WIDTH_HEIGHT, GOLDEN_RATIO

with warnings.catch_warnings():
    warnings.filterwarnings("ignore", category=Warning)
    from statsmodels import api as sm  # https://github.com/statsmodels/statsmodels/issues/3814


class MLExperimentResults:
    """
    Contains the logic to explore the results of a machine learning experiment. There
    are also functions that parse the information from a BaseSearchCV object (e.g. GridSearchCV,
    RandomizedSearchCV, BayesSearchCV).

    The results can be saved/loaded to/from a standardized YAML file via
    `to_yaml_file()`/`from_yaml_file()`

    A MLExperimentResults object can be instantiated via `from_sklearn_searchCV()`, `__init__()`,
    or `from_yaml_file()`. The `__init__()` function allows people to transform experiment results
    from other sources into a dictionary (see __init__ function documentation for details on
    expected format) so that it can be used with this class.

    Trial: A single training iteration on a specific set of hyper-parameters (e.g. a given set of
    model parameters, transformations, etc.)

    Experiment: a collection of trials. An experiment can be a single run using, for example,
    GridSearchCV or BayesSearchCV. If a GridSearchCV experiment has 30 combinations of
    hyper-parameters to try, each of those combinations is a Trial. The collection of 30
    combinations is the experiment.
    """

    def __init__(self, info: dict):
        """
        Creates a MLExperimentResults object from a dictionary.

        Args:
            info:
                a dictionary with the experiment information and format, as an example:

                - `description`: any text to provide context into the ML experiment
                - `cross_validation_type`: any text
                - `higher_score_is_better`: boolean indicating whether or not a higher score/metric
                    is better e.g. True for AUC, False for RMSE
                - `number_of_splits`: This is the number of times the model is trained in a single
                    trial i.e. single cross-validation session. For example, a 5-fold 2-repeat CV
                    has 10 splits.
                - `score_names`: the names of the score(s)
                - `parameter_names`: the names of the parameters
                - `parameter_names_mapping`: a mapping between actual parameters and friendlier
                    names
                - `test_score_averages`: for each score, the average score value among all splits
                    (i.e. across the cross-validation run) for a single trial
                - `test_score_standard_deviations`: for each score, the standard deviation score
                    among all splits (i.e. across the cross-validation run) for a single trial
                - `train_score_averages`: (optional) same as test_score_xxx above, but for training
                    scores
                - `train_score_standard_deviations`: (optional) same as test_score_xxx above, but
                    for training standard deviations
                - `trials`: A list of dictionary. Each item in the list (i.e. the dictionary)
                    corresponds to a single trial and a single set of hyper-parameters. The
                    dictionary has an item for each of the hyper-parameters, the name of the
                    hyper-parameter as the key and the corresponding value
                - `timings`: (optional) A list of timings for fitting/scoring

            ```
            {
                'description': 'test description',
                'cross_validation_type': "<class 'sklearn.model_selection._search.GridSearchCV'>",
                'higher_score_is_better': True,
                'number_of_splits': 3,
                'score_names': ['ROC/AUC', 'F1', 'Pos. Pred. Val', 'True Pos. Rate'],
                'parameter_names': [
                    'model__max_features', 'model__n_estimators',
                    'preparation__non_numeric_pipeline__encoder_chooser__transformer'
                ],
                'parameter_names_mapping': {
                    'model__max_features': 'max_features',
                    'model__n_estimators': 'n_estimators',
                    'preparation__non_numeric_pipeline__encoder_chooser__transformer': 'encoder'
                },
                'test_score_averages': {
                    'ROC/AUC': [
                        nan, nan, nan, nan, 0.7163279567387703, 0.7072491564040325,
                        0.7458632917328131, 0.7451951355826116
                    ],
                    'F1': [
                        nan, nan, nan, nan, 0.4039887486499992, 0.3854065126792399,
                        0.4137381215646683, 0.4308356053261986
                    ],
                    'Pos. Pred. Val': [
                        nan, nan, nan, nan, 0.6297753300113645, 0.5468751302897644,
                        0.6015296073435609, 0.6113328664799252
                    ],
                    'True Pos. Rate': [
                        nan, nan, nan, nan, 0.2993876710919008, 0.29902233199547185,
                        0.31574945970978696, 0.3363795410106
                    ]
                },
                'test_score_standard_deviations': {
                    'ROC/AUC': [
                        nan, nan, nan, nan, 0.015733457430657828, 0.01795608557324064,
                        0.021358294698372086, 0.0019082263491945234
                    ],
                    'F1': [
                        nan, nan, nan, nan, 0.05090125909373751, 0.07559891495722325,
                        0.04259039056830544, 0.041365868651721927
                    ],
                    'Pos. Pred. Val': [
                        nan, nan, nan, nan, 0.026117735279684196, 0.06663645906293561,
                        0.041309705841820844, 0.06827042947757211
                    ],
                    'True Pos. Rate': [
                        nan, nan, nan, nan, 0.0497780260716827, 0.07150204558331273,
                        0.038463218144256496, 0.0460214315468713
                    ]
                },
                'train_score_averages': {
                    'ROC/AUC': [
                        nan, nan, nan, nan, 0.9998943476077663, 0.9992031557521965, 1.0, 1.0
                    ],
                    'F1': [nan, nan, nan, nan, 0.9765745804947125, 0.9658091912335657, 1.0, 1.0],
                    'Pos. Pred. Val': [nan, nan, nan, nan, 1.0, 0.9956132756132755, 1.0, 1.0],
                    'True Pos. Rate': [
                        nan, nan, nan, nan, 0.9543319834542148, 0.9377688364562852, 1.0, 1.0
                    ]
                },
                'train_score_standard_deviations': {
                    'ROC/AUC': [
                        nan, nan, nan, nan, 7.439517969409452e-05, 0.0005401862948682569,
                        6.409875621278546e-17, 0.0
                    ],
                    'F1': [
                        nan, nan, nan, nan, 0.007673191451896241, 0.005478380082149172, 0.0, 0.0
                    ],
                    'Pos. Pred. Val': [nan, nan, nan, nan, 0.0, 0.0031026880007722375, 0.0, 0.0],
                    'True Pos. Rate': [
                        nan, nan, nan, nan, 0.014730511057304973, 0.00871005201368303, 0.0, 0.0
                    ]
                },
                'trials': [
                    {
                        'model__max_features': 100,
                        'model__n_estimators': 10,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'OneHotEncoder()'
                    },
                    {
                        'model__max_features': 100,
                        'model__n_estimators': 10,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'CustomOrdinalEncoder()'
                    },
                    {
                        'model__max_features': 100,
                        'model__n_estimators': 50,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'OneHotEncoder()'
                    },
                    {
                        'model__max_features': 100,
                        'model__n_estimators': 50,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'CustomOrdinalEncoder()'
                    },
                    {
                        'model__max_features': 'sqrt',
                        'model__n_estimators': 10,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'OneHotEncoder()'
                    },
                    {
                        'model__max_features': 'sqrt',
                        'model__n_estimators': 10,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'CustomOrdinalEncoder()'
                    },
                    {
                        'model__max_features': 'sqrt',
                        'model__n_estimators': 50,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'OneHotEncoder()'
                    },
                    {
                        'model__max_features': 'sqrt',
                        'model__n_estimators': 50,
                        'preparation__non_numeric_pipeline__encoder_chooser__transformer':
                            'CustomOrdinalEncoder()'
                    }
                ],
                'timings': {
                    'fit time averages': [
                        0.02420210838317871, 0.047980546951293945, 0.053688764572143555,
                        0.07731231053670247, 0.03327099482218424, 0.05605125427246094,
                        0.09772396087646484, 0.12112665176391602
                    ],
                    'fit time standard deviations': [
                        0.0015644331364039865, 0.0007074507470172489,
                        0.00019471963214876084, 0.0006047002838356386,
                        0.0014654790113907423, 0.000618793945947827,
                        0.00044132423197131406, 0.0007073387039515561
                    ],
                    'score time averages': [
                        0.0, 0.0, 0.0, 0.0, 0.020992437998453777, 0.05231904983520508,
                        0.03004598617553711, 0.06252868970235188
                    ],
                    'score time standard deviations': [
                        0.0, 0.0, 0.0, 0.0, 0.0003703884785102133,
                        0.0004282967971518099, 0.00014070830842031841,
                        0.0004855695604911932
                    ]
                }
            }
            ```
        """
        self._dict = info
        self._dataframe = None
        self._labeled_dataframe = None

    @classmethod
    def from_sklearn_search_cv(  # noqa: PLR0912, PLR0915
            cls,
            searcher: BaseSearchCV,
            higher_score_is_better: bool = True,
            description: str = "",
            parameter_name_mappings: dict | None = None) -> 'MLExperimentResults':  # noqa: UP037
        """
        Extracts the results from a SearchCV object (e.g.
        sklearn.model_selection.GridSearch/RandomSearch, skopt.BayesSearchCV), which are converted
        to a dictionary with the hierarchy expected by the MLExperimentResults class.

        Args:
            searcher:
                A `BaseSearchCV` object that has either used a string passed to the `scoring`
                parameter of the constructor (e.g. `GridSearchCV(..., scoring='auc', ...)` or a
                dictionary with metric names as keys and callables as values.

                An example of the dictionary option:

                    scores = {
                        'ROC/AUC': SCORERS['roc_auc'],
                        'F1': make_scorer(f1_score, greater_is_better=True),
                        'Pos. Pred. Val': make_scorer(precision_score, greater_is_better=True),
                        'True Pos. Rate': make_scorer(recall_score, greater_is_better=True),
                    }
                    grid_search = GridSearchCV(..., scoring=scores, ...)

            higher_score_is_better:
                If True, higher scores are better; if False, lower scores are better.
                A value of False assumes that the scores returned from sklearn are negative and
                will multiple the values by -1.

            description:
                An optional string to save in the dictionary; the intent

            parameter_name_mappings:
                A dictionary containing the parameter names returned by the searchCV object as keys
                (which should correspond to the path of the pipeline(s) corresponding to the
                parameter) and the new, friendlier, names that can be displayed in graphs and
                tables.

                For example:

                    {
                        'model__max_features': 'max_features',
                        'model__n_estimators': 'n_estimators',
                        'prep__non_numeric__encoder__transformer': 'encoder',
                        'prep__numeric__impute__transformer': 'imputer',
                        'prep__numeric__scaling__transformer': 'scaler'
                    }
        """
        def string_if_not_number(obj):  # noqa: ANN202, ANN001
            if isinstance(obj, (int, float, complex)):
                return obj
            # convert to a string, but convert e.g. `XGBoostClassifier(parameter=x, etc)` to
            # `XGBoostClassifier()`
            string_value = str(obj)
            string_value = string_value.replace('\n', '')
            string_value = re.sub(r'Classifier\(.+\)', 'Classifier()', string_value)
            string_value = re.sub(r'Regressor\(.+\)', 'Regressor()', string_value)
            string_value = re.sub(r'Regression\(.+\)', 'Regression()', string_value)
            string_value = re.sub(r'LinearSVC\(.+\)', 'LinearSVC()', string_value)

            string_value = string_value.replace("PCA(n_components='mle')", "PCA('mle')")
            return string_value.\
                replace("OneHotEncoder(handle_unknown='ignore')", "OneHotEncoder()")

        cv_results_dict = {
            'description': description,
            'cross_validation_type': str(type(searcher)),
            'higher_score_is_better': higher_score_is_better,
        }

        if isinstance(searcher.scoring, dict):
            score_names = list(searcher.scoring.keys())
        elif isinstance(searcher.scoring, str):
            score_names = [searcher.scoring]
        else:
            message = 'The `searcher` does not have a string or dictionary .scoring property. ' \
                      'Cannot extract scores.'
            raise HelpskParamValueError(message)

        # get number of splits (e.g. 5 fold 2 repeat cross validation has 10 splits)
        # I could check the .cv param of the searcher object but not sure all types of cv objects
        # have the same parameters e.g. searcher.cv.n_repeats
        # if there is only 1 score, we need to look for e.g. "split0_test_score"
        # if there are multiple scores we need to look for e.g. "split0_test_ROC/AUC" but we don't
        # want to duplicate the counts e.g. we don't want to also capture
        # "split0_test_True Pos. Rate"
        if len(score_names) == 1:
            match_value = "split\\d_test_score"
        else:
            match_value = "split\\d_test_" + score_names[0]

        number_of_splits = len(
            [x for x in searcher.cv_results_ if bool(match(match_value, x))],
        )
        cv_results_dict['number_of_splits'] = number_of_splits
        cv_results_dict['score_names'] = score_names

        parameter_names = []
        # If there are multiple search spaces, then there might be different, but overlapping,
        # parameters in each search space. We have to collect all of the parameters and then get
        # the unique list (via `set()`).
        # Note that i'm doing this with nested loops (rather than simply using `set()`) so that I
        # can retain the exact order (set() does not retain order)
        for params in searcher.cv_results_['params']:
            for key in params:
                if key not in parameter_names:
                    parameter_names += [key]

        # if we pass in parameter mappings, make sure each parameter is accounted for i.e. the keys
        # of the mappings should be identical (in any order, (via `set()`)) to the parameter names
        if parameter_name_mappings:
            param_mapping_keys = list(parameter_name_mappings.keys())
            assert set(param_mapping_keys) == set(parameter_names)  # ensure equal, unordered
            # use the keys from the mappings as parameter names (rather than just passing param
            # names directly) so that the order of the param names is retained; which will be used
            # to determine order in e.g. to_data_frame()
            cv_results_dict['parameter_names'] = param_mapping_keys
            cv_results_dict['parameter_names_mapping'] = parameter_name_mappings
        else:
            cv_results_dict['parameter_names'] = parameter_names

        number_of_trials = len(searcher.cv_results_['mean_fit_time'])

        # convert test scores to dictionaries
        if len(score_names) == 1:
            test_score_averages = searcher.cv_results_['mean_test_score'].tolist()
            test_score_standard_deviations = searcher.cv_results_['std_test_score'].tolist()
            assert len(test_score_averages) == number_of_trials
            assert len(test_score_standard_deviations) == number_of_trials
            cv_results_dict['test_score_averages'] = {score_names[0]: test_score_averages}
            cv_results_dict['test_score_standard_deviations'] = {
                score_names[0]: test_score_standard_deviations,
            }
        else:
            averages_dict = {}
            standard_deviations_dict = {}
            for score in score_names:
                averages = searcher.cv_results_['mean_test_' + score].tolist()
                standard_deviations = searcher.cv_results_['std_test_' + score].tolist()
                assert len(averages) == number_of_trials
                assert len(standard_deviations) == number_of_trials
                averages_dict[score] = averages
                standard_deviations_dict[score] = standard_deviations

            cv_results_dict['test_score_averages'] = averages_dict
            cv_results_dict['test_score_standard_deviations'] = standard_deviations_dict

        # if higher_score_is_better is False, sklearn will return negative numbers; I want actual
        # values
        if not higher_score_is_better:
            averages = cv_results_dict['test_score_averages']
            for key in averages:
                cv_results_dict['test_score_averages'][key] = [-1 * x for x in averages[key]]

        # convert training scores to dictionaries, if training scores exists
        # i.e. if return_train_score=True for the SearchCV object
        if ('mean_train_score' in searcher.cv_results_) \
                or ('mean_train_' + score_names[0] in searcher.cv_results_):
            if len(score_names) == 1:
                train_score_averages = searcher.cv_results_['mean_train_score'].tolist()
                train_score_standard_deviations = searcher.cv_results_['std_train_score'].tolist()

                assert len(train_score_averages) == number_of_trials
                assert len(train_score_standard_deviations) == number_of_trials

                cv_results_dict['train_score_averages'] = {score_names[0]: train_score_averages}
                cv_results_dict['train_score_standard_deviations'] = {
                    score_names[0]: train_score_standard_deviations,
                }
            else:
                averages_dict = {}
                standard_deviations_dict = {}
                for score in score_names:
                    averages = searcher.cv_results_['mean_train_' + score].tolist()
                    standard_deviations = searcher.cv_results_['std_train_' + score].tolist()

                    assert len(averages) == number_of_trials
                    assert len(standard_deviations) == number_of_trials

                    averages_dict[score] = averages
                    standard_deviations_dict[score] = standard_deviations

                cv_results_dict['train_score_averages'] = averages_dict
                cv_results_dict['train_score_standard_deviations'] = standard_deviations_dict

            # if higher_score_is_better is False, sklearn will return negative numbers; I want
            # actual values
            if not higher_score_is_better:
                averages = cv_results_dict['train_score_averages']
                for key in averages:
                    cv_results_dict['train_score_averages'][key] = [-1 * x for x in averages[key]]

        assert len(searcher.cv_results_['params']) == number_of_trials

        cv_results_dict['trials'] = [
            {
                key: string_if_not_number(value)
                for key, value in searcher.cv_results_['params'][index].items()
            }
            for index in range(len(searcher.cv_results_['params']))
        ]

        fit_time_averages = searcher.cv_results_['mean_fit_time'].tolist()
        fit_time_standard_deviations = searcher.cv_results_['std_fit_time'].tolist()
        score_time_averages = searcher.cv_results_['mean_score_time'].tolist()
        score_time_standard_deviations = searcher.cv_results_['std_score_time'].tolist()

        assert len(fit_time_averages) == number_of_trials
        assert len(fit_time_standard_deviations) == number_of_trials
        assert len(score_time_averages) == number_of_trials
        assert len(score_time_standard_deviations) == number_of_trials

        cv_results_dict['timings'] = {
            'fit time averages': fit_time_averages,
            'fit time standard deviations': fit_time_standard_deviations,
            'score time averages': score_time_averages,
            'score time standard deviations': score_time_standard_deviations,
        }

        return MLExperimentResults(info=cv_results_dict)

    @classmethod
    def from_yaml_file(cls, yaml_file_name: str) -> 'MLExperimentResults':  # noqa: UP037
        """Creates a MLExperimentResults object from a yaml file created by `to_yaml_file()`."""
        with open(yaml_file_name) as file:
            cv_dict = yaml.safe_load(file)

        return MLExperimentResults(info=cv_dict)

    def to_yaml_file(self, yaml_file_name: str) -> None:
        """Saves the self._cv_dict dictionary to a yaml file."""
        with open(yaml_file_name, 'w') as file:
            yaml.dump(self._dict, file, default_flow_style=False, sort_keys=False)

    def to_dataframe(
            self,
            sort_by_score: bool = True,
            exclude_zero_variance_params: bool = True,
            query: str | None = None) -> pd.DataFrame:
        """
        Converts the score information from the SearchCV object into a pd.DataFrame.

        Args:
            sort_by_score:
                if True, sorts the dataframe starting with the best (primary) score to the worst
                score. Secondary scores are not considered. If False, leave the results in the
                order that the trials were executed.

            exclude_zero_variance_params:
                if True, exclude columns that only have 1 unique value

            query:
                a string that queries the resulting pd.DataFrame (passed directly to pandas
                `.query()`)

                For example, if multiple models are being searched, and `model` is a parameter name
                (and a resulting column), then a query value of `"model == 'LogisticRegression()'"`
                would return only the rows where the value of the `model` column matches
                `LogisticRegression()`.

        Returns:
            a DataFrame containing score information for each cross-validation trial. A single row
            corresponds to one trial (i.e. one set of hyper-parameters that were cross-validated).
        """
        if self._dataframe is None:
            for score_name in self.score_names:
                confidence_intervals = st.t.interval(
                    confidence=0.95,  # confidence interval
                    # number_of_splits is sample-size
                    df=self.number_of_splits - 1,  # degrees of freedom
                    loc=self.test_score_averages[score_name],
                    scale=self.score_standard_errors(score_name=score_name),
                )

                # only give confidence intervals for the primary score
                self._dataframe = pd.concat(
                    [
                        self._dataframe,
                        pd.DataFrame({
                            score_name + " Mean": self.test_score_averages[score_name],
                            score_name + " 95CI.LO": confidence_intervals[0],
                            score_name + " 95CI.HI": confidence_intervals[1],
                        }),
                    ],
                    axis=1,
                )

            self._dataframe = pd.concat(
                [
                    self._dataframe,
                    pd.DataFrame.from_dict(self.trials)[self.parameter_names_original],
                ],
                axis=1,
            )

            if self.parameter_names_mapping:
                self._dataframe = self._dataframe.rename(columns=self.parameter_names_mapping)

        copy = self._dataframe.copy(deep=True)

        if sort_by_score:
            copy = copy.iloc[self.best_trial_indexes]

        # need to query after we sort so that e.g. best_trial_indexes correspond to same rows
        if query:
            copy = copy.query(query)

        # need to do this after querying because the it's possible a columns could become
        # zero-variance after the query
        if exclude_zero_variance_params:
            zero_variance_columns = [x for x in self.parameter_names if len(copy[x].unique()) == 1]
            copy = copy.drop(columns=zero_variance_columns)

        return copy.dropna(axis=1, how='all')  # drop columns that have all NAs

    def to_formatted_dataframe(
            self,
            round_by: int = 3,
            num_rows: int = 500,
            primary_score_only: bool = False,
            exclude_zero_variance_params: bool = True,
            query: str | None = None,
            include_rank: bool = False,
            return_style: bool = True,
            sort_by_score: bool = True) -> pd.DataFrame | Styler:
        """
        Converts the score information from the SearchCV object into a
        pd.DataFrame or a Styler object, formatted accordingly.

        The Hyper-Parameter columns will be highlighted in blue where the primary
        score (i.e. first column) for the trial (i.e. the row i.e. the combination of
        hyper-parameters that were cross validated) is within 1 standard error of the top primary
        score (i.e. first column first row).

        Args:
            round_by:
                the number of digits to round by for the score columns (does not round the
                parameter columns)
            num_rows:
                the number of rows to return in the resulting DataFrame.
            primary_score_only:
                if True, then only include the primary score.
            exclude_zero_variance_params:
                if True, exclude columns that only have 1 unique value
            query:
                a string that queries the resulting pd.DataFrame (passed directly to pandas
                `.query()`)

                For example, if multiple models are being searched, and `model` is a parameter
                name (and a resulting column), then a query value of
                `"model == 'LogisticRegression()'"` would return
                only the rows where the value of the `model` column matches `LogisticRegression()`.
            include_rank:
                if True, include a column to show the index of score rank.
            return_style:
                If True, return Styler object, else return pd.DataFrame
            sort_by_score:
                if True, sorts the dataframe starting with the best (primary) score to the worst
                score. Secondary scores are not considered.

        Returns:
            Returns either pd.DataFrame or pd.DataFrame.Styler.
        """
        cv_dataframe = self.to_dataframe(
            sort_by_score=sort_by_score,
            exclude_zero_variance_params=exclude_zero_variance_params,
            query=query,
        )
        cv_dataframe = cv_dataframe.head(num_rows)

        # if, for example, we are querying (and returning the style, i.e. highlighting rows within
        # 1 standard error), then we need to only include the indexes that remain after querying
        indexes_within_1_standard_error = [
            x for x in self.indexes_within_1_standard_error if x in cv_dataframe.index
        ]

        score_columns = list(
            cv_dataframe.columns[
                cv_dataframe.columns.str.endswith((' Mean', ' 95CI.LO', ' 95CI.HI'))
            ],
        )
        if primary_score_only:
            columns_to_drop = [
                x for x in score_columns if not x.startswith(self.primary_score_name)
            ]
            cv_dataframe = cv_dataframe.drop(columns=columns_to_drop)

        cv_dataframe = cv_dataframe.\
            round(dict(zip(score_columns, [round_by] * len(score_columns))))

        final_columns = list(cv_dataframe.columns)  # save for style logic

        if include_rank:
            cv_dataframe['rank'] = list(range(1, cv_dataframe.shape[0] + 1))
            final_columns = ["rank", *final_columns]
            cv_dataframe = cv_dataframe[final_columns]

        if return_style:
            original_data = cv_dataframe
            cv_dataframe = cv_dataframe.style

            for score in self.score_names:
                mean_key = score + ' Mean'
                ci_low_key = score + ' 95CI.LO'
                ci_high_key = score + ' 95CI.HI'

                if self.higher_score_is_better:
                    mean_color = hcolor.Colors.PIGMENT_GREEN.value
                else:
                    mean_color = hcolor.Colors.CADMIUM_ORANGE.value

                if mean_key in final_columns:
                    cv_dataframe.bar(
                        subset=[mean_key],
                        color=mean_color,
                        vmin=original_data[mean_key].min(),
                    )

                if ci_low_key in final_columns:
                    cv_dataframe. \
                        bar(
                            subset=[ci_high_key],
                            color=hcolor.GRAY,
                            vmin=original_data[ci_high_key].min(),
                        ). \
                        pipe(
                            hstyle.bar_inverse,
                            subset=[ci_low_key],
                            color=hcolor.GRAY,
                            min_value=original_data[ci_low_key].min(),
                        )

                cv_dataframe.pipe(hstyle.format, round_by=round_by, hide_index=True)

            # highlight trials whose primary score (i.e. first column of `results` dataframe) is
            # within 1 standard error of the top primary score (i.e. first column first row).

            def highlight_cols(s):   # noqa
                return f'background-color: {hcolor.Colors.PASTEL_BLUE.value}'

            # we might have removed columns (e.g. that don't have any variance) so check that the
            # columns were in the final set
            columns_to_highlight = [x for x in self.parameter_names if x in final_columns]
            cv_dataframe.map(
                highlight_cols,
                subset=pd.IndexSlice[indexes_within_1_standard_error, columns_to_highlight],
            )

        return cv_dataframe

    def to_labeled_dataframe(self, query: str | None = None) -> pd.DataFrame:
        """
        Returns a pd.DataFrame similar to `to_dataframe()` with additional columns 'Trial Index'
        and 'labels' (which are the labels corresponding to the `trial_label` property and collapse
        all the name/values for the hyper-parameters into a single string).

        Is mainly used internally to generate graphs, but is useful for users
        creating custom graphs that are not yet implemented by the class.

        Args:
            query:
                a string that queries the resulting pd.DataFrame (passed directly to pandas
                `.query()`)

                For example, if multiple models are being searched, and `model` is a parameter name
                (and a resulting column), then a query value of `"model == 'LogisticRegression()'"`
                would return only the rows where the value of the `model` column matches
                `LogisticRegression()`.
        """
        sort_by_score = False  # leave original trial order
        labeled_dataframe = self.to_dataframe(sort_by_score=sort_by_score, query=query)
        # cache columns to move Iteration column to front
        columns = labeled_dataframe.columns.to_list()
        labeled_dataframe['Trial Index'] = np.arange(1, labeled_dataframe.shape[0] + 1)
        labeled_dataframe = labeled_dataframe[["Trial Index", *columns]]
        # create the labels that will be used in the plotly hover text
        # only include the labels that correspond to the remaining trials after `query`
        trial_labels = self.trial_labels(order_from_best_to_worst=sort_by_score)
        trial_labels = [trial_labels[x] for x in labeled_dataframe.index]
        labeled_dataframe['label'] = [
            x.replace('{', '<br>').replace(', ', '<br>').replace('}', '') for x in trial_labels
        ]

        return labeled_dataframe

    ####
    # The following properties expose the highest levels of the underlying dictionary/yaml
    ####
    @property
    def description(self) -> str:
        """The description passed to `description`."""
        return self._dict['description']

    @property
    def higher_score_is_better(self) -> bool:
        """The value passed to `higher_score_is_better`."""
        return self._dict['higher_score_is_better']

    @property
    def cross_validation_type(self) -> str:
        """The string representation of the SearchCV object."""
        return self._dict['cross_validation_type']

    @property
    def number_of_splits(self) -> int:
        """
        The number of times the model is trained in a single trial i.e. single cross-validation
        session. For example, a 5-fold 2-repeat CV has 10 splits.
        """
        return self._dict['number_of_splits']

    @property
    def score_names(self) -> list:
        """Returns a list of the names of the scores."""
        return self._dict['score_names']

    @property
    def parameter_names_original(self) -> list[str]:
        """
        Returns the original parameter names (i.e. the path generated by the scikit-learn
        pipelines.
        """
        return self._dict['parameter_names']

    @property
    def parameter_names(self) -> list[str]:
        """
        Returns either the original parameter names if no
        `parameter_names_mapping` was provided, or it returns the new parameter names (i.e. the
        values from `parameter_names_mapping`).
        """
        if self.parameter_names_mapping:
            return list(self.parameter_names_mapping.values())

        return self.parameter_names_original

    @property
    def parameter_names_mapping(self) -> dict:
        """
        The dictionary passed to `parameter_name_mappings` which is used to convert the original
        names to more friendly names, specified as the the values.
        """
        return self._dict.get('parameter_names_mapping')

    @property
    def test_score_rankings(self) -> dict:
        """
        The rankings of each of the test scores. See `trial_rankings` documentation for
        explanation.
        """

        def get_rankings(score_averages: list) -> list:
            multiplier = -1 if self.higher_score_is_better else 1
            return list(st.rankdata(np.array(score_averages) * multiplier).astype(int))

        return {
            score: get_rankings(averages)
            for score, averages in self._dict['test_score_averages'].items()
        }

    @property
    def test_score_averages(self) -> dict:
        """The test score averages, from the searcher.cv_results_ object."""
        return self._dict['test_score_averages']

    @property
    def test_score_standard_deviations(self) -> dict:
        """The test score standard deviations, from the searcher.cv_results_ object."""
        return self._dict['test_score_standard_deviations']

    @property
    def train_score_averages(self) -> dict:
        """The training score averages, from the searcher.cv_results_ object, if provided."""
        return self._dict.get('train_score_averages')

    @property
    def train_score_standard_deviations(self) -> dict:
        """
        The training score standard deviations, from the searcher.cv_results_ object, if
        provided.
        """
        return self._dict.get('train_score_standard_deviations')

    @property
    def trials(self) -> list:
        """The "trials" i.e. the hyper-parameter combinations in order of execution."""
        return self._dict['trials']

    def trial_labels(self, order_from_best_to_worst: bool = True) -> list[str]:
        """
        An trial is a set of hyper-parameters that were cross validated. The corresponding label
        for each trial is a single string containing all of the hyper-parameter names and values in
        the format of `{param1: value1, param2: value2}`, excluding hyper-parameters that only have
        a single value.

        Args:
            order_from_best_to_worst: if True, returns the labels in order from the best score to
            the worst score, which should match the ordered of .to_dataframe() or
            .to_formatted_dataframe()` using the default values for those functions. If False,
            returns the labels in order that they were ran by the cross validation object.

        Returns:
            a pd.Series the same length as `number_of_trials` containing a str
        """
        def create_hyper_param_labels(trial) -> list:  # noqa: ANN001
            """
            Creates a list of strings that represent the name/value pair for each
            hyper-parameter.
            """
            return [
                f"{self.parameter_names_mapping[x] if self.parameter_names_mapping and x in self.parameter_names_mapping else x}: {trial[x]}"  # noqa
                # for parameter spaces that have multiple models (and different parameters per
                # model), we need to make sure that the parameter name is actually in the trial
                # e.g. the parameter name could correspond to the logistic regression space but we
                # could be iterating over the xgboost space
                for x in self.parameter_names_original if x in trial
            ]
        # create_hyper_param_labels(trial=self.trials[0])

        def create_trial_label(trial) -> str:  # noqa: ANN001
            return f"{{{hstring.collapse(create_hyper_param_labels(trial), separate=', ')}}}"
        # create_trial_label(trial=self.trials[0])

        labels = [create_trial_label(x) for x in self.trials]

        if order_from_best_to_worst:
            labels = [x for _, x in sorted(zip(self.trial_rankings, labels))]

        return labels

    @property
    def timings(self) -> dict:
        """The timings providing by searcher.cv_results_."""
        return self._dict['timings']

    ####
    # The following properties are additional helpers
    ####
    @property
    def number_of_trials(self) -> int:
        """
        "A single trial contains the cross validation results for a single set of
        hyper-parameters. The 'number of trials' is basically the number of combinations of
        different hyper-parameters that were cross validated.
        """
        return len(self.trials)

    @property
    def numeric_parameters(self) -> list[str]:
        """Returns a list of parameters names corresponding to numeric columns."""
        return [
            x for x in get_numeric_columns(dataframe=self.to_dataframe())
            if x in self.parameter_names
        ]

    @property
    def non_numeric_parameters(self) -> list[str]:
        """Returns a list of parameters names corresponding to non-numeric columns."""
        return [x for x in get_non_numeric_columns(dataframe=self.to_dataframe())
                if x in self.parameter_names]

    @property
    def number_of_scores(self) -> int:
        """The number of scores passed to the SearchCV object."""
        return len(self.score_names)

    @property
    def primary_score_name(self) -> str:
        """
        The first scorer passed to the SearchCV will be treated as the primary score. This
        property returns the name of the score.
        """
        return self.score_names[0]

    @property
    def primary_score_averages(self) -> np.array:
        """
        The first scorer passed to the SearchCV will be treated as the primary score. This
        property returns the average score (across all splits) for each trial. Note that the
        average scores are the weighted averages
        https://stackoverflow.com/questions/44947574/what-is-the-meaning-of-mean-test-score-in-cv-result.
        """
        return np.array(self.test_score_averages[self.primary_score_name])

    def score_standard_errors(self, score_name: str) -> np.array:
        """
        The first scorer passed to the SearchCV will be treated as the primary score. This
        property returns the standard error associated with the mean score of each trial, for the
        primary score.
        """
        score_standard_deviations = self.test_score_standard_deviations[score_name]
        return np.array(score_standard_deviations) / math.sqrt(self.number_of_splits)

    @property
    def trial_rankings(self) -> np.array:
        """
        The ranking of the corresponding index, in terms of best to worst "primary" (i.e. first)
        score.

        For example, assume this property returned the following list :
            [5, 6, 7, 8, 3, 4, 1, 2]
            This means that the 6th index/trial had the highest ranking (1); and that the 3rd index
            had the worst ranking (8)

        This differs from `best_trial_indexes` which returns the order of indexes (of the trials)
        from best to worst.
        So in the example above, the first value returned in the `best_trial_indexes` array would
        be 6 because the best score (across trials) is at index 6 (i.e. the 7th trial).
        The last value in the array returned by `best_trial_indexes` would be 3, because the worst
        score is at index 3 (i.e. the 4th trial).

        Note that `trial_rankings` starts at 1 while best_trial_indexes starts at 0.
        """
        return np.array(self.test_score_rankings[self.primary_score_name])

    @property
    def best_trial_indexes(self) -> np.array:
        """
        The indexes of best to worst "primary" (i.e. first) scores. See documentation for
        `trial_rankings` to understand the differences between the two properties.
        """
        return np.argsort(self.trial_rankings)

    @property
    def best_score_index(self) -> int:
        """The index of best "primary" (i.e. first) score."""
        return self.best_trial_indexes[0]

    @property
    def best_params(self) -> dict:
        """
        The *best* score. "Best" could be the highest or lowest depending on
        `higher_score_is_better`) associated with the "primary" (i.e. first) score.
        """
        best_params = self.trials[self.best_score_index].copy()

        if self.parameter_names_mapping:
            best_params = {
                value:
                    best_params[key]
                    for key, value in self.parameter_names_mapping.items()
                    if key in best_params
            }

        return best_params

    @property
    def best_score(self) -> float:
        """
        The "best" score (could be the highest or lowest depending on `higher_score_is_better`)
        associated with the primary score.
        """
        return self.primary_score_averages[self.best_score_index]

    @property
    def best_standard_error(self) -> float:
        """The standard error associated with the best of the primary scores."""
        return self.score_standard_errors(score_name=self.primary_score_name)[self.best_score_index]  # noqa

    @property
    def indexes_within_1_standard_error(self) -> list:
        """
        Returns the trial indexes where the primary scores (i.e. first scorer
        passed to SearchCV object; i.e. first column of the to_dataframe() DataFrame) are within 1
        standard error of the highest primary score.
        """
        cv_dataframe = self.to_dataframe(sort_by_score=True)

        if self.higher_score_is_better:
            return list(
                cv_dataframe.index[
                    cv_dataframe.iloc[:, 0] >= self.best_score - self.best_standard_error
                ],
            )

        return list(
            cv_dataframe.index[
                cv_dataframe.iloc[:, 0] <= self.best_score + self.best_standard_error
            ],
        )

    @property
    def fit_time_averages(self) -> np.array:
        """
        Returns a list of floats; one value for each trial (i.e. a single set of hyper-params).
        Each value is the average number of seconds that the trial took to fit the model, per split
        (i.e. the average fit time of all splits).
        """
        return np.array(self.timings['fit time averages'])

    @property
    def fit_time_standard_deviations(self) -> np.array:
        """
        Returns a list of floats; one value for each trial (i.e. a single set of hyper-params).
        Each value is the standard deviation of seconds that the trial took to fit the model, per
        split (i.e. the standard deviation of fit time across all splits).
        """
        return np.array(self.timings['fit time standard deviations'])

    @property
    def score_time_averages(self) -> np.array:
        """
        Returns a list of floats; one value for each trial (i.e. a single set of hyper-params).
        Each value is the average number of seconds that the trial took to score the model, per
        split (i.e. the average score time of all splits).
        """
        return np.array(self.timings['score time averages'])

    @property
    def score_time_standard_deviations(self) -> np.array:
        """
        Returns a list of floats; one value for each trial (i.e. a single set of hyper-params).
        Each value is the standard deviation of seconds that the trial took to score the model,
        per split (i.e. the standard deviation of score time across all splits).
        """
        return np.array(self.timings['score time standard deviations'])

    @property
    def trial_fit_times(self) -> np.array:
        """
        For each trial, it is the amount of time it took to fit the model.

        Calculated by Average fit time for each trial multiplied by the number of splits per trial.

        self.fit_time_averages * self.number_of_splits

        Returns
            array containing the fit time for each trial
        """
        return self.fit_time_averages * self.number_of_splits

    @property
    def fit_time_total(self) -> float:
        """Total fit time across all trials."""
        return float(np.sum(self.trial_fit_times))

    @property
    def trial_score_times(self) -> np.array:
        """
        For each trial, it is the amount of time it took to score the model.

        Calculated by Average score time for each trial multiplied by the number of splits per
        trial.

        self.score_time_averages * self.number_of_splits

        Returns
            array containing the score time for each trial
        """
        return self.score_time_averages * self.number_of_splits

    @property
    def score_time_total(self) -> float:
        """Total score time across all trials."""
        return float(np.sum(self.trial_score_times))

    @property
    def average_time_per_trial(self) -> float:
        """Average time per trial."""
        return float(np.mean(self.trial_fit_times + self.trial_score_times))

    @property
    def total_time(self) -> float:
        """Total time it took across all trials."""
        return self.fit_time_total + self.score_time_total

    def plot_performance_across_trials(
            self,
            size: str | None = None,
            color: str | None = None,
            color_continuous_scale: list[str] = px.colors.diverging.balance,
            facet_by: str | None = None,
            facet_num_col: int = 3,
            query: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (scatter-plot) of the primary score (y-axis) in order of trial
        execution (x-axis). Especially useful for e.g. BayesSearchCV.

        Args:
            size:
                The name of a hyper-parameter, the values of which will be used to determine the
                size of the corresponding points in the plot. This value is passed to plotly.
            color:
                The name of a hyper-parameter, the values of which will be used to determine the
                color of the corresponding points in the plot. This value is passed to plotly.
            color_continuous_scale:
                A list of strings that should contain valid CSS-colors. This value is passed to
                plotly.
            facet_by:
                name of the hyper-parameter to facet by.
            facet_num_col:
                the max number of columns in the graph to facet
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        score_column = self.primary_score_name + " Mean"

        title = "Performance Over Time (Across Trials)<br>" \
                "<sup>This graph shows the average CV score across all trials, in order of " \
                "execution.</sup>"
        if size is not None:
            title = title + \
                "<br><sup>The size of the point corresponds to the value of <b>" \
                f"'{size}'</b>.</sup>"

        labeled_data = self.to_labeled_dataframe(query=query)
        if facet_by:
            labeled_data['Trial Index'] = labeled_data.\
                groupby(facet_by)["Trial Index"].\
                rank(method="first", ascending=True)
        # only include the columns we need, so that we don't unnecessarily drop rows with NA (i.e.
        # NAs in columns not used in the graph)
        columns = [x for x in ['Trial Index', score_column, size, color, facet_by, 'label']
                   if x is not None]
        labeled_data = labeled_data[columns]
        labeled_data = labeled_data.dropna(axis=0, how='any')
        labeled_data = labeled_data.reset_index(drop=True)

        fig = px.scatter(
            data_frame=labeled_data,
            x='Trial Index',
            y=score_column,
            size=size,
            color=color,
            color_continuous_scale=color_continuous_scale,
            trendline='lowess',
            facet_col=facet_by,
            facet_col_wrap=facet_num_col,
            labels={
                score_column: f"Average CV Score<br>({self.primary_score_name})",
            },
            title=title,
            custom_data=['label'],
            height=height,
            width=width,
        )
        fig.update_traces(
            hovertemplate="<br>".join([
                "Trial Index: %{x}",
                score_column + ": " + "%{y}",
                "<br>Parameters: %{customdata[0]}",
            ]),
        )
        return fig

    def plot_parameter_values_across_trials(
            self,
            query: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (scatter-plot per numeric parameter) of the parameter's values
        (y-axis) in order of trial execution (x-axis). Especially useful for e.g. BayesSearchCV.

        Args:
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        color_continuous_scale = px.colors.diverging.RdYlGn
        if not self.higher_score_is_better:
            color_continuous_scale = color_continuous_scale.reverse()

        score_column = self.primary_score_name + " Mean"

        labeled_data = self.to_labeled_dataframe(query=query)
        labeled_long = pd.melt(
            labeled_data,
            id_vars=['Trial Index', score_column, 'label'],
            value_vars=[x for x in self.numeric_parameters if x in labeled_data.columns],
            var_name='parameter',
        )

        fig = px.scatter(
            data_frame=labeled_long,
            x='Trial Index',
            y='value',
            color=score_column,
            color_continuous_scale=color_continuous_scale,
            facet_col='parameter',
            facet_col_wrap=3,
            trendline='lowess',
            labels={
                'value': 'Parameter Value',
            },
            title="Parameter Values Evaluated Over Time (Across Trials)<br>"
                  "<sup>This graph shows the parameter values evaluated across all trials.<br>"
                  "The color corresponds to the average CV score associated with that trial/point."
                  "</sup>",
            custom_data=['label', score_column],
            height=height,
            width=width,
        )
        fig.update_traces(
            hovertemplate="<br>".join([
                "Trial Index: %{x}",
                "Parameter Value: %{y}",
                score_column + ": %{customdata[1]}",
                "<br>Parameters: %{customdata[0]}",
            ]),
        )
        fig.update_yaxes(matches=None, showticklabels=True)
        return fig

    def plot_parallel_coordinates(
            self,
            include_all_scores: bool = True,
            query: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (parallel-coordinates) of the numeric parameters' values (y-axis),
        along with the corresponding score averages.

        Args:
            include_all_scores:
                Only applicable if the results have multiple scores.
                If True, includes all of the scores. If False, includes only the primary score.
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        color_continuous_scale = px.colors.diverging.RdYlGn
        if not self.higher_score_is_better:
            color_continuous_scale = color_continuous_scale.reverse()

        primary_score_column = self.primary_score_name + " Mean"

        if include_all_scores:
            score_columns = [score + " Mean" for score in self.score_names]
        else:
            score_columns = [primary_score_column]

        # NOTE: sort_by_score=False because there is a weird bug in plotly such that if the index
        # is not 0-x than the order seems to get messed up
        # https://github.com/plotly/plotly.py/issues/3576
        # https://github.com/plotly/plotly.py/issues/3577
        data = self.to_dataframe(sort_by_score=False, query=query)
        numeric_columns = [x for x in self.numeric_parameters if x in data.columns]
        data = data[numeric_columns + score_columns].dropna(axis=0)
        data = data.reset_index(drop=True)
        return px.parallel_coordinates(
            data,
            color=primary_score_column,
            color_continuous_scale=color_continuous_scale,
            height=height,
            width=width,
            title="Parallel Coordinates of Hyper-Parameters and Score Averages<br>",
        )

    def plot_scatter_matrix(
            self,
            include_all_scores: bool = True,
            query: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (scatter-matrix) of the all parameters and scores.

        Args:
            include_all_scores:
                Only applicable if the results have multiple scores.
                If True, includes all of the scores. If False, includes only the primary score.
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        color_continuous_scale = px.colors.diverging.RdYlGn
        if not self.higher_score_is_better:
            color_continuous_scale = color_continuous_scale.reverse()

        primary_score_column = self.primary_score_name + " Mean"

        if include_all_scores:
            score_columns = [score + " Mean" for score in self.score_names]
        else:
            score_columns = [primary_score_column]

        # NOTE: sort_by_score=False because there is a weird bug in plotly such that if the index
        # is not 0-x than the order seems to get messed up
        # https://github.com/plotly/plotly.py/issues/3576
        # https://github.com/plotly/plotly.py/issues/3577
        data = self.to_dataframe(sort_by_score=False, query=query)
        columns = [x for x in self.parameter_names if x in data.columns]
        data = data[score_columns + columns]
        data = data.reset_index(drop=True)
        return px.scatter_matrix(
            data,
            color=primary_score_column,
            color_continuous_scale=color_continuous_scale,
            height=height,
            width=width,
        )

    def plot_performance_numeric_params(
            self,
            query: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (scatter-matrix) of the each of the numeric parameters' values
        (x-axis) vs the primary score (y-axis).

        Args:
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        color_continuous_scale = px.colors.diverging.RdYlGn
        if not self.higher_score_is_better:
            color_continuous_scale = color_continuous_scale.reverse()

        primary_score_column = self.primary_score_name + " Mean"

        data = self.to_labeled_dataframe(query=query)
        columns = [x for x in self.numeric_parameters if x in data.columns]
        labeled_long = pd.melt(
            data,
            id_vars=[primary_score_column, 'label'],
            value_vars=columns,
            var_name='parameter',
        )

        fig = px.scatter(
            data_frame=labeled_long,
            x='value',
            y=primary_score_column,
            color=primary_score_column,
            color_continuous_scale=color_continuous_scale,
            facet_col='parameter',
            facet_col_wrap=2,
            trendline='lowess',
            labels={'value': 'Parameter Value'},
            title="Variable Performance<br><sup>Numeric Parameters</sup>",
            custom_data=['label', primary_score_column],
            height=height,
            width=width,
        )
        fig.update_traces(
            hovertemplate="<br>".join([
                "Parameter Value: %{x}",
                primary_score_column + ": %{customdata[1]}",
                "<br>Parameters: %{customdata[0]}",
            ]),
        )
        fig.update_xaxes(matches=None, showticklabels=True)
        return fig

    def plot_performance_non_numeric_params(
            self,
            query: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (box-plot) of the each of the non-numeric parameters' values
        (x-axis) vs the primary score (y-axis).

        Args:
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        # color_continuous_scale = px.colors.diverging.RdYlGn
        # if not self.higher_score_is_better:
        #     color_continuous_scale = color_continuous_scale.reverse()

        primary_score_column = self.primary_score_name + " Mean"

        data = self.to_labeled_dataframe(query=query)
        columns = [x for x in self.non_numeric_parameters if x in data.columns]
        labeled_long = pd.melt(
            data,
            id_vars=[primary_score_column, 'label'],
            value_vars=columns,
            var_name='parameter',
        )

        scatter = px.scatter(
            data_frame=labeled_long,
            x='value',
            y=primary_score_column,
            # color=primary_score_column,
            # color_continuous_scale=color_continuous_scale,
            facet_col='parameter',
            facet_col_wrap=2,
            labels={'value': 'Parameter Value'},
            title="Variable Performance<br><sup>Non-Numeric Parameters</sup>",
            custom_data=['label', primary_score_column],
            height=height,
            width=width,
        )
        scatter.update_traces(
            hovertemplate="<br>".join([
                "Parameter Value: %{x}",
                primary_score_column + ": %{customdata[1]}",
                "<br>Parameters: %{customdata[0]}",
            ]),
        )
        scatter.update_xaxes(matches=None, showticklabels=True)
        fig = px.box(
            data_frame=labeled_long,
            x='value',
            y=primary_score_column,
            facet_col='parameter',
            facet_col_wrap=2,
            labels={'value': 'Parameter Value'},
            title="Variable Performance<br><sup>Non-Numeric Parameters</sup>",
            custom_data=['label', primary_score_column],
            height=height,
            width=width,
        )
        for x in range(len(scatter.data)):
            fig.add_trace(scatter.data[x])

        fig.update_xaxes(matches=None, showticklabels=True)
        return fig

    def plot_score_vs_parameter(
            self,
            parameter: str,
            query: str | None = None,
            size: str | None = None,
            color: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (scatter-plot) of the primary score (y-axis) vs a given parameter
        (x-axis).

        Args:
            parameter:
                The name of a hyper-parameter to plot against the score, on the x-axis.
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            size:
                The name of a hyper-parameter, the values of which will be used to determine the
                size of the corresponding points in the plot. This value is passed to plotly.
            color:
                The name of a hyper-parameter, the values of which will be used to determine the
                color of the corresponding points in the plot. This value is passed to plotly.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        color_continuous_scale = px.colors.diverging.balance
        if not self.higher_score_is_better:
            color_continuous_scale = color_continuous_scale.reverse()

        primary_score_column = self.primary_score_name + " Mean"
        title = f"Primary Score ({self.primary_score_name}) vs <b>{parameter}</b>"
        if size:
            title = title \
                + "<br><sup>The size of the point corresponds to the value of <b>" \
                + f"'{size}'</b>.</sup>"

        data = self.to_labeled_dataframe(query=query)
        # only include the columns we need, so that we don't unnecessarily drop rows with NA (i.e.
        # NAs in columns not used in the graph)
        columns = [
            x for x in [parameter, primary_score_column, size, color, 'label'] if x is not None
        ]
        data = data[columns]
        data = data.dropna(axis=0, how='any')
        data = data.reset_index(drop=True)
        fig = px.scatter(
            data_frame=data,
            x=parameter,
            y=primary_score_column,
            size=size,
            color=color,
            color_continuous_scale=color_continuous_scale,
            trendline='lowess',
            labels={
                primary_score_column:
                    f"Average Cross Validation Score ({self.primary_score_name})",
            },
            title=title,
            custom_data=['label'],
            height=height,
            width=width,
        )
        fig.update_traces(
            hovertemplate="<br>".join([
                "Parameter Value: %{x}",
                primary_score_column + ": " + "%{y}",
                "<br>Parameters: %{customdata[0]}",
            ]),
        )
        return fig

    def plot_parameter_vs_parameter(
            self,
            parameter_x: str,
            parameter_y: str,
            query: str | None = None,
            size: str | None = None,
            height: float = 600,
            width: float = 600 * GOLDEN_RATIO) -> plotly.graph_objects.Figure:
        """
        Returns a Plotly Figure (scatter-plot) between two hyper-parameters.

        Args:
            parameter_x:
                The name of a hyper-parameter to plot against another parameter, on the x-axis.
            parameter_y:
                The name of a hyper-parameter to plot against another parameter, on the y-axis.
            query:
                string to be passed to `to_dataframe()`; see documentation for that method.
            size:
                The name of a hyper-parameter, the values of which will be used to determine the
                size of the corresponding points in the plot. This value is passed to plotly.
            height:
                The height of the plot. This value is passed to plotly.
            width:
                The width of the plot. This value is passed to plotly.
        """
        color_continuous_scale = px.colors.diverging.RdYlGn
        if not self.higher_score_is_better:
            color_continuous_scale = color_continuous_scale.reverse()

        primary_score_column = self.primary_score_name + " Mean"
        title = f"<b>{parameter_y}</b> vs <b>{parameter_x}</b>"

        scaled_size = None
        labeled_data = self.to_labeled_dataframe(query=query)

        # only include the columns we need, so that we don't unnecessarily drop rows with NA (i.e.
        # NAs in columns not used in the graph)
        columns = [
            x for x in [parameter_x, parameter_y, primary_score_column, size, 'label']
            if x is not None
        ]
        labeled_data = labeled_data[columns]
        labeled_data = labeled_data.dropna(axis=0, how='any')
        labeled_data = labeled_data.reset_index(drop=True)

        if size:
            title = title \
                + "<br><sup>The size of the point corresponds to the value of <b>" \
                + f"'{size}'</b>.</sup>"
            if size in self.numeric_parameters:
                # need to do this or else the points are all the same size
                # but only if size has numeric values
                scaled_size = MinMaxScaler(feature_range=(0.1, 0.9)).\
                    fit_transform(labeled_data[[size]]).reshape(1, -1)
                scaled_size = scaled_size.tolist()[0]

        fig = px.scatter(
            data_frame=labeled_data,
            x=parameter_x,
            y=parameter_y,
            size=scaled_size,
            color=primary_score_column,
            color_continuous_scale=color_continuous_scale,
            trendline='lowess',
            title=title,
            custom_data=['label', primary_score_column],
            height=height,
            width=width,
        )
        fig.update_traces(
            hovertemplate="<br>".join([
                parameter_x + ": %{x}",
                parameter_y + ": %{y}",
                primary_score_column + ": " + "%{customdata[1]}",
                "<br>Parameters: %{customdata[0]}",
            ]),
        )
        return fig


class TwoClassEvaluator:
    """Calculates various metrics for Two Class (i.e. 0's/1's) prediction scenarios."""

    def __init__(self,
                 actual_values: np.ndarray,
                 predicted_scores: np.ndarray,
                 positive_class: str = 'Positive Class',
                 negative_class: str = 'Negative Class',
                 score_threshold: float = 0.5,
                 ):
        """
        Args:
            actual_values:
                array of 0's and 1's
            predicted_scores:
                array of decimal/float values from `predict_proba()`; NOT the actual class
            positive_class:
                string of the name/label of the positive class (i.e. value of 1). In other words,
                not 'positive' in the sense of 'good' but 'positive' as in 'True/False Positive'.
            negative_class:
                string of the name/label of the negative class (i.e. value of 0). In other words,
                not 'negative' in the sense of 'good' but 'negative' as in 'True/False Negative'.
            score_threshold:
                the score/probability threshold for turning scores into 0's and 1's and
                corresponding labels.
        """
        assert len(actual_values) == len(predicted_scores)

        if not all(np.unique(actual_values) == [0, 1]):
            message = "Values of `actual_values` should 0 or 1. Found " \
                f"`{np.unique(actual_values)}`"
            raise HelpskParamValueError(message)

        if not all(np.logical_and(predicted_scores >= 0, predicted_scores <= 1)):
            message = "Values of `predicted_scores` should be between 0 and 1."
            raise HelpskParamValueError(message)

        self._positive_class = positive_class
        self._negative_class = negative_class
        self._actual_values = actual_values
        self._predicted_scores = predicted_scores
        self.score_threshold = score_threshold
        predicted_values = np.where(predicted_scores > self.score_threshold, 1, 0)
        self._confusion_matrix = confusion_matrix(y_true=actual_values, y_pred=predicted_values)

        self.sample_size = len(actual_values)
        assert self.sample_size == self._confusion_matrix.sum()

        true_neg, false_pos, false_neg, true_pos = self._confusion_matrix.ravel()

        self._actual_positives = true_pos + false_neg
        assert self._actual_positives == sum(self._actual_values == 1)

        self._actual_negatives = true_neg + false_pos

        self._true_negatives = true_neg
        self._false_positives = false_pos
        self._false_negatives = false_neg
        self._true_positives = true_pos

        self.auc = roc_auc_score(y_true=actual_values, y_score=predicted_scores)
        self.average_precision_score = average_precision_score(
            y_true=actual_values, y_score=predicted_scores,
        )

    @property
    def true_positive_rate(self) -> float:
        """True Positive Rate."""
        return 0 if self._actual_positives == 0 else self._true_positives / self._actual_positives

    @property
    def true_negative_rate(self) -> float:
        """True Negative Rate i.e. Specificity."""
        return 0 if self._actual_negatives == 0 else self._true_negatives / self._actual_negatives

    @property
    def false_negative_rate(self) -> float:
        """False Negative Rate."""
        return 0 if self._actual_positives == 0 else self._false_negatives / self._actual_positives

    @property
    def false_positive_rate(self) -> float:
        """False Positive Rate."""
        return 0 if self._actual_negatives == 0 else self._false_positives / self._actual_negatives

    @property
    def accuracy(self) -> float | None:
        """Accuracy."""
        return None if self.sample_size == 0 else \
            (self._true_negatives + self._true_positives) / self.sample_size

    @property
    def error_rate(self) -> float | None:
        """error_rate."""
        return None if self.sample_size == 0 else \
            (self._false_positives + self._false_negatives) / self.sample_size

    @property
    def positive_predictive_value(self) -> float:
        """Positive Predictive Value i.e. Precision."""
        return 0 if (self._true_positives + self._false_positives) == 0 else \
            self._true_positives / (self._true_positives + self._false_positives)

    @property
    def negative_predictive_value(self) -> float:
        """Negative Predictive Value."""
        return 0 if (self._true_negatives + self._false_negatives) == 0 else \
            self._true_negatives / (self._true_negatives + self._false_negatives)

    @property
    def prevalence(self) -> float | None:
        """Prevalence."""
        return None if self.sample_size == 0 else \
            self._actual_positives / self.sample_size

    @property
    def kappa(self) -> float | None:
        """Kappa."""
        if self.sample_size == 0 or \
                ((self._true_negatives + self._false_negatives) / self.sample_size) == 0:
            return None
        # proportion of the actual agreements
        # add the proportion of all instances where the predicted type and actual type agree
        pr_a = (self._true_negatives + self._true_positives) / self.sample_size
        # probability of both predicted and actual being negative
        p_negative_prediction_and_actual = \
            ((self._true_negatives + self._false_positives) / self.sample_size) * \
            ((self._true_negatives + self._false_negatives) / self.sample_size)
        # probability of both predicted and actual being positive
        p_positive_prediction_and_actual = \
            self.prevalence * ((self._false_positives + self._true_positives) / self.sample_size)
        # probability that chance alone would lead the predicted and actual values to match, under
        # the assumption that both are selected randomly (i.e. implies independence) according to
        # the observed proportions (probability of independent events = P(A & B) == P(A) * P(B)
        pr_e = p_negative_prediction_and_actual + p_positive_prediction_and_actual
        return (pr_a - pr_e) / (1 - pr_e)

    @property
    def f1_score(self) -> float:
        """F1 Score. https://en.wikipedia.org/wiki/F-score."""
        return self.fbeta_score(beta=1)

    def fbeta_score(self, beta: float) -> float:
        """
        :param beta: The `beta` parameter determines the weight of precision in the combined score.
            `beta < 1` lends more weight to precision (i.e. positive predictive value), while
            `beta > 1` favors recall (i.e. true positive rate)
            (`beta -> 0` considers only precision, `beta -> inf` only recall).
            http://scikit-learn.org/stable/modules/generated/sklearn.metrics.fbeta_score.html
        :return:
        """
        if self.positive_predictive_value is None or self.sensitivity is None or \
                (self.positive_predictive_value + self.sensitivity) == 0:
            return 0

        return (1 + (beta ** 2)) * (self.positive_predictive_value * self.sensitivity) / \
               (((beta ** 2) * self.positive_predictive_value) + self.sensitivity)

    @property
    def sensitivity(self) -> float:
        """Sensitivity i.e. True Positive Rate."""
        return self.true_positive_rate

    @property
    def specificity(self) -> float:
        """Specificity i.e. True Negative Rate."""
        return self.true_negative_rate

    @property
    def precision(self) -> float:
        """Precision i.e. Positive Predictive Value."""
        return self.positive_predictive_value

    @property
    def recall(self) -> float:
        """Recall i.e. True Positive Rate."""
        return self.true_positive_rate

    @property
    def all_metrics(self) -> dict:
        """All of the metrics are returned as a dictionary."""
        auc = 'Area under the ROC curve (true pos. rate vs false pos. rate); ' \
            'ranges from 0.5 (purely random classifier) to 1.0 (perfect classifier)'

        tpr = f'{self.true_positive_rate:.1%} of positive instances were correctly identified.; ' \
            f'i.e. {self._true_positives} "{self._positive_class}" labels were correctly ' \
            f'identified out of {self._actual_positives} instances; a.k.a Sensitivity/Recall'

        tnr = f'{self.true_negative_rate:.1%} of negative instances were correctly identified.; ' \
            f'i.e. {self._true_negatives} "{self._negative_class}" labels were correctly ' \
            f'identified out of {self._actual_negatives} instances'

        fpr = f'{self.false_positive_rate:.1%} of negative instances were incorrectly identified '\
            f'as positive; ' \
            f'i.e. {self._false_positives} "{self._negative_class}" labels were incorrectly ' \
            f'identified as "{self._positive_class}", out of {self._actual_negatives} instances'

        fnr = f'{self.false_negative_rate:.1%} of positive instances were incorrectly identified '\
            f'as negative; ' \
            f'i.e. {self._false_negatives} "{self._positive_class}" labels were incorrectly ' \
            f'identified as "{self._negative_class}", out of {self._actual_positives} instances'

        ppv = f'When the model claims an instance is positive, it is correct ' \
            f'{self.positive_predictive_value:.1%} of the time; ' \
            f'i.e. out of the {self._true_positives + self._false_positives} times the model ' \
            f'predicted "{self._positive_class}", it was correct {self._true_positives} ' \
            f'times; a.k.a precision'

        npv = f'When the model claims an instance is negative, it is correct ' \
            f'{self.negative_predictive_value:.1%} of the time; ' \
            f'i.e. out of the {self._true_negatives + self._false_negatives} times the model ' \
            f'predicted "{self._negative_class}", it was correct {self._true_negatives} times'

        f1 = 'The F1 score can be interpreted as a weighted average of the precision and recall, '\
            'where an F1 score reaches its best value at 1 and worst score at 0.'

        # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score
        # https://sinyi-chou.github.io/python-sklearn-precision-recall/
        average_precision_score = 'Precision/Recall AUC is calculated with `average_precision` ' \
            'which summarizes a precision-recall curve as the weighted mean ' \
            'of precisions achieved at each threshold. See sci-kit learn ' \
            'documentation for caveats.'

        accuracy = f'{self.accuracy:.1%} of instances were correctly identified'
        error = f'{self.error_rate:.1%} of instances were incorrectly identified'
        prevalence = f'{self.prevalence:.1%} of the data are positive; i.e. out of ' \
            f'{self.sample_size} total observations; {self._actual_positives} are labeled ' \
            f'as "{self._positive_class}"'
        total_obs = f'There are {self.sample_size} total observations; i.e. sample size'

        return {'AUC': (self.auc, auc),
                'True Positive Rate': (self.true_positive_rate, tpr),
                'True Negative Rate': (self.true_negative_rate, tnr),
                'False Positive Rate': (self.false_positive_rate, fpr),
                'False Negative Rate': (self.false_negative_rate, fnr),
                'Positive Predictive Value': (self.positive_predictive_value, ppv),
                'Negative Predictive Value': (self.negative_predictive_value, npv),
                'F1 Score': (self.f1_score, f1),
                'Precision/Recall AUC': (self.average_precision_score, average_precision_score),
                'Accuracy': (self.accuracy, accuracy),
                'Error Rate': (self.error_rate, error),
                '% Positive': (self.prevalence, prevalence),
                'Total Observations': (self.sample_size, total_obs)}

    def all_metrics_df(
            self,
            return_explanations: bool = True,
            dummy_classifier_strategy: str | list | None = 'prior',
            dummy_classifier_constant: int = 1,
            return_style: bool = False,
            round_by: int | None = None) -> pd.DataFrame | Styler:
        """
        All of the metrics are returned as a DataFrame.

        Args:
            return_explanations:
                if True, then return descriptions of score and more information in an additional
                column
            dummy_classifier_strategy:
                if not None, then returns column(s) corresponding to the scores from predictions of
                sklearn.dummy.DummyClassifier, based on the strategy (or strategies) provided.
                Valid values correspond to values of `strategy` parameter listed
                https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyClassifier.html

                If a list is passed in (e.g. ['prior', 'uniform'], then one score column per value
                is added.

                If None is passed, then no additional columns are added.
            dummy_classifier_constant:
                The explicit constant as predicted by the “constant” strategy for the
                DummyClassifier.
                This parameter is useful only for the “constant” dummy_classifier_strategy.
            return_style:
                if True, return styler object; else return dataframe
            round_by:
                the number of digits to round by; if None, then don't round
        """
        result = pd.DataFrame.from_dict({key: value[0] for key, value in self.all_metrics.items()},
                                        orient='index',
                                        columns=['Score'])

        score_columns = ['Score']

        if dummy_classifier_strategy:
            if isinstance(dummy_classifier_strategy, str):
                dummy_classifier_strategy = [dummy_classifier_strategy]

            for strategy in dummy_classifier_strategy:
                dummy = DummyClassifier(strategy=strategy, constant=dummy_classifier_constant)
                # https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyClassifier.html
                # "All strategies make predictions that ignore the input feature values passed as
                # the X argument to fit and predict. The predictions, however, typically depend on
                # values observed in the y parameter passed to fit."
                _ = dummy.fit(X=self._actual_values, y=self._actual_values)
                dummy_probabilities = dummy.predict_proba(X=self._actual_values)
                dummy_probabilities = dummy_probabilities[:, 1]
                dummy_evaluator = TwoClassEvaluator(
                    actual_values=self._actual_values,
                    predicted_scores=dummy_probabilities,
                    score_threshold=self.score_threshold,
                )

                dummy_scores = dummy_evaluator.all_metrics_df(
                    return_explanations=False,
                    dummy_classifier_strategy=None,
                    return_style=False,
                )
                column_name = f"Dummy ({strategy})"
                score_columns = [*score_columns, column_name]
                dummy_scores = dummy_scores.rename(columns={'Score': column_name})
                result = pd.concat([result, dummy_scores], axis=1)

        if return_explanations:
            explanations = pd.DataFrame.from_dict(
                {key: value[1] for key, value in self.all_metrics.items()},
                orient='index',
                columns=['Explanation'],
            )
            result = pd.concat([result, explanations], axis=1)

        if round_by:
            for column in score_columns:
                result[column] = result[column].round(round_by)

        if return_style:
            subset_scores = [x for x in result.index.to_numpy() if x != 'Total Observations']

            subset_scores = pd.IndexSlice[result.loc[subset_scores, :].index, score_columns]
            subset_negative_bad = pd.IndexSlice[
                result.loc[['False Positive Rate', 'False Negative Rate'], score_columns].index,
                score_columns,
            ]
            subset_secondary = pd.IndexSlice[
                result.loc[['Accuracy', 'Error Rate', '% Positive'], score_columns].index,
                score_columns,
            ]
            subset_total_observations = pd.IndexSlice[
                result.loc[['Total Observations'], score_columns].index,
                score_columns,
            ]

            result = result.style

            if round_by:
                result = result.format(precision=round_by)

            result = result.format(subset=subset_total_observations, thousands=',', precision=0)

            result = result. \
                bar(
                    subset=subset_scores, color=hcolor.Colors.PIGMENT_GREEN.value,
                    vmin=0, vmax=1,
                ). \
                bar(subset=subset_negative_bad, color=hcolor.Colors.POPPY.value, vmin=0, vmax=1). \
                bar(subset=subset_secondary, color=hcolor.GRAY, vmin=0, vmax=1)

        return result

    def plot_confusion_matrix(self) -> None:
        """Plots a heatmap of the confusion matrix."""
        labels = np.array([
            [
                f'True Negatives\n{self._true_negatives}\n{self._true_negatives / self.sample_size:.1%}',  # noqa
                f'False Positives\n{self._false_positives}\n{self._false_positives / self.sample_size:.1%}'  # noqa
            ],
            [
                f'False Negatives\n{self._false_negatives}\n{self._false_negatives / self.sample_size:.1%}',  # noqa
                f'True Positives\n{self._true_positives}\n{self._true_positives / self.sample_size:.1%}'  # noqa
            ],
        ])

        axis = plt.subplot()
        sns.heatmap(self._confusion_matrix, annot=labels, cmap='Blues', ax=axis, fmt='')
        # labels, title and ticks
        axis.set_xlabel('Predicted')
        axis.set_ylabel('Actual')
        # axis.set_title('Confusion Matrix');
        axis.xaxis.set_ticklabels([self._negative_class, self._positive_class])
        axis.yaxis.set_ticklabels([self._negative_class, self._positive_class])
        plt.tight_layout()

    def _get_auc_curve_dataframe(self) -> pd.DataFrame:
        """
        Returns a dataframe containing the AUC line (i.e. a column of score thresholds, and the
        corresponding True Positive and False Positive Rate (as columns) for the corresponding
        score threshold.

        (A score threshold is the value for which you would predict a positive label if the value
        of the score is above the threshold (e.g. usually 0.5).
        """
        def get_true_pos_false_pos(threshold: float) -> tuple[float, float, float]:
            temp_eval = TwoClassEvaluator(
                actual_values=self._actual_values,
                predicted_scores=self._predicted_scores,
                score_threshold=threshold,
            )
            return threshold, temp_eval.true_positive_rate, temp_eval.false_positive_rate

        auc_curve = [get_true_pos_false_pos(threshold=x) for x in np.arange(0.0, 1.01, 0.01)]
        return pd.DataFrame(
            auc_curve,
            columns=['threshold', 'True Positive Rate', 'False Positive Rate'],
        )

    def _get_threshold_curve_dataframe(
            self,
            score_threshold_range: tuple[float, float] = (0.1, 0.9),
            threshold_interval: float = 0.025) -> pd.DataFrame:
        """
        Returns a dataframe containing various score thresholds from 0 to 1 (i.e. cutoff point
        where score will be labeled as a 'positive' event, and various rates (e.g. True Positive
        Rate, False Positive Rate, etc.) for the corresponding score threshold.

        (A score threshold is the value for which you would predict a positive label if the value
        of the score is above the threshold (e.g. usually 0.5).

        Args:
            score_threshold_range:
                range of score thresholds to plot (x-axis); tuple with minimum threshold in first
                index and maximum threshold in second index.
            threshold_interval:
                the interval used to determine the specific points in used in the
                score_threshold_range
        """
        def get_threshold_scores(threshold: float):  # noqa: ANN202
            temp_eval = TwoClassEvaluator(actual_values=self._actual_values,
                                          predicted_scores=self._predicted_scores,
                                          score_threshold=threshold)

            return threshold, \
                temp_eval.true_positive_rate, \
                temp_eval.false_positive_rate, \
                temp_eval.positive_predictive_value, \
                temp_eval.false_negative_rate, \
                temp_eval.true_negative_rate

        threshold_curves = [
            get_threshold_scores(threshold=x)
            for x in np.arange(score_threshold_range[0], score_threshold_range[1] + threshold_interval, threshold_interval)  # noqa: E501
        ]

        return pd.DataFrame(
            threshold_curves,
            columns=[
                'Score Threshold',
                'True Pos. Rate (Recall)',
                'False Pos. Rate',
                'Pos. Predictive Value (Precision)',
                'False Neg. Rate',
                'True Neg. Rate (Specificity)',
            ],
        )

    def plot_roc_auc_curve(
            self,
            figure_size: tuple = STANDARD_WIDTH_HEIGHT,
            return_plotly: bool = True,
            plot_threshold: bool = True) -> None | _figure.Figure:
        """
        Plots the ROC AUC.

        Args:
            figure_size:
                tuple containing `(width, height)` of plot. The default height is defined by
                `helpsk.plot.STANDARD_HEIGHT`, and the default width is
                `helpsk.plot.STANDARD_HEIGHT / helpsk.plot.GOLDEN_RATIO`
            return_plotly:
                If True, return plotly object. Otherwise, use matplotlib and end function with
                call: `plt.tight_layout()`
            plot_threshold:
                If True, indicate the score threshold (e.g. 0.5) as a large point.
        """
        auc_curve = self._get_auc_curve_dataframe()
        title = f"ROC AUC: {self.auc:.3f}"

        if return_plotly:
            if plot_threshold:
                title += f"<br><sub>The threshold of {round(self.score_threshold, 2)} " \
                    "is indicated with a large point.</sub>"

            fig = px.line(
                data_frame=auc_curve,
                x='False Positive Rate',
                y='True Positive Rate',
                color_discrete_sequence=[hcolor.Colors.DOVE_GRAY.value],
                height=550,
                width=550 * GOLDEN_RATIO,
                title=title,
            )
            fig.add_trace(
                px.scatter(
                    data_frame=auc_curve,
                    x='False Positive Rate',
                    y='True Positive Rate',
                    color='threshold',
                ).data[0],
            )
            if plot_threshold:
                fig.add_trace(
                    px.scatter(
                        data_frame=auc_curve.query(f'threshold == {round(self.score_threshold, 2)}'),  # noqa
                        x='False Positive Rate',
                        y='True Positive Rate',
                        size=[2],
                    ).data[0],
                )
            return fig

        plt.figure(figsize=figure_size)
        axis = sns.lineplot(
            data=auc_curve,
            x='False Positive Rate', y='True Positive Rate',
            ci=None,
        )
        axis.set_title(title)
        for i, (x, y, s) in enumerate(zip(auc_curve['False Positive Rate'],
                                          auc_curve['True Positive Rate'],
                                          auc_curve['threshold'])):
            if i % 5 == 0:
                axis.text(x, y, f'{s:.3}')
        axis.set_xticks(np.arange(0, 1.1, .1))
        axis.set_yticks(np.arange(0, 1.1, .1))
        plt.grid()
        plt.tight_layout()
        return None

    def plot_precision_recall_auc_curve(
            self,
            score_threshold_range: tuple[float, float] = (0.1, 0.9),
            threshold_interval: float = 0.025,
            figure_size: tuple = STANDARD_WIDTH_HEIGHT,
            return_plotly: bool = True,
            plot_threshold: bool = True) -> None | _figure.Figure:
        """
        Plots the ROC AUC.

        Args:
            score_threshold_range:
                range of score thresholds to plot (x-axis); tuple with minimum threshold in first
                index and maximum threshold in second index.
            threshold_interval:
                the interval used to determine the specific points in used in the
                score_threshold_range
            figure_size:
                tuple containing `(width, height)` of plot. The default height is defined by
                `helpsk.plot.STANDARD_HEIGHT`, and the default width is
                `helpsk.plot.STANDARD_HEIGHT / helpsk.plot.GOLDEN_RATIO`
            return_plotly:
                If True, return plotly object. Otherwise, use matplotlib and end function with
                call: `plt.tight_layout()`
            plot_threshold:
                If True, indicate the score threshold (e.g. 0.5) as a large point.
        """
        precision_recall_data = self._get_threshold_curve_dataframe(
            score_threshold_range=score_threshold_range,
            threshold_interval=threshold_interval,
        )
        precision_recall = precision_recall_data[[
            'Score Threshold',
            'Pos. Predictive Value (Precision)',
            'True Pos. Rate (Recall)',
        ]]
        title = f"Precision/Recall AUC: {self.average_precision_score:.3f}"

        if return_plotly:
            if plot_threshold:
                title += f"<br><sub>The threshold of {round(self.score_threshold, 2)} " \
                         "is indicated with a large point.</sub>"

            fig = px.line(
                data_frame=precision_recall,
                x='True Pos. Rate (Recall)',
                y='Pos. Predictive Value (Precision)',
                color_discrete_sequence=[hcolor.Colors.DOVE_GRAY.value],
                height=550,
                width=550 * GOLDEN_RATIO,
                title=title,
            )
            fig.add_trace(
                px.scatter(
                    data_frame=precision_recall,
                    x='True Pos. Rate (Recall)',
                    y='Pos. Predictive Value (Precision)',
                    color='Score Threshold',
                ).data[0],
            )
            if plot_threshold:
                fig.add_trace(
                    px.scatter(
                        data_frame=precision_recall.query(
                            f'`Score Threshold` == {round(self.score_threshold, 2)}',
                        ),
                        x='True Pos. Rate (Recall)',
                        y='Pos. Predictive Value (Precision)',
                        size=[2],
                    ).data[0],
                )
            return fig

        plt.figure(figsize=figure_size)
        axis = sns.lineplot(
            data=precision_recall,
            x='True Pos. Rate (Recall)',
            y='Pos. Predictive Value (Precision)',
            ci=None,
        )
        axis.set_title(title)
        for i, (x, y, s) in enumerate(zip(precision_recall['True Pos. Rate (Recall)'],
                                          precision_recall['Pos. Predictive Value (Precision)'],
                                          precision_recall['Score Threshold'])):
            if i % 5 == 0:
                axis.text(x, y, f'{s:.3}')
        axis.set_xticks(np.arange(0, 1.1, .1))
        axis.set_yticks(np.arange(0, 1.1, .1))
        plt.grid()
        plt.tight_layout()
        return None

    def plot_threshold_curves(
            self,
            score_threshold_range: tuple[float, float] = (0.1, 0.9),
            threshold_interval: float = 0.025,
            figure_size: tuple = STANDARD_WIDTH_HEIGHT,
            return_plotly: bool = True,
            plot_threshold: bool = False) -> None | _figure.Figure:
        """
        Plots various scores (e.g. True Positive Rate, False Positive Rate, etc.) for various
        score thresholds. (A score threshold is the value for which you would predict a positive
        label if the value of the score is above the threshold (e.g. usually 0.5).

        Args:
            score_threshold_range:
                range of score thresholds to plot (x-axis); tuple with minimum threshold in first
                index and maximum threshold in second index.
            threshold_interval:
                the interval used to determine the specific points in used in the
                score_threshold_range
            figure_size:
                tuple containing `(width, height)` of plot. The default height is defined by
                `helpsk.plot.STANDARD_HEIGHT`, and the default width is
                `helpsk.plot.STANDARD_HEIGHT / helpsk.plot.GOLDEN_RATIO`
            return_plotly:
                If True, return plotly object. Otherwise, use matplotlib and end function with
                call: `plt.tight_layout()`
            plot_threshold:
                If True, indicate the score threshold (e.g. 0.5) with a vertical line.
        """
        threshold_curves = self._get_threshold_curve_dataframe(
            score_threshold_range=score_threshold_range,
            threshold_interval=threshold_interval,
        )

        if return_plotly:
            custom_colors = [
                hcolor.Colors.PASTEL_BLUE.value,
                hcolor.Colors.CUSTOM_GREEN.value,
                hcolor.Colors.YELLOW_PEPPER.value,
                hcolor.Colors.CRAIL.value,
                hcolor.Colors.CADMIUM_ORANGE.value,
            ]

            title = "Tradeoffs Across Various Score Thresholds"
            if plot_threshold:
                title += "<br><sub>Black line is threshold of " \
                    f"{round(self.score_threshold, 2)}.</sub>"

            fig = px.line(
                data_frame=pd.melt(frame=threshold_curves, id_vars=['Score Threshold']),
                x='Score Threshold',
                y='value',
                color='variable',
                color_discrete_sequence=custom_colors,
                labels={
                    'variable': 'Rate Type',
                    'value': 'Rate',
                },
                height=550,
                width=550 * GOLDEN_RATIO,
                title=title,
            )
            if plot_threshold:
                fig = fig.add_vline(
                    x=round(self.score_threshold, 2),
                    line_color=hcolor.Colors.BLACK_SHADOW.value,
                )
            return fig

        plt.figure(figsize=figure_size)
        axis = sns.lineplot(
            x='Score Threshold', y='value', hue='variable',
            data=pd.melt(frame=threshold_curves, id_vars=['Score Threshold']),
        )
        axis.set_xticks(np.arange(score_threshold_range[0], score_threshold_range[1] + 0.1, 0.1))
        axis.set_yticks(np.arange(0, 1.1, .1))

        if plot_threshold:
            plt.vlines(x=self.score_threshold, ymin=0, ymax=1, colors='black')
        plt.grid()
        plt.tight_layout()
        return None

    def plot_precision_recall_tradeoff(
            self,
            score_threshold_range: tuple[float, float] = (0.1, 0.9),
            figure_size: tuple = STANDARD_WIDTH_HEIGHT,
            return_plotly: bool = True,
            plot_threshold: bool = False) -> None | _figure.Figure:
        """
        Plots the tradeoff between precision (i.e. positive predict value) and recall (i.e.
        True Positive Rate) for various score thresholds. (A score threshold is the value for which
        you would predict a positive label if the value of the score is above the threshold (e.g.
        usually 0.5).

        Args:
            score_threshold_range:
                range of score thresholds to plot (x-axis); tuple with minimum threshold in first
                index and maximum threshold in second index.
            threshold_interval:
                the interval used to determine the specific points in used in the
                score_threshold_range
            figure_size:
                tuple containing `(width, height)` of plot. The default height is defined by
                `helpsk.plot.STANDARD_HEIGHT`, and the default width is
                `helpsk.plot.STANDARD_HEIGHT / helpsk.plot.GOLDEN_RATIO`
            return_plotly:
                If True, return plotly object. Otherwise, use matplotlib and end function with
                call: `plt.tight_layout()`
            plot_threshold:
                If True, indicate the score threshold (e.g. 0.5) with a vertical line.
        """
        threshold_curves = self._get_threshold_curve_dataframe(
            score_threshold_range=score_threshold_range,
        )
        threshold_curves = threshold_curves[[
            'Score Threshold',
            'True Pos. Rate (Recall)',
            'Pos. Predictive Value (Precision)',
        ]]

        if return_plotly:
            custom_colors = [
                hcolor.Colors.PASTEL_BLUE.value,
                #     hcolor.Colors.CUSTOM_GREEN.value,
                hcolor.Colors.YELLOW_PEPPER.value,
                #     hcolor.Colors.CRAIL.value,
                #     hcolor.Colors.CADMIUM_ORANGE.value,
            ]
            title = "Precision Recall Tradeoff"
            if plot_threshold:
                title += f"<br><sub>Black line is default threshold of {round(self.score_threshold, 2)}.</sub>"  # noqa

            fig = px.line(
                data_frame=pd.melt(
                    frame=threshold_curves[[
                        'Score Threshold',
                        'True Pos. Rate (Recall)',
                        'Pos. Predictive Value (Precision)',
                    ]],
                    id_vars=['Score Threshold'],
                ),
                x='Score Threshold',
                y='value',
                color='variable',
                color_discrete_sequence=custom_colors,
                labels={
                    'variable': 'Rate',
                    'value': 'Value',
                },
                height=550,
                width=550 * GOLDEN_RATIO,
                title=title,
            )
            if plot_threshold:
                fig = fig.add_vline(
                    x=round(self.score_threshold, 2),
                    line_color=hcolor.Colors.BLACK_SHADOW.value,
                )
            return fig

        plt.figure(figsize=figure_size)
        axis = sns.lineplot(x='Score Threshold', y='value', hue='variable',
                            data=pd.melt(frame=threshold_curves, id_vars=['Score Threshold']))
        axis.set_xticks(np.arange(score_threshold_range[0], score_threshold_range[1] + 0.1, 0.1))
        axis.set_yticks(np.arange(0, 1.1, .1))

        if plot_threshold:
            plt.vlines(x=self.score_threshold, ymin=0, ymax=1, colors='black')
        plt.grid()
        plt.tight_layout()
        return None

    def calculate_lift_gain(
            self,
            num_buckets: int = 20,
            return_style: bool = False,
            include_all_info: bool = False) -> pd.DataFrame | Styler:
        """
        https://www.listendata.com/2014/08/excel-template-gain-and-lift-charts.html.

        Gain is the % of positive (actual) events we have 'captured' i.e. located by looking at the
        top x% of predicted scores, such that the highest scores are looked at first.
        For example, if the percentile is `5%` and the gain value is `0.3`, we can say that if we
        randomly searched `5%` of the data, we would expect to uncover about `5%` of the positive
        events; however, we have uncovered 30% of events by searching the highest 5% of scores.
        Lift is simply the ratio of the percent of events that what we have uncovered for a given
        percentile of data (i.e. gain) divided by what we would have expected by random chance
        (i.e. the percentile). So in the previous example, we uncovered 30% of positive events by
        searching the top 5% of scores; whereas if we took a random sample, we would have only
        expected to find 5% of the positive events. The lift is `0.3 / 0.05` which is `6`; meaning
        we found `6` times the amount of positive events by searching the top 5% of scores, than if
        we were to randomly sample the data.
        """
        data = pd.DataFrame({
            'predicted_scores': self._predicted_scores,
            'actual_values': self._actual_values,
        })
        data = data.sort_values(['predicted_scores'], ascending=False)

        # .qcut gets percentiles
        bins = pd.qcut(
            x=data['predicted_scores'],
            q=num_buckets,
            labels=list(range(100, 0, round(-100 / num_buckets))),
        )

        data['Percentile'] = bins

        def gain_grouping(group: pd.DataFrame) -> pd.Series:
            results = {
                '# of Obs.': len(group.actual_values),
                '# of Pos. Events': sum(group.actual_values == 1),
            }
            return pd.Series(results, index=['# of Obs.', '# of Pos. Events'])

        gain_lift_data = data.groupby('Percentile').apply(gain_grouping)
        temp = pd.DataFrame({'# of Obs.': 0, '# of Pos. Events': 0}, index=[0])
        temp.index.names = ['Percentile']
        gain_lift_data = pd.concat([gain_lift_data, temp])
        gain_lift_data = gain_lift_data.sort_index(ascending=True)

        gain_lift_data['Cumul. Pos. Events'] = gain_lift_data['# of Pos. Events'].cumsum()
        gain_lift_data['Gain'] = gain_lift_data['Cumul. Pos. Events'] / self._actual_positives
        gain_lift_data = gain_lift_data.loc[~(gain_lift_data.index == 0), :]
        gain_lift_data['Lift'] = gain_lift_data['Gain'] / (gain_lift_data.index.to_numpy() / 100)

        if not include_all_info:
            gain_lift_data = gain_lift_data[['Gain', 'Lift']]

        gain_lift_data = gain_lift_data.round(2)

        if return_style:
            gain_lift_data = gain_lift_data.style
            gain_lift_data.format(precision=2). \
                bar(subset='Gain', color=hcolor.Colors.PASTEL_BLUE.value,
                    vmin=0, vmax=1). \
                bar(subset='Lift', color=hcolor.Colors.PASTEL_BLUE.value, vmin=1)

        return gain_lift_data

    def plot_predicted_scores_histogram(self) -> None:
        """Return a histogram of the predicted scores."""
        pd.Series(self._predicted_scores).hist()
        plt.tight_layout()

    def plot_actual_vs_predict_histogram(self, plot_threshold: bool = False) -> None:
        """
        Return a histogram of the actual vs predicted scores.

        plot_threshold:
                If True, indicate the score threshold (e.g. 0.5) with a vertical line.
        """
        data = pd.DataFrame({
            'Predicted Scores': self._predicted_scores,
            'Labels': [
                self._positive_class if x == 1 else self._negative_class
                for x in self._actual_values
            ],
        })
        fig = px.box(
            data,
            x="Predicted Scores",
            y='Labels',
            color="Labels",
            points='all',
            labels={'Labels': ''},
            title="Distribution of Prediction Scores",
        )
        if plot_threshold:
            fig.add_vline(x=self.score_threshold)
        fig.update_layout(showlegend=False)
        return fig


class RegressionEvaluator:
    """Evaluates models for regression (i.e. numeric outcome) problems."""

    def __init__(
            self,
            actual_values: np.ndarray,
            predicted_values: np.ndarray):
        """
        Args:
            actual_values:
                the actual values
            predicted_values:
                the predicted values.
        """
        if not isinstance(actual_values, np.ndarray):
            actual_values = np.array(actual_values)

        if not isinstance(predicted_values, np.ndarray):
            predicted_values = np.array(predicted_values)

        if actual_values.ndim == 2:
            actual_values = actual_values.flatten()

        if predicted_values.ndim == 2:
            predicted_values = predicted_values.flatten()

        assert len(actual_values) == len(predicted_values)

        self._actual_values = actual_values
        self._predicted_values = predicted_values
        self._residuals = actual_values - predicted_values
        self._standard_deviation = np.std(actual_values)
        self._mean_squared_error = float(np.mean(np.square(actual_values - predicted_values)))
        self._mean_absolute_error = float(np.mean(np.abs(actual_values - predicted_values)))
        self._r_squared = r2_score(y_true=actual_values, y_pred=predicted_values)

    @property
    def mean_absolute_error(self) -> float:
        """Mean Absolute Error."""
        return self._mean_absolute_error

    @property
    def mean_squared_error(self) -> float:
        """Mean Squared Error."""
        return self._mean_squared_error

    @property
    def root_mean_squared_error(self) -> float:
        """Root Mean Squared Error."""
        return np.sqrt(self.mean_squared_error)

    @property
    def rmse_to_st_dev(self) -> float:
        """
        The ratio of RMSE to the standard deviation of the actual values.
        Gives an indication of how large the errors are to the actual data.
        """
        return self.root_mean_squared_error / self._standard_deviation

    @property
    def r_squared(self) -> float:
        """R Squared."""
        return self._r_squared

    @property
    def total_observations(self) -> int:
        """The total number of observations i.e. sample size."""
        return len(self._actual_values)

    @property
    def all_metrics(self) -> dict:
        """Returns a dictionary of the most common error metrics for regression problems."""
        return {
            'Mean Absolute Error (MAE)': self.mean_absolute_error,
            'Root Mean Squared Error (RMSE)': self.root_mean_squared_error,
            'RMSE to Standard Deviation of Target': self.rmse_to_st_dev,
            'R Squared': self.r_squared,
            'Total Observations': self.total_observations,
        }

    def all_metrics_df(
            self,
            dummy_regressor_strategy: str | list | None = 'mean',
            dummy_regressor_constant: int = 1,
            return_style: bool = False,
            round_by: int | None = None) -> pd.DataFrame | Styler:
        """
        All of the metrics are returned as a DataFrame.

        Args:
            dummy_regressor_strategy:
                if not None, then returns column(s) corresponding to the scores from predictions of
                sklearn.dummy.DummyRegressor, based on the strategy (or strategies) provided. Valid
                values correspond to values of `strategy` parameter listed
                https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyRegressor.html

                If a list is passed in (e.g. ['prior', 'uniform'], then one score column per value
                is added.

                If None is passed, then no additional columns are added.
            dummy_regressor_constant:
                The explicit constant as predicted by the “constant” strategy for the
                DummyRegressor.
                This parameter is useful only for the “constant” dummy_regressor_strategy.
            return_style:
                if True, return styler object; else return dataframe
            round_by:
                the number of digits to round by; if None, then don't round
        """
        result = pd.DataFrame.from_dict(self.all_metrics, orient='index', columns=['Score'])

        score_columns = ['Score']

        if dummy_regressor_strategy:
            if isinstance(dummy_regressor_strategy, str):
                dummy_regressor_strategy = [dummy_regressor_strategy]

            for strategy in dummy_regressor_strategy:
                dummy = DummyRegressor(strategy=strategy, constant=dummy_regressor_constant)
                # https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyClassifier.html
                # "All strategies make predictions that ignore the input feature values passed as
                # the X argument to fit and predict. The predictions, however, typically depend on
                # values observed in the y parameter passed to fit."
                _ = dummy.fit(X=self._actual_values, y=self._actual_values)
                dummy_predictions = dummy.predict(X=self._actual_values)
                dummy_evaluator = RegressionEvaluator(actual_values=self._actual_values,
                                                      predicted_values=dummy_predictions)

                dummy_scores = dummy_evaluator.all_metrics_df(dummy_regressor_strategy=None,
                                                              return_style=False)
                column_name = f"Dummy ({strategy})"
                score_columns = [*score_columns, column_name]
                dummy_scores = dummy_scores.rename(columns={'Score': column_name})
                result = pd.concat([result, dummy_scores], axis=1)

        if round_by is not None:
            result.iloc[0:2] = result.iloc[0:2].round(round_by)

        if return_style:
            subset_scores = pd.IndexSlice[
                result.loc[
                    ['Mean Absolute Error (MAE)', 'Root Mean Squared Error (RMSE)'],
                    score_columns,
                ].index,
                score_columns,
            ]
            subset_secondary = pd.IndexSlice[result.loc[['RMSE to Standard Deviation of Target',
                                                         'R Squared'],
                                                        score_columns].index, score_columns]
            subset_total_observations = pd.IndexSlice[
                result.loc[['Total Observations'], score_columns].index,
                score_columns,
            ]
            result = result.style
            if round_by is not None:
                result = result.format(subset=subset_scores, thousands=',', precision=round_by)
            else:
                result = result.format(subset=subset_scores, thousands=',')
            result = result.format(subset=subset_secondary, precision=3)
            result = result.format(subset=subset_total_observations, thousands=',', precision=0)

        return result

    def plot_residuals_vs_fits(self, figure_size: tuple = STANDARD_WIDTH_HEIGHT) -> None:
        """
        Plots residuals vs fitted values.

        Args:
            figure_size:
                tuple containing `(width, height)` of plot. The default height is defined by
                `helpsk.plot.STANDARD_HEIGHT`, and the default width is
                `helpsk.plot.STANDARD_HEIGHT / helpsk.plot.GOLDEN_RATIO`
        """
        lowess = sm.nonparametric.lowess
        loess_points = lowess(self._residuals, self._predicted_values)
        loess_x, loess_y = zip(*loess_points)

        plt.figure(figsize=figure_size)
        plt.plot(loess_x, loess_y, color='r')
        plt.scatter(x=self._predicted_values, y=self._residuals, s=8, alpha=0.5)
        plt.title('Residuals vs. Fitted Values')
        plt.xlabel('Fitted Values')
        plt.ylabel('Residuals (Actual - Predicted)')

    def plot_predictions_vs_actuals(self, figure_size: tuple = STANDARD_WIDTH_HEIGHT) -> object:
        """
        Plots predictions vs actual values.

        Args:
            figure_size:
                tuple containing `(width, height)` of plot. The default height is defined by
                `helpsk.plot.STANDARD_HEIGHT`, and the default width is
                `helpsk.plot.STANDARD_HEIGHT / helpsk.plot.GOLDEN_RATIO`
        """
        lowess = sm.nonparametric.lowess
        loess_points = lowess(self._predicted_values, self._actual_values)
        loess_x, loess_y = zip(*loess_points)

        plt.figure(figsize=figure_size)
        plt.plot(loess_x, loess_y, color='r', alpha=0.5, label='Loess (Predictions vs Actuals)')
        plt.plot(
            self._actual_values, self._actual_values,
            color='b', alpha=0.5, label='Perfect Prediction',
        )
        plt.scatter(x=self._actual_values, y=self._predicted_values, s=8, alpha=0.5)
        plt.title('Predicted Values vs. Actual Values')
        plt.xlabel('Actuals')
        plt.ylabel('Predicted')
        axis = plt.gca()
        handles, labels = axis.get_legend_handles_labels()
        axis.legend(handles, labels)
        plt.figtext(
            0.99, 0.01,
            'Note: observations above blue line mean model is over-predicting; below means under-'
            'predicting.',
            horizontalalignment='right',
        )
        return axis

    def plot_residuals_vs_actuals(self, figure_size: tuple = STANDARD_WIDTH_HEIGHT) -> None:
        """
        Plots residuals vs actuals values.

        Args:
            figure_size:
                tuple containing `(width, height)` of plot. The default height is defined by
                `helpsk.plot.STANDARD_HEIGHT`, and the default width is
                `helpsk.plot.STANDARD_HEIGHT / helpsk.plot.GOLDEN_RATIO`
        """
        lowess = sm.nonparametric.lowess
        loess_points = lowess(self._residuals, self._actual_values)
        loess_x, loess_y = zip(*loess_points)

        plt.figure(figsize=figure_size)
        plt.plot(loess_x, loess_y, color='r')
        plt.scatter(x=self._actual_values, y=self._residuals, s=8, alpha=0.5)
        plt.title('Residuals vs. Actual Values')
        plt.xlabel('Actual')
        plt.ylabel('Residuals (Actual - Predicted)')
        plt.figtext(
            0.99, 0.01,
            'Note: Actual > Predicted => Under-predicting (positive residual); negative residuals '
            'mean over-predicting',
            horizontalalignment='right',
        )


class TwoClassModelComparison:
    """Compares multiple models trained on Two Class (i.e. 0's/1's) prediction scenarios."""

    def __init__(
            self,
            actual_values: np.ndarray,
            predicted_scores: dict,
            positive_class: str = 'Positive Class',
            negative_class: str = 'Negative Class',
            score_threshold: float = 0.5):
        """
        Args:
            actual_values:
                array of 0's and 1's
            predicted_scores:
                dictionary per model with key as the name of the model and value that is an array
                of decimal/float values from `predict_proba()`; NOT the actual class
            positive_class:
                string of the name/label of the positive class (i.e. value of 1). In other words,
                not 'positive' in the sense of 'good' but 'positive' as in 'True/False Positive'.
            negative_class:
                string of the name/label of the negative class (i.e. value of 0). In other words,
                not 'negative' in the sense of 'good' but 'negative' as in 'True/False Negative'.
            score_threshold:
                the score/probability threshold for turning scores into 0's and 1's and
                corresponding labels.
        """
        assert isinstance(predicted_scores, dict)

        for values in predicted_scores.values():
            assert len(actual_values) == len(values)

        self._positive_class = positive_class
        self._negative_class = negative_class
        self._actual_values = actual_values
        self._predicted_scores = predicted_scores
        self.score_threshold = score_threshold

        self._evaluators = {key: TwoClassEvaluator(actual_values=actual_values,
                                                   predicted_scores=value,
                                                   positive_class=positive_class,
                                                   negative_class=negative_class,
                                                   score_threshold=score_threshold)
                            for key, value in predicted_scores.items()}

    def all_metrics_df(
            self,
            dummy_classifier_strategy: str | list | None = 'prior',
            dummy_classifier_constant: int = 1,
            return_style: bool = False,
            round_by: int | None = None) -> pd.DataFrame | Styler:
        """
        All of the metrics are returned as a DataFrame.

        Args:
            dummy_classifier_strategy:
                if not None, then returns column(s) corresponding to the scores from predictions of
                sklearn.dummy.DummyClassifier, based on the strategy (or strategies) provided.
                Valid values correspond to values of `strategy` parameter listed
                https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyClassifier.html

                If a list is passed in (e.g. ['prior', 'uniform'], then one score column per value
                is added.

                If None is passed, then no additional columns are added.
            dummy_classifier_constant:
                The explicit constant as predicted by the “constant” strategy for the
                DummyClassifier.
                This parameter is useful only for the “constant” dummy_classifier_strategy.
            return_style:
                if True, return styler object; else return dataframe
            round_by:
                the number of digits to round by; if None, then don't round.
        """
        result = None
        last_key = list(self._evaluators.keys())[-1]

        for key, value in self._evaluators.items():
            dummy_strategy = dummy_classifier_strategy if key == last_key else None

            scores = value.all_metrics_df(
                return_explanations=False,
                dummy_classifier_strategy=dummy_strategy,
                dummy_classifier_constant=dummy_classifier_constant,
            )
            scores = scores.rename(columns={'Score': key})
            result = pd.concat([result, scores], axis=1)

        result = result.loc[[
            'AUC', 'F1 Score',
            'True Positive Rate', 'True Negative Rate',
            'False Positive Rate', 'False Negative Rate',
            'Positive Predictive Value', 'Negative Predictive Value',
        ]]

        result = result.transpose()

        if round_by:
            for column in result.columns:
                result[column] = result[column].round(round_by)

        if return_style:
            positive_scores = [x for x in result.columns if not x.startswith('False')]
            negative_scores = [x for x in result.columns if x.startswith('False')]

            result = result.style

            if round_by:
                result = result.format(precision=round_by)

            result = result. \
                bar(
                    subset=positive_scores, color=hcolor.Colors.PIGMENT_GREEN.value,
                    vmin=0, vmax=1,
                ). \
                bar(subset=negative_scores, color=hcolor.Colors.POPPY.value, vmin=0, vmax=1)

        return result

    def plot_metrics_comparison(
            self,
            dummy_classifier_strategy: str | list | None = 'prior',
            dummy_classifier_constant: int = 1) -> _figure.Figure:
        """
        Returns a Plotly object of a bar-chart of the metrics across all of the models.

        Args:
            dummy_classifier_strategy:
                if not None, then returns column(s) corresponding to the scores from predictions of
                sklearn.dummy.DummyClassifier, based on the strategy (or strategies) provided.
                Valid values correspond to values of `strategy` parameter listed
                https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyClassifier.html

                If a list is passed in (e.g. ['prior', 'uniform'], then one score column per value
                is added.

                If None is passed, then no additional columns are added.
            dummy_classifier_constant:
                The explicit constant as predicted by the “constant” strategy for the
                DummyClassifier.
                This parameter is useful only for the “constant” dummy_classifier_strategy.
        """
        score_data = self.all_metrics_df(
            dummy_classifier_strategy=dummy_classifier_strategy,
            dummy_classifier_constant=dummy_classifier_constant,
        ).transpose()

        score_data = score_data.reset_index()

        colors = [e.value for e in hcolor.Colors]
        fig = px.bar(
            data_frame=score_data.melt(id_vars='index'),
            y='variable',
            x='value',
            facet_col='index',
            facet_col_wrap=2,
            color='variable',
            color_discrete_sequence=colors,
            barmode='group',
            height=1000,
            labels={'index': 'Score'},
            title="Model Comparison",
        )
        fig.update_layout(showlegend=False)
        fig.update_yaxes(title=None)
        return fig

    def plot_roc_curves(self) -> _figure.Figure:
        """Returns a plotly object representing the ROC curves across all models."""
        result = None

        for key, value in self._evaluators.items():
            auc_data = value._get_auc_curve_dataframe()
            auc_data['Model'] = key
            result = pd.concat([result, auc_data], axis=0)

        colors = [e.value for e in hcolor.Colors]
        fig = px.line(
            data_frame=result,
            x='False Positive Rate',
            y='True Positive Rate',
            color='Model',
            color_discrete_sequence=colors,
            height=550,
            width=550 * GOLDEN_RATIO,
            custom_data=['threshold', 'Model'],
            title="ROC Curve of Models",
        )
        for index in range(len(self._evaluators)):
            scatter_1 = px.scatter(
                data_frame=result,
                x='False Positive Rate',
                y='True Positive Rate',
                color='Model',
                color_discrete_sequence=colors,
                custom_data=['threshold', 'Model'],
            )
            scatter_1.data[index]['showlegend'] = False
            fig.add_trace(scatter_1.data[index])
            query = f"threshold == 0.5 & Model == '{list(self._evaluators.keys())[index]}'"
            matches = result.query(query)
            matches = matches.reset_index(drop=True)
            scatter_2 = px.scatter(
                data_frame=matches,
                x='False Positive Rate',
                y='True Positive Rate',
                color='Model',
                color_discrete_sequence=[colors[index], *colors],
                custom_data=['threshold', 'Model'],
                size=[2],
            )
            scatter_2.data[0]['showlegend'] = False
            fig.add_trace(
                scatter_2.data[0],
            )

        fig.update_traces(
            hovertemplate="<br>".join([
                "Model: %{customdata[1]}<br><br>"
                "False Positive Rate: %{x}",
                "True Positive Rate: %{y}",
                "Threshold: %{customdata[0]}",
            ]),
        )
        return fig
