from dataclasses import dataclass
from os.path import dirname, join
from typing import Generic, Literal, TypeVar, overload

import numpy as np
import pandas as pd

from .._types import FloatNDArray

Frame = TypeVar('Frame', pd.DataFrame, FloatNDArray)
Series = TypeVar('Series', pd.Series, FloatNDArray)


@dataclass(frozen=True)
class Dataset(Generic[Frame, Series]):
    """
    Container object for datasets returned by the load functions.

    Attributes
    ----------
    data : :class:`pandas:pandas.DataFrame` or :class:`numpy:numpy.ndarray`
        Features of the dataset.
    target : :class:`pandas:pandas.Series` or :class:`numpy:numpy.ndarray`
        The classification labels.
    tp_cost : :class:`pandas:pandas.Series`, :class:`numpy:numpy.ndarray` or float
        The cost of true positives.
    tn_cost : :class:`pandas:pandas.Series`, :class:`numpy:numpy.ndarray` or float
        The cost of true negatives.
    fp_cost : :class:`pandas:pandas.Series`, :class:`numpy:numpy.ndarray` or float
        The cost of false positives.
    fn_cost : :class:`pandas:pandas.Series`, :class:`numpy:numpy.ndarray` or float
        The cost of false negatives.
    feature_names : :class:`pandas:pandas.Series` or :class:`numpy:numpy.ndarray`
        The meaning of the features.
    target_names : :class:`pandas:pandas.Series` or :class:`numpy:numpy.ndarray`
        The meaning of the labels.
    name : str
        The name of the dataset.
    DESCR : str
        The full description of the dataset
    """

    data: Frame
    target: Series
    tp_cost: Series | float
    tn_cost: Series | float
    fp_cost: Series | float
    fn_cost: Series | float
    feature_names: Series
    target_names: Series
    name: str
    DESCR: str


@overload
def load_churn_tv_subscriptions(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[False] = False,
) -> Dataset[pd.DataFrame, pd.Series]: ...


@overload
def load_churn_tv_subscriptions(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[False] = False,
) -> Dataset[FloatNDArray, FloatNDArray]: ...


@overload
def load_churn_tv_subscriptions(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[True],
) -> tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]: ...


@overload
def load_churn_tv_subscriptions(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[True],
) -> tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]: ...


def load_churn_tv_subscriptions(
    *, as_frame: bool = False, return_X_y_costs: bool = False
) -> (
    Dataset[pd.DataFrame, pd.Series]
    | Dataset[FloatNDArray, FloatNDArray]
    | tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]
    | tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]
):
    """
    Load the TV Subscription Churn dataset (binary classification).

    The goal is to predict whether a customer will churn or not.
    The target variable is whether the customer churned, 'yes' = 1 and 'no' = 0.

    This dataset is from a TV cable provider containing all 9410 customers active during the first semester of 2014.
    Features names are anonymized to protect the privacy of the customers.

    For additional information about the dataset, consult the :ref:`User Guide <churn_tv_subscriptions>`.

    =================   ==============
    Classes                          2
    Churners                       455
    Non-churners                  8955
    Samples                       9410
    Features                        45
    =================   ==============

    Parameters
    ----------
    as_frame : bool, default=False
        If True, the output will be a pandas DataFrames or Series instead of numpy arrays.
    return_X_y_costs : bool, default=False
        If True, return (data, target, tp_cost, fp_cost, tn_cost, fn_cost) instead of a Dataset object.

    Returns
    -------
    dataset : :class:`~empulse.datasets.Dataset` or tuple of (data, target, tp_cost, fp_cost, tn_cost, fn_cost)
        Returns a Dataset object if `return_X_y_costs=False` (default), otherwise a tuple.

    Notes
    -----
    Cost matrix

    .. list-table::

        * -
          - Actual positive :math:`y_i = 1`
          - Actual negative :math:`y_i = 0`
        * - Predicted positive :math:`\\hat{y}_i = 1`
          - ``tp_cost`` :math:`= \\gamma_i d_i + (1 - \\gamma_i) (CLV_i + c_i)`
          - ``fp_cost`` :math:`= d_i + c_i`
        * - Predicted negative :math:`\\hat{y}_i = 0`
          - ``fn_cost`` :math:`= CLV_i`
          - ``tn_cost`` :math:`= 0`

    with
        - :math:`\\gamma_i` : probability of the customer accepting the retention offer
        - :math:`CLV_i` : customer lifetime value of the retained customer
        - :math:`d_i` : cost of incentive offered to the customer
        - :math:`c_i` : cost of contacting the customer

    References
    ----------
    .. [1] A. Correa Bahnsen, D.Aouada, B, Ottersten,
           `"A novel cost-sensitive framework for customer churn predictive modeling"
           <http://www.decisionanalyticsjournal.com/content/pdf/s40165-015-0014-6.pdf>`__,
           Decision Analytics, 2:5, 2015.

    Examples
    --------

    .. code-block:: python

        from empulse.datasets import load_churn_tv_subscriptions
        from sklearn.model_selection import train_test_split

        dataset = load_churn_tv_subscriptions()
        X_train, X_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, random_state=42
        )
    """
    module_path = dirname(__file__)
    raw_data = pd.read_csv(
        join(module_path, 'data', 'churn_tv_subscriptions.csv.gz'), delimiter=',', compression='gzip'
    )
    with open(join(module_path, 'descriptions', 'churn_tv_subscriptions.rst'), encoding='utf-8') as f:
        description = f.read()
    data = raw_data.iloc[:, 1:-5]

    if return_X_y_costs:
        if as_frame:
            return data, raw_data.target, raw_data['C_TP'], raw_data['C_FP'], raw_data['C_TN'], raw_data['C_FN']
        else:
            return (
                data.to_numpy(),
                raw_data.target.to_numpy().astype(np.int8),
                raw_data['C_TP'].to_numpy(),
                raw_data['C_FP'].to_numpy(),
                raw_data['C_TN'].to_numpy(),
                raw_data['C_FN'].to_numpy(),
            )
    else:
        return Dataset(
            data=data.to_numpy() if not as_frame else data,
            target=raw_data.target.to_numpy().astype(np.int8) if not as_frame else raw_data['target'],
            tp_cost=raw_data.C_TP.to_numpy() if not as_frame else raw_data['C_TP'],
            fp_cost=raw_data.C_FP.to_numpy() if not as_frame else raw_data['C_FP'],
            tn_cost=raw_data.C_TN.to_numpy() if not as_frame else raw_data['C_TN'],
            fn_cost=raw_data.C_FN.to_numpy() if not as_frame else raw_data['C_FN'],
            feature_names=data.columns.to_numpy() if not as_frame else data.columns,
            target_names=np.array(['no churn', 'churn']) if not as_frame else pd.Series(['no churn', 'churn']),
            name='Churn TV subscriptions',
            DESCR=description,
        )


@overload
def load_upsell_bank_telemarketing(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[False] = False,
    interest_rate: float = 0.02463333,
    term_deposit_fraction: float = 0.25,
    contact_cost: float = 1,
) -> Dataset[pd.DataFrame, pd.Series]: ...


@overload
def load_upsell_bank_telemarketing(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[False] = False,
    interest_rate: float = 0.02463333,
    term_deposit_fraction: float = 0.25,
    contact_cost: float = 1,
) -> Dataset[FloatNDArray, FloatNDArray]: ...


@overload
def load_upsell_bank_telemarketing(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[True],
    interest_rate: float = 0.02463333,
    term_deposit_fraction: float = 0.25,
    contact_cost: float = 1,
) -> tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]: ...


@overload
def load_upsell_bank_telemarketing(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[True],
    interest_rate: float = 0.02463333,
    term_deposit_fraction: float = 0.25,
    contact_cost: float = 1,
) -> tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]: ...


def load_upsell_bank_telemarketing(
    *,
    as_frame: bool = False,
    return_X_y_costs: bool = False,
    interest_rate: float = 0.02463333,
    term_deposit_fraction: float = 0.25,
    contact_cost: float = 1,
) -> (
    Dataset[pd.DataFrame, pd.Series]
    | Dataset[FloatNDArray, FloatNDArray]
    | tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]
    | tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]
):
    """
    Load the bank telemarketing dataset (binary classification).

    The goal is to predict whether a client will subscribe to a term deposit after being called by the bank.
    The target variable is whether the client subscribed to the term deposit, 'yes' = 1 and 'no' = 0.

    The dataset is related to a direct marketing campaigns (phone calls) of a Portuguese banking institution.
    The marketing campaigns were based on phone calls.
    Often, more than one contact to the same client was required,
    in order to access if the product (bank term deposit) would be or not subscribed.

    Features recorded before the contact event are removed from the original dataset [1]_ to avoid data leakage.
    Only clients with a positive balance are considered, since clients in debt are not eligible for term deposits.

    For a full data description and additional information about the dataset,
    consult the :ref:`User Guide <upsell_bank_telemarketing>`.

    =================   ==============
    Classes                          2
    Subscribers                   4787
    Non-subscribers              33144
    Samples                      37931
    Features                        10
    =================   ==============

    Parameters
    ----------
    as_frame : bool, default=False
        If True, the output will be a pandas DataFrames or Series instead of numpy arrays.
    return_X_y_costs : bool, default=False
        If True, return (data, target, tp_cost, fp_cost, tn_cost, fn_cost) instead of a Dataset object.
    interest_rate : float, default=0.02463333
        Interest rate of the term deposit.
    term_deposit_fraction : float, default=0.25
        Fraction of the client's balance that is deposited in the term deposit.
    contact_cost : float, default=1
        Cost of contacting the client.

    Returns
    -------
    dataset : :class:`~empulse.datasets.Dataset` or tuple of (data, target, tp_cost, fp_cost, tn_cost, fn_cost)
        Returns a Dataset object if `return_X_y_costs=False` (default), otherwise a tuple.

    Notes
    -----
    Cost matrix

    .. list-table::

        * -
          - Actual positive :math:`y_i = 1`
          - Actual negative :math:`y_i = 0`
        * - Predicted positive :math:`\\hat{y}_i = 1`
          - ``tp_cost`` :math:`= c`
          - ``fp_cost`` :math:`= c`
        * - Predicted negative :math:`\\hat{y}_i = 0`
          - ``fn_cost`` :math:`= r \\, d_i \\, b_i`
          - ``tn_cost`` :math:`= 0`

    with
        - :math:`c` : cost of contacting the client
        - :math:`r` : interest rate of the term deposit
        - :math:`d_i` : fraction of the client's balance that is deposited in the term deposit
        - :math:`b_i` : client's balance

    Using default parameters, it is assumed that :math:`c = 1`, :math:`r = 0.02463333`, :math:`d_i = 0.25`
    for all clients.

    References
    ----------
    .. [1] Moro, S., Rita, P., & Cortez, P. (2014).
           Bank Marketing [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C5K306.

    .. [2] S. Moro, R. Laureano and P. Cortez.
           Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology.
           In P. Novais et al. (Eds.),
           Proceedings of the European Simulation and Modelling Conference
           - ESM'2011, pp. 117-121, Guimaraes, Portugal, October, 2011. EUROSIS. [bank.zip]

    .. [3] A. Correa Bahnsen, A. Stojanovic, D.Aouada, B, Ottersten,
           `"Improving Credit Card Fraud Detection with Calibrated Probabilities"
           <http://albahnsen.com/files/%20Improving%20Credit%20Card%20Fraud%20Detection%20by%20using%20Calibrated%20Probabilities%20-%20Publish.pdf>`__,
           in Proceedings of the fourteenth SIAM International Conference on Data Mining,
           677-685, 2014.

    Examples
    --------

    .. code-block:: python

        from empulse.datasets import load_upsell_bank_telemarketing
        from sklearn.model_selection import train_test_split

        dataset = load_upsell_bank_telemarketing()
        X_train, X_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, random_state=42
        )

    """
    module_path = dirname(__file__)
    raw_data = pd.read_csv(join(module_path, 'data', 'bankmarketing.csv.gz'), delimiter=';', compression='gzip')
    with open(join(module_path, 'descriptions', 'bankmarketing.rst'), encoding='utf-8') as f:
        description = f.read()

    # only use features pre-contact:
    # 1 - age (numeric)
    # 2 - job : type of job (categorical: "admin.","unknown","unemployed","management","housemaid","entrepreneur",
    #                        "student","blue-collar","self-employed","retired","technician","services")
    # 3 - marital : marital status (categorical: "married","divorced","single";
    #                               note: "divorced" means divorced or widowed)
    # 4 - education (categorical: "unknown","secondary","primary","tertiary")
    # 5 - default: has credit in default? (binary: "yes","no")
    # 6 - balance: average yearly balance, in euros (numeric)
    # 7 - housing: has housing loan? (binary: "yes","no")
    # 8 - loan: has personal loan? (binary: "yes","no")
    # 15 - previous: number of contacts performed before this campaign and for this client (numeric)
    # 16 - poutcome: outcome of the previous marketing campaign (categorical: "unknown","other","failure","success")

    # Following features exclude because are collected after the contact event
    # # related with the last contact of the current campaign:
    # 9 - contact: contact communication type (categorical: "unknown","telephone","cellular")
    # 10 - day: last contact day of the month (numeric)
    # 11 - month: last contact month of year (categorical: "jan", "feb", "mar", ..., "nov", "dec")
    # 12 - duration: last contact duration, in seconds (numeric)
    # # other attributes:
    # 13 - campaign: number of contacts performed during this campaign and for this client
    # 14 - pdays: number of days that passed by after the client was last contacted from a
    #       previous campaign (numeric, -1 means client was not previously contacted)

    # Filter if balance>0
    raw_data = raw_data.loc[raw_data['balance'] > 0]
    target = (raw_data.y.to_numpy() == 'yes').astype(np.int8)
    data = raw_data[
        ['age', 'balance', 'previous', 'job', 'marital', 'education', 'default', 'housing', 'loan', 'poutcome']
    ]

    fp_cost = contact_cost
    fn_cost = np.maximum(data['balance'].to_numpy() * interest_rate * term_deposit_fraction, contact_cost)
    tp_cost = contact_cost
    tn_cost = 0.0

    data.loc[:, 'default'] = data['default'].map({'yes': 1, 'no': 0})
    data.loc[:, 'housing'] = data['housing'].map({'yes': 1, 'no': 0})
    data.loc[:, 'loan'] = data['loan'].map({'yes': 1, 'no': 0})

    data = data.astype({
        'age': np.uint8,
        'balance': np.int32,
        'previous': np.uint8,
        'job': 'category',
        'marital': 'category',
        'education': 'category',
        'default': np.uint8,
        'housing': np.uint8,
        'loan': np.uint8,
        'poutcome': 'category',
    })

    data = data.rename(
        columns={
            'poutcome': 'previous_outcome',
            'loan': 'has_personal_loan',
            'housing': 'has_housing_loan',
            'default': 'has_credit_in_default',
        }
    )

    if return_X_y_costs:
        if as_frame:
            return (
                data,
                pd.Series(target, name='subscription'),
                tp_cost,
                fp_cost,
                tn_cost,
                pd.Series(fn_cost, name='fn_cost'),
            )
        else:
            return (data.to_numpy(), target, tp_cost, fp_cost, tn_cost, fn_cost)
    else:
        target_names = ['no subscription', 'subscription']
        return Dataset(
            data=data.to_numpy() if not as_frame else data,
            target=target if not as_frame else pd.Series(target, name='subscription'),
            tp_cost=tp_cost,
            fp_cost=fp_cost,
            tn_cost=tn_cost,
            fn_cost=fn_cost if not as_frame else pd.Series(fn_cost, name='fn_cost'),
            feature_names=data.columns.to_numpy() if not as_frame else data.columns,
            target_names=np.array(target_names) if not as_frame else pd.Series(target_names, name='target'),
            name='Bank Telemarketing',
            DESCR=description,
        )


@overload
def load_give_me_some_credit(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[False] = False,
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> Dataset[pd.DataFrame, pd.Series]: ...


@overload
def load_give_me_some_credit(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[False] = False,
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> Dataset[FloatNDArray, FloatNDArray]: ...


@overload
def load_give_me_some_credit(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[True],
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]: ...


@overload
def load_give_me_some_credit(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[True],
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]: ...


def load_give_me_some_credit(
    *,
    as_frame: bool = False,
    return_X_y_costs: bool = False,
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> (
    Dataset[pd.DataFrame, pd.Series]
    | Dataset[FloatNDArray, FloatNDArray]
    | tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]
    | tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]
):
    """
    Load the "Give Me Some Credit" Kaggle credit scoring competition dataset (binary classification).

    The goal is to predict whether a customer will default on a loan in the next two years.
    The target variable is whether the customer defaulted, 'yes' = 1 and 'no' = 0.

    Only customers with a positive monthly income and a debt ratio less than 1 are considered.

    For a full data description and additional information about the dataset,
    consult the :ref:`User Guide <give_me_some_credit>`.

    =================   ==============
    Classes                          2
    Defaulters                    7616
    Non-defaulters              105299
    Samples                     112915
    Features                        10
    =================   ==============

    Parameters
    ----------
    as_frame : bool, default=False
        If True, the output will be a pandas DataFrames or Series instead of numpy arrays.
    return_X_y_costs : bool, default=False
        If True, return (data, target, tp_cost, fp_cost, tn_cost, fn_cost) instead of a Dataset object.
    interest_rate : float, default=0.02463333
        Annual interest rate of the term deposit.
    fund_cost : float, default=0.0294
        Annual cost of funds.
    max_credit_line : float, default=25000
        The maximum amount a client can borrow.
    loss_given_default : float, default=0.75
        The fraction of the loan amount which is lost if the client defaults.
    term_length_months : int, default=24
        The length of the loan term in months.
    loan_to_income_ratio : float, default=3
        The ratio of the loan amount to the client's income.

    Returns
    -------
    dataset : :class:`~empulse.datasets.Dataset` or tuple of (data, target, tp_cost, fp_cost, tn_cost, fn_cost)
        Returns a Dataset object if `return_X_y_costs=False` (default), otherwise a tuple.

    Notes
    -----
    Cost matrix

    .. list-table::

        * -
          - Actual positive :math:`y_i = 1`
          - Actual negative :math:`y_i = 0`
        * - Predicted positive :math:`\\hat{y}_i = 1`
          - ``tp_cost`` :math:`= 0`
          - ``fp_cost`` :math:`= r_i + -\\bar{r} \\cdot \\pi_0 + \\bar{Cl} \\cdot L_{gd} \\cdot \\pi_1`
        * - Predicted negative :math:`\\hat{y}_i = 0`
          - ``fn_cost`` :math:`= Cl_i \\cdot L_{gd}`
          - ``tn_cost`` :math:`= 0`

    with
        - :math:`r_i` : loss in profit by rejecting what would have been a good loan
        - :math:`\\bar{r}` : average loss in profit by rejecting what would have been a good loan
        - :math:`\\pi_0` : percentage of defaulters
        - :math:`\\pi_1` : percentage of non-defaulters
        - :math:`Cl_i` : credit line of the client
        - :math:`\\bar{Cl}` : average credit line
        - :math:`L_{gd}` : the fraction of the loan amount which is lost if the client defaults

    References
    ----------
    .. [1] A. Correa Bahnsen, D.Aouada, B, Ottersten,
           "Example-Dependent Cost-Sensitive Logistic Regression for Credit Scoring",
           in Proceedings of the International Conference on Machine Learning and Applications, 2014.

    Examples
    --------

    .. code-block:: python

        from empulse.datasets import load_give_me_some_credit
        from sklearn.model_selection import train_test_split

        dataset = load_give_me_some_credit()
        X_train, X_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, random_state=42
        )

    """
    module_path = dirname(__file__)
    raw_data = pd.read_csv(join(module_path, 'data', 'creditscoring1.csv.gz'), delimiter=',', compression='gzip')
    with open(join(module_path, 'descriptions', 'creditscoring1.rst'), encoding='utf-8') as f:
        description = f.read()

    # Exclude MonthlyIncome = nan or =0 or DebtRatio >1
    raw_data = raw_data.dropna()
    raw_data = raw_data.loc[(raw_data['MonthlyIncome'] > 0)]
    raw_data = raw_data.loc[(raw_data['DebtRatio'] < 1)]

    target = raw_data['SeriousDlqin2yrs'].to_numpy().astype(np.int64)

    data = raw_data.drop(['SeriousDlqin2yrs', 'id'], axis=1)

    # Calculate cost_mat (see[1])
    cost_mat_parameters = {
        'int_r': interest_rate / 12,
        'int_cf': fund_cost / 12,
        'cl_max': max_credit_line,
        'n_term': term_length_months,
        'k': loan_to_income_ratio,
        'lgd': loss_given_default,
    }

    pi_1 = target.mean()
    # cost_mat[FP,FN,TP,TN]
    cost_mat = _creditscoring_costmat(
        data['MonthlyIncome'].to_numpy(), data['DebtRatio'].to_numpy(), pi_1, cost_mat_parameters
    )

    # unroll into separate costs
    fp_cost = cost_mat[:, 0]
    fn_cost = cost_mat[:, 1]

    # normalize feature names
    column_mapping = {
        'RevolvingUtilizationOfUnsecuredLines': 'revolving_utilization',
        'age': 'age',
        'NumberOfTime30-59DaysPastDueNotWorse': 'n_times_late_30_59_days',
        'DebtRatio': 'debt_ratio',
        'MonthlyIncome': 'monthly_income',
        'NumberOfOpenCreditLinesAndLoans': 'n_open_credit_lines',
        'NumberOfTimes90DaysLate': 'n_times_late_over_90_days',
        'NumberRealEstateLoansOrLines': 'n_real_estate_loans',
        'NumberOfTime60-89DaysPastDueNotWorse': 'n_times_late_60_89_days',
        'NumberOfDependents': 'n_dependents',
    }

    data = data.rename(columns=column_mapping)

    new_order = [
        'monthly_income',
        'debt_ratio',
        'revolving_utilization',
        'age',
        'n_dependents',
        'n_open_credit_lines',
        'n_real_estate_loans',
        'n_times_late_30_59_days',
        'n_times_late_60_89_days',
        'n_times_late_over_90_days',
    ]

    # Reorder columns
    data = data.reindex(columns=new_order)

    data = data.astype({
        'monthly_income': np.float64,
        'debt_ratio': np.float64,
        'revolving_utilization': np.float64,
        'age': np.uint8,
        'n_dependents': np.uint8,
        'n_open_credit_lines': np.uint8,
        'n_real_estate_loans': np.uint8,
        'n_times_late_30_59_days': np.uint8,
        'n_times_late_60_89_days': np.uint8,
        'n_times_late_over_90_days': np.uint8,
    })

    if return_X_y_costs:
        if as_frame:
            return (
                data,
                pd.Series(target, name='default'),
                0.0,
                pd.Series(fp_cost, name='fp_cost'),
                0.0,
                pd.Series(fn_cost, name='fn_cost'),
            )
        else:
            return (data.to_numpy(), target, 0.0, fp_cost, 0.0, fn_cost)
    else:
        target_names = ['no default', 'default']
        return Dataset(
            data=data.to_numpy() if not as_frame else data,
            target=target if not as_frame else pd.Series(target, name='default'),
            tp_cost=0.0,
            fp_cost=fp_cost if not as_frame else pd.Series(fp_cost, name='fp_cost'),
            tn_cost=0.0,
            fn_cost=fn_cost if not as_frame else pd.Series(fn_cost, name='fn_cost'),
            feature_names=data.columns.to_numpy() if not as_frame else data.columns,
            target_names=np.array(target_names) if not as_frame else pd.Series(target_names, name='target'),
            name='Give Me Some Credit',
            DESCR=description,
        )


@overload
def load_credit_scoring_pakdd(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[False] = False,
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> Dataset[pd.DataFrame, pd.Series]: ...


@overload
def load_credit_scoring_pakdd(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[False] = False,
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> Dataset[FloatNDArray, FloatNDArray]: ...


@overload
def load_credit_scoring_pakdd(
    *,
    as_frame: Literal[True],
    return_X_y_costs: Literal[True],
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]: ...


@overload
def load_credit_scoring_pakdd(
    *,
    as_frame: Literal[False] = False,
    return_X_y_costs: Literal[True],
    interest_rate: float = 0.0479,
    fund_cost: float = 0.0294,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]: ...


def load_credit_scoring_pakdd(
    *,
    as_frame: bool = False,
    return_X_y_costs: bool = False,
    interest_rate: float = 0.63,
    fund_cost: float = 0.165,
    max_credit_line: float = 25000,
    loss_given_default: float = 0.75,
    term_length_months: int = 24,
    loan_to_income_ratio: float = 3,
) -> (
    Dataset[pd.DataFrame, pd.Series]
    | Dataset[FloatNDArray, FloatNDArray]
    | tuple[pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]
    | tuple[FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray, FloatNDArray]
):
    """
    Load the credit scoring PAKDD 2009 competition dataset (binary classification).

    The goal is to predict whether a customer will default on a loan in the next two years.
    The target variable is whether the customer defaulted, 'yes' = 1 and 'no' = 0.

    Only clients with a personal income between 100 and 10000 are considered.

    For a full data description and additional information about the dataset,
    consult the :ref:`User Guide <credit_scoring_pakdd>`.

    =================   ==============
    Classes                          2
    Defaulters                    7743
    Non-defaulters               31195
    Samples                      38938
    Features                        25
    =================   ==============

    Parameters
    ----------
    as_frame : bool, default=False
        If True, the output will be a pandas DataFrames or Series instead of numpy arrays.
    return_X_y_costs : bool, default=False
        If True, return (data, target, tp_cost, fp_cost, tn_cost, fn_cost) instead of a Dataset object.
    interest_rate : float, default=0.63
        Annual interest rate of the term deposit.
    fund_cost : float, default=0.165
        Annual cost of funds.
    max_credit_line : float, default=25000
        The maximum amount a client can borrow.
    loss_given_default : float, default=0.75
        The amount of the loan amount which is lost if the client defaults.
    term_length_months : int, default=24
        The length of the loan term in months.
    loan_to_income_ratio : float, default=3
        The ratio of the loan amount to the client's income.

    Returns
    -------
    dataset : :class:`~empulse.datasets.Dataset` or tuple of (data, target, tp_cost, fp_cost, tn_cost, fn_cost)
        Returns a Dataset object if `return_X_y_costs=False` (default), otherwise a tuple.

    Notes
    -----
    Cost matrix

    .. list-table::

        * -
          - Actual positive :math:`y_i = 1`
          - Actual negative :math:`y_i = 0`
        * - Predicted positive :math:`\\hat{y}_i = 1`
          - ``tp_cost`` :math:`= 0`
          - ``fp_cost`` :math:`= r_i + -\\bar{r} \\cdot \\pi_0 + \\bar{Cl} \\cdot L_{gd} \\cdot \\pi_1`
        * - Predicted negative :math:`\\hat{y}_i = 0`
          - ``fn_cost`` :math:`= Cl_i \\cdot L_{gd}`
          - ``tn_cost`` :math:`= 0`

    with
        - :math:`r_i` : loss in profit by rejecting what would have been a good loan
        - :math:`\\bar{r}` : average loss in profit by rejecting what would have been a good loan
        - :math:`\\pi_0` : percentage of defaulters
        - :math:`\\pi_1` : percentage of non-defaulters
        - :math:`Cl_i` : credit line of the client
        - :math:`\\bar{Cl}` : average credit line
        - :math:`L_{gd}` : the fraction of the loan amount which is lost if the client defaults

    References
    ----------
    .. [1] A. Correa Bahnsen, D.Aouada, B, Ottersten,
           "Example-Dependent Cost-Sensitive Logistic Regression for Credit Scoring",
           in Proceedings of the International Conference on Machine Learning and Applications,
           , 2014.

    Examples
    --------

    .. code-block:: python

        from empulse.datasets import load_credit_scoring_pakdd
        from sklearn.model_selection import train_test_split

        dataset = load_credit_scoring_pakdd()
        X_train, X_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, random_state=42
        )

    """
    module_path = dirname(__file__)
    raw_data = pd.read_csv(join(module_path, 'data', 'creditscoring2.csv.gz'), delimiter='\t', compression='gzip')
    with open(join(module_path, 'descriptions', 'creditscoring2.rst'), encoding='utf-8') as f:
        description = f.read()

    # Exclude TARGET_LABEL_BAD=1 == 'N'
    raw_data = raw_data.loc[raw_data['TARGET_LABEL_BAD=1'] != 'N']

    # Exclude 100<PERSONAL_NET_INCOME<10000
    raw_data = raw_data.loc[(raw_data['PERSONAL_NET_INCOME'].to_numpy().astype(np.float64) > 100)]
    raw_data = raw_data.loc[(raw_data['PERSONAL_NET_INCOME'].to_numpy().astype(np.float64) < 10000)]

    target = raw_data['TARGET_LABEL_BAD=1'].to_numpy().astype(np.int64)
    data = raw_data.drop(['TARGET_LABEL_BAD=1'], axis=1)

    # drop the last column
    data = data.iloc[:, :-1]

    continuous_columns = ['MATE_INCOME', 'PERSONAL_NET_INCOME']
    integer_columns = [
        'ID_SHOP',
        'AGE',
        'AREA_CODE_RESIDENCIAL_PHONE',
        'PAYMENT_DAY',
        'SHOP_RANK',
        'MONTHS_IN_RESIDENCE',
        'MONTHS_IN_THE_JOB',
        'PROFESSION_CODE',
        'QUANT_ADDITIONAL_CARDS_IN_THE_APPLICATION',
    ]
    data[continuous_columns] = data[continuous_columns].astype('float64')
    data[integer_columns] = data[integer_columns].astype('int32')

    # Calculate cost_mat (see[1])
    cost_mat_parameters = {
        'int_r': interest_rate / 12,
        'int_cf': fund_cost / 12,
        'cl_max': max_credit_line * 0.33,
        'n_term': term_length_months,
        'k': loan_to_income_ratio,
        'lgd': loss_given_default,
    }

    n_samples = data.shape[0]
    pi_1 = target.mean()
    monthly_income = data['PERSONAL_NET_INCOME'].to_numpy() * 0.33
    cost_mat = _creditscoring_costmat(monthly_income, np.zeros(n_samples), pi_1, cost_mat_parameters)

    # unroll into separate costs
    fp_cost = cost_mat[:, 0]
    fn_cost = cost_mat[:, 1]

    # convert all columns which start with flag to integers
    flag_columns = [col for col in data.columns if col.startswith('FLAG')]
    data[flag_columns] = (data[flag_columns] == 'Y').astype(np.int8)

    # normalize feature names
    data.columns = data.columns.str.lower().str.replace('#', '').str.replace('quant', 'n')
    data.columns = data.columns.str.replace('_in_the_application', '').str.replace('residencial', 'residential')
    data.loc[:, 'sex'] = data['sex'].map({'M': 1, 'F': 0})
    # fill in single  missing value as male
    data.loc[:, 'sex'] = data.sex.fillna('1')

    column_mapping = {
        'flag_residence_town_eq_working_town': 'lives_in_work_town',
        'flag_residence_state_eq_working_state': 'lives_in_work_state',
        'flag_residential_address_eq_postal_address': 'has_same_postal_address',
        'flag_residential_phone': 'has_residential_phone',
        'sex': 'is_male',
        'flag_mothers_name': 'filled_in_mothers_name',
        'flag_fathers_name': 'filled_in_fathers_name',
        'mate_income': 'partner_income',
        'flag_other_card': 'has_other_card',
        'flag_mobile_phone': 'has_mobile_phone',
        'flag_contact_phone': 'has_contact_phone',
        'cod_application_booth': 'application_booth_code',
        'flag_card_insurance_option': 'has_card_insurance',
        'id_shop': 'shop_code',
    }
    data = data.rename(columns=column_mapping)

    # remap values of matiral_status to more readable values
    data.loc[:, 'marital_status'] = data['marital_status'].map({
        'S': 'single',
        'M': 'married',
        'D': 'divorced',
        'W': 'widowed',
        'O': 'other',
    })
    # remap values of residence_type to more readable values
    data.loc[:, 'residence_type'] = data['residence_type'].map({
        'P': 'owned',
        'A': 'rented',
        'C': 'parents',
        'O': 'other',
    })

    # Desired column order
    new_order = [
        'age',
        'personal_net_income',
        'partner_income',
        'months_in_residence',
        'months_in_the_job',
        'payment_day',
        'n_banking_accounts',
        'n_additional_cards',
        'is_male',
        'has_residential_phone',
        'has_mobile_phone',
        'has_contact_phone',
        'has_same_postal_address',
        'has_other_card',
        'has_card_insurance',
        'lives_in_work_town',
        'lives_in_work_state',
        'filled_in_mothers_name',
        'filled_in_fathers_name',
        'shop_rank',
        'marital_status',
        'residence_type',
        'area_code_residential_phone',
        'shop_code',
        'application_booth_code',
        'profession_code',
    ]

    # Reorder columns
    data = data.reindex(columns=new_order)

    data = data.astype({
        'age': np.uint8,
        'personal_net_income': np.float32,
        'partner_income': np.float32,
        'months_in_residence': np.uint8,
        'months_in_the_job': np.uint8,
        'payment_day': np.uint8,
        'n_banking_accounts': np.uint8,
        'n_additional_cards': np.uint8,
        'is_male': np.uint8,
        'has_residential_phone': np.uint8,
        'has_mobile_phone': np.uint8,
        'has_contact_phone': np.uint8,
        'has_same_postal_address': np.uint8,
        'has_other_card': np.uint8,
        'has_card_insurance': np.uint8,
        'lives_in_work_town': np.uint8,
        'lives_in_work_state': np.uint8,
        'filled_in_mothers_name': np.uint8,
        'filled_in_fathers_name': np.uint8,
        'shop_rank': 'category',
        'marital_status': 'category',
        'residence_type': 'category',
        'area_code_residential_phone': 'category',
        'shop_code': 'category',
        'application_booth_code': 'category',
        'profession_code': 'category',
    })

    # remove flag_card_insurance_option
    data = data.drop(['has_card_insurance'], axis=1)

    if return_X_y_costs:
        if as_frame:
            return (
                data,
                pd.Series(target, name='default'),
                0.0,
                pd.Series(fp_cost, name='fp_cost'),
                0.0,
                pd.Series(fn_cost, name='fn_cost'),
            )
        else:
            return (data.to_numpy(), target, 0.0, fp_cost, 0.0, fn_cost)
    else:
        target_names = ['no default', 'default']
        return Dataset(
            data=data.to_numpy() if not as_frame else data,
            target=target if not as_frame else pd.Series(target, name='default'),
            tp_cost=0.0,
            fp_cost=fp_cost if not as_frame else pd.Series(fp_cost, name='fp_cost'),
            tn_cost=0.0,
            fn_cost=fn_cost if not as_frame else pd.Series(fn_cost, name='fn_cost'),
            feature_names=data.columns.to_numpy() if not as_frame else data.columns,
            target_names=np.array(target_names) if not as_frame else pd.Series(target_names, name='target'),
            name='Credit Scoring PAKDD 2009',
            DESCR=description,
        )


def _creditscoring_costmat(
    income: FloatNDArray, debt: FloatNDArray, pi_1: float, cost_mat_parameters: dict[str, float]
) -> FloatNDArray:
    """Private function to calculate the cost matrix of credit scoring models.

    Parameters
    ----------
    income : array of shape = [n_samples]
        Monthly income of each example

    debt : array of shape = [n_samples]
        Debt ratio each example

    pi_1 : float
        Percentage of positives in the training set

    References
    ----------
    .. [1] A. Correa Bahnsen, D.Aouada, B, Ottersten,
           "Example-Dependent Cost-Sensitive Logistic Regression for Credit Scoring",
           in Proceedings of the International Conference on Machine Learning and Applications,
           , 2014.

    Returns
    -------
    cost_mat : array-like of shape = [n_samples, 4]
        Cost matrix of the classification problem
        Where the columns represent the costs of: false positives, false negatives,
        true positives and true negatives, for each example.
    """

    def calculate_a(cl_i: float, int_: float, n_term: float) -> float:
        return float(cl_i * ((int_ * (1 + int_) ** n_term) / ((1 + int_) ** n_term - 1)))

    def calculate_pv(a: float, int_: float, n_term: float) -> float:
        return float(a / int_ * (1 - 1 / (1 + int_) ** n_term))

    # Calculate credit line Cl
    def calculate_cl(k: float, inc_i: float, cl_max: float, debt_i: float, int_r: float, n_term: float) -> float:
        cl_k = k * inc_i
        a = calculate_a(cl_k, int_r, n_term)
        cl_debt = calculate_pv(inc_i * min(a / inc_i, 1 - debt_i), int_r, n_term)
        return min(cl_k, cl_max, cl_debt)

    # calculate costs
    def calculate_cost_fn(cl_i: float, lgd: float) -> float:
        return cl_i * lgd

    def calculate_cost_fp(
        cl_i: float, int_r: float, n_term: float, int_cf: float, pi_1: float, lgd: float, cl_avg: float
    ) -> float:
        a = calculate_a(cl_i, int_r, n_term)
        pv = calculate_pv(a, int_cf, n_term)
        r = pv - cl_i
        r_avg = calculate_pv(calculate_a(cl_avg, int_r, n_term), int_cf, n_term) - cl_avg
        cost_fp = r - (1 - pi_1) * r_avg + pi_1 * calculate_cost_fn(cl_avg, lgd)
        return max(0, cost_fp)

    v_calculate_cost_fp = np.vectorize(calculate_cost_fp)
    v_calculate_cost_fn = np.vectorize(calculate_cost_fn)

    v_calculate_cl = np.vectorize(calculate_cl)

    # Parameters
    k = cost_mat_parameters['k']
    int_r = cost_mat_parameters['int_r']
    n_term = cost_mat_parameters['n_term']
    int_cf = cost_mat_parameters['int_cf']
    lgd = cost_mat_parameters['lgd']
    cl_max = cost_mat_parameters['cl_max']

    cl = v_calculate_cl(k, income, cl_max, debt, int_r, n_term)
    cl_avg = cl.mean()

    n_samples = income.shape[0]
    cost_mat = np.zeros((n_samples, 4))  # cost_mat[FP,FN,TP,TN]
    cost_mat[:, 0] = v_calculate_cost_fp(cl, int_r, n_term, int_cf, pi_1, lgd, cl_avg)
    cost_mat[:, 1] = v_calculate_cost_fn(cl, lgd)
    cost_mat[:, 2] = 0.0
    cost_mat[:, 3] = 0.0

    return cost_mat
