from sklearn.ensemble import (
    RandomForestClassifier,
    GradientBoostingClassifier,
    AdaBoostClassifier,
    BaggingClassifier,
)
from sklearn.svm import SVC,SVR
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.linear_model import (
    LassoCV,
    LogisticRegression,LinearRegression,
    Lasso,
    Ridge,
    RidgeClassifierCV,
    ElasticNet,
)
from sklearn.feature_selection import RFE
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
import xgboost as xgb  # Make sure you have xgboost installed

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import (
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    roc_auc_score,
    confusion_matrix,
    matthews_corrcoef,
    roc_curve,
    auc,
    balanced_accuracy_score,
    precision_recall_curve,
    average_precision_score,
)
from imblearn.over_sampling import SMOTE
from sklearn.pipeline import Pipeline
from collections import defaultdict
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from typing import Dict, Any, Optional, List, Union
import numpy as np
import pandas as pd
from . import ips
from . import plot
import matplotlib.pyplot as plt
import seaborn as sns

plt.style.use("paper")
import logging
import warnings

logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger()

# Ignore specific warnings (UserWarning in this case)
warnings.filterwarnings("ignore", category=UserWarning)
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier


def features_knn(
    x_train: pd.DataFrame, y_train: pd.Series, knn_params: dict
) -> pd.DataFrame:
    """
    A distance-based classifier that assigns labels based on the majority label of nearest neighbors.
    when to use:
        Effective for small to medium datasets with a low number of features.
        It does not directly provide feature importances but can be assessed through feature permutation or similar methods.
    Recommended Use: Effective for datasets with low feature dimensionality and well-separated clusters.

    Fits KNeighborsClassifier and approximates feature influence using permutation importance.
    """
    knn = KNeighborsClassifier(**knn_params)
    knn.fit(x_train, y_train)
    importances = permutation_importance(
        knn, x_train, y_train, n_repeats=30, random_state=1, scoring="accuracy"
    )
    return pd.DataFrame(
        {"feature": x_train.columns, "importance": importances.importances_mean}
    ).sort_values(by="importance", ascending=False)


#! 1. Linear and Regularized Regression Methods
# 1.1 Lasso
def features_lasso(
    x_train: pd.DataFrame, y_train: pd.Series, lasso_params: dict
) -> np.ndarray:
    """
    Lasso (Least Absolute Shrinkage and Selection Operator):
    A regularized linear regression method that uses L1 penalty to shrink coefficients, effectively
    performing feature selection by zeroing out less important ones.
    """
    lasso = LassoCV(**lasso_params)
    lasso.fit(x_train, y_train)
    # Get non-zero coefficients and their corresponding features
    coefficients = lasso.coef_
    importance_df = pd.DataFrame(
        {"feature": x_train.columns, "importance": np.abs(coefficients)}
    )
    return importance_df[importance_df["importance"] > 0].sort_values(
        by="importance", ascending=False
    )


# 1.2 Ridge regression
def features_ridge(
    x_train: pd.DataFrame, y_train: pd.Series, ridge_params: dict
) -> np.ndarray:
    """
    Ridge Regression: A linear regression technique that applies L2 regularization, reducing coefficient
    magnitudes to avoid overfitting, especially with multicollinearity among features.
    """
    from sklearn.linear_model import RidgeCV

    ridge = RidgeCV(**ridge_params)
    ridge.fit(x_train, y_train)

    # Get the coefficients
    coefficients = ridge.coef_

    # Create a DataFrame to hold feature importance
    importance_df = pd.DataFrame(
        {"feature": x_train.columns, "importance": np.abs(coefficients)}
    )
    return importance_df[importance_df["importance"] > 0].sort_values(
        by="importance", ascending=False
    )


# 1.3 Elastic Net(Enet)
def features_enet(
    x_train: pd.DataFrame, y_train: pd.Series, enet_params: dict
) -> np.ndarray:
    """
    Elastic Net (Enet): Combines L1 and L2 penalties (lasso and ridge) in a linear model, beneficial
    when features are highly correlated or for datasets with more features than samples.
    """
    from sklearn.linear_model import ElasticNetCV

    enet = ElasticNetCV(**enet_params)
    enet.fit(x_train, y_train)
    # Get the coefficients
    coefficients = enet.coef_
    # Create a DataFrame to hold feature importance
    importance_df = pd.DataFrame(
        {"feature": x_train.columns, "importance": np.abs(coefficients)}
    )
    return importance_df[importance_df["importance"] > 0].sort_values(
        by="importance", ascending=False
    )


# 1.4 Partial Least Squares Regression for Generalized Linear Models (plsRglm): Combines regression and
# feature reduction, useful for high-dimensional data with correlated features, such as genomics.

#! 2.Generalized Linear Models and Extensions
# 2.1


#!3.Tree-Based and Ensemble Methods
# 3.1 Random Forest(RF)
def features_rf(
    x_train: pd.DataFrame, y_train: pd.Series, rf_params: dict
) -> np.ndarray:
    """
    An ensemble of decision trees that combines predictions from multiple trees for classification or
    regression, effective with high-dimensional, complex datasets.
    when to use:
        Handles high-dimensional data well.
        Robust to overfitting due to averaging of multiple trees.
        Provides feature importance, which can help in understanding the influence of different genes.
    Fit Random Forest and return sorted feature importances.
    Recommended Use: Great for classification problems, especially when you have many features (genes).
    """
    rf = RandomForestClassifier(**rf_params)
    rf.fit(x_train, y_train)
    return pd.DataFrame(
        {"feature": x_train.columns, "importance": rf.featuress_}
    ).sort_values(by="importance", ascending=False)


# 3.2 Gradient Boosting Trees
def features_gradient_boosting(
    x_train: pd.DataFrame, y_train: pd.Series, gb_params: dict
) -> pd.DataFrame:
    """
    An ensemble of decision trees that combines predictions from multiple trees for classification or regression, effective with
    high-dimensional, complex datasets.
    Gradient Boosting
    Strengths:
        High predictive accuracy and works well for both classification and regression.
        Can handle a mixture of numerical and categorical features.
    Recommended Use:
        Effective for complex relationships and when you need a powerful predictive model.
    Fit Gradient Boosting classifier and return sorted feature importances.
    Recommended Use: Effective for complex datasets with many features (genes).
    """
    gb = GradientBoostingClassifier(**gb_params)
    gb.fit(x_train, y_train)
    return pd.DataFrame(
        {"feature": x_train.columns, "importance": gb.feature_importances_}
    ).sort_values(by="importance", ascending=False)


# 3.3 XGBoost
def features_xgb(
    x_train: pd.DataFrame, y_train: pd.Series, xgb_params: dict
) -> pd.DataFrame:
    """
    XGBoost: An advanced gradient boosting technique, faster and more efficient than GBM, with excellent predictive performance on structured data.
    """
    import xgboost as xgb

    xgb_model = xgb.XGBClassifier(**xgb_params)
    xgb_model.fit(x_train, y_train)
    return pd.DataFrame(
        {"feature": x_train.columns, "importance": xgb_model.feature_importances_}
    ).sort_values(by="importance", ascending=False)


# 3.4.decision tree
def features_decision_tree(
    x_train: pd.DataFrame, y_train: pd.Series, dt_params: dict
) -> pd.DataFrame:
    """
    A single decision tree classifier effective for identifying key decision boundaries in data.
    when to use:
        Good for capturing non-linear patterns.
        Provides feature importance scores for each feature, though it may overfit on small datasets.
        Efficient for low to medium-sized datasets, where interpretability of decisions is key.
    Recommended Use: Useful for interpretable feature importance analysis in smaller or balanced datasets.

    Fits DecisionTreeClassifier and returns sorted feature importances.
    """
    dt = DecisionTreeClassifier(**dt_params)
    dt.fit(x_train, y_train)
    return pd.DataFrame(
        {"feature": x_train.columns, "importance": dt.feature_importances_}
    ).sort_values(by="importance", ascending=False)


# 3.5 bagging
def features_bagging(
    x_train: pd.DataFrame, y_train: pd.Series, bagging_params: dict
) -> pd.DataFrame:
    """
    A bagging ensemble of models, often used with weak learners like decision trees, to reduce variance.
    when to use:
        Helps reduce overfitting, especially on high-variance models.
        Effective when the dataset has numerous features and may benefit from ensemble stability.
    Recommended Use: Beneficial for high-dimensional or noisy datasets needing ensemble stability.

    Fits BaggingClassifier and returns averaged feature importances from underlying estimators if available.
    """
    bagging = BaggingClassifier(**bagging_params)
    bagging.fit(x_train, y_train)

    # Calculate feature importance by averaging importances across estimators, if feature_importances_ is available.
    if hasattr(bagging.estimators_[0], "feature_importances_"):
        importances = np.mean(
            [estimator.feature_importances_ for estimator in bagging.estimators_],
            axis=0,
        )
        return pd.DataFrame(
            {"feature": x_train.columns, "importance": importances}
        ).sort_values(by="importance", ascending=False)
    else:
        # If the base estimator does not support feature importances, fallback to permutation importance.
        importances = permutation_importance(
            bagging, x_train, y_train, n_repeats=30, random_state=1, scoring="accuracy"
        )
        return pd.DataFrame(
            {"feature": x_train.columns, "importance": importances.importances_mean}
        ).sort_values(by="importance", ascending=False)


#! 4.Support Vector Machines
def features_svm(
    x_train: pd.DataFrame, y_train: pd.Series, rfe_params: dict
) -> np.ndarray:
    """
    Suitable for classification tasks where the number of features is much larger than the number of samples.
        1. Effective in high-dimensional spaces and with clear margin of separation.
        2. Works well for both linear and non-linear classification (using kernel functions).
    Select features using RFE with SVM.When combined with SVM, RFE selects features that are most critical for the decision boundary,
        helping reduce the dataset to a more manageable size without losing much predictive power.
    SVM (Support Vector Machines),supports various kernels (linear, rbf, poly, and sigmoid), is good at handling high-dimensional
        data and finding an optimal decision boundary between classes, especially when using the right kernel.
    kernel: ["linear", "rbf", "poly", "sigmoid"]
        'linear': simplest kernel that attempts to separate data by drawing a straight line (or hyperplane) between classes. It is effective
            when the data is linearly separable, meaning the classes can be well divided by a straight boundary.
                Advantages:
                    - Computationally efficient for large datasets.
                    - Works well when the number of features is high, which is common in genomic data where you may have thousands of genes
                        as features.
        'rbf':  a nonlinear kernel that maps the input data into a higher-dimensional space to find a decision boundary. It works well for
            data that is not linearly separable in its original space.
                Advantages:
                    - Handles nonlinear relationships between features and classes
                    - Often better than a linear kernel when there is no clear linear decision boundary in the data.
        'poly': Polynomial Kernel: computes similarity between data points based on polynomial functions of the input features. It can model
            interactions between features to a certain degree, depending on the polynomial degree chosen.
                Advantages:
                    - Allows modeling of feature interactions.
                    - Can fit more complex relationships compared to linear models.
        'sigmoid':  similar to the activation function in neural networks, and it works well when the data follows an S-shaped decision boundary.
                Advantages:
                - Can approximate the behavior of neural networks.
                - Use case: It’s not as widely used as the RBF or linear kernel but can be explored when there is some evidence of non-linear
                    S-shaped relationships.
    """
    # SVM (Support Vector Machines)
    svc = SVC(kernel=rfe_params["kernel"])  # ["linear", "rbf", "poly", "sigmoid"]
    # RFE(Recursive Feature Elimination)
    selector = RFE(svc, n_features_to_select=rfe_params["n_features_to_select"])
    selector.fit(x_train, y_train)
    return x_train.columns[selector.support_]


#! 5.Bayesian and Probabilistic Methods
def features_naive_bayes(x_train: pd.DataFrame, y_train: pd.Series) -> list:
    """
    Naive Bayes: A probabilistic classifier based on Bayes' theorem, assuming independence between features, simple and fast, especially
    effective for text classification and other high-dimensional data.
    """
    from sklearn.naive_bayes import GaussianNB

    nb = GaussianNB()
    nb.fit(x_train, y_train)
    probabilities = nb.predict_proba(x_train)
    # Limit the number of features safely, choosing the lesser of half the features or all columns
    n_features = min(x_train.shape[1] // 2, len(x_train.columns))
    
    # Sort probabilities, then map to valid column indices
    sorted_indices = np.argsort(probabilities.max(axis=1))[:n_features]
    
    # Ensure indices are within the column bounds of x_train
    valid_indices = sorted_indices[sorted_indices < len(x_train.columns)]
    
    return x_train.columns[valid_indices]


#! 6.Linear Discriminant Analysis (LDA)
def features_lda(x_train: pd.DataFrame, y_train: pd.Series) -> list:
    """
    Linear Discriminant Analysis (LDA): Projects data onto a lower-dimensional space to maximize class separability, often used as a dimensionality
    reduction technique before classification on high-dimensional data.
    """
    from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

    lda = LinearDiscriminantAnalysis()
    lda.fit(x_train, y_train)
    coef = lda.coef_.flatten()
    # Create a DataFrame to hold feature importance
    importance_df = pd.DataFrame(
        {"feature": x_train.columns, "importance": np.abs(coef)}
    )

    return importance_df[importance_df["importance"] > 0].sort_values(
        by="importance", ascending=False
    )


def features_adaboost(
    x_train: pd.DataFrame, y_train: pd.Series, adaboost_params: dict
) -> pd.DataFrame:
    """
    AdaBoost
    Strengths:
        Combines multiple weak learners to create a strong classifier.
        Focuses on examples that are hard to classify, improving overall performance.
    Recommended Use:
        Can be effective for boosting weak models in a genomics context.
    Fit AdaBoost classifier and return sorted feature importances.
    Recommended Use: Great for classification problems with a large number of features (genes).
    """
    ada = AdaBoostClassifier(**adaboost_params)
    ada.fit(x_train, y_train)
    return pd.DataFrame(
        {"feature": x_train.columns, "importance": ada.feature_importances_}
    ).sort_values(by="importance", ascending=False)


import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset
from skorch import NeuralNetClassifier  # sklearn compatible


class DNNClassifier(nn.Module):
    def __init__(self, input_dim, hidden_dim=128, output_dim=2, dropout_rate=0.5):
        super(DNNClassifier, self).__init__()

        self.hidden_layer1 = nn.Sequential(
            nn.Linear(input_dim, hidden_dim),
            nn.ReLU(),
            nn.Dropout(dropout_rate),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ReLU(),
        )

        self.hidden_layer2 = nn.Sequential(
            nn.Linear(hidden_dim, hidden_dim), nn.ReLU(), nn.Dropout(dropout_rate)
        )

        # Adding a residual connection between hidden layers
        self.residual = nn.Linear(input_dim, hidden_dim)

        self.output_layer = nn.Sequential(
            nn.Linear(hidden_dim, output_dim), nn.Softmax(dim=1)
        )

    def forward(self, x):
        residual = self.residual(x)
        x = self.hidden_layer1(x)
        x = x + residual  # Residual connection
        x = self.hidden_layer2(x)
        x = self.output_layer(x)
        return x


def validate_classifier(
    clf,
    x_train: pd.DataFrame,
    y_train: pd.Series,
    x_test: pd.DataFrame,
    y_test: pd.Series,
    metrics: list = ["accuracy", "precision", "recall", "f1", "roc_auc"],
    cv_folds: int = 5,
) -> dict:
    """
    Perform cross-validation for a given classifier and return average scores for specified metrics on training data.
    Then fit the best model on the full training data and evaluate it on the test set.

    Parameters:
    - clf: The classifier to be validated.
    - x_train: Training features.
    - y_train: Training labels.
    - x_test: Test features.
    - y_test: Test labels.
    - metrics: List of metrics to evaluate (e.g., ['accuracy', 'roc_auc']).
    - cv_folds: Number of cross-validation folds.

    Returns:
    - results: Dictionary containing average cv_train_scores and cv_test_scores.
    """
    cv_train_scores = {metric: [] for metric in metrics}
    skf = StratifiedKFold(n_splits=cv_folds)
    # Perform cross-validation
    for metric in metrics:
        try:
            if metric == "roc_auc" and len(set(y_train)) == 2:
                scores = cross_val_score(
                    clf, x_train, y_train, cv=skf, scoring="roc_auc"
                )
                cv_train_scores[metric] = (
                    np.nanmean(scores) if not np.isnan(scores).all() else float("nan")
                )
            else:
                score = cross_val_score(clf, x_train, y_train, cv=skf, scoring=metric)
                cv_train_scores[metric] = score.mean()
        except Exception as e:
            cv_train_scores[metric] = float("nan")
    clf.fit(x_train, y_train)

    # Evaluate on the test set
    cv_test_scores = {}
    for metric in metrics:
        if metric == "roc_auc" and len(set(y_test)) == 2:
            try:
                y_prob = clf.predict_proba(x_test)[:, 1]
                cv_test_scores[metric] = roc_auc_score(y_test, y_prob)
            except AttributeError:
                cv_test_scores[metric] = float("nan")
        else:
            score_func = globals().get(
                f"{metric}_score"
            )  # Fetching the appropriate scoring function
            if score_func:
                try:
                    y_pred = clf.predict(x_test)
                    cv_test_scores[metric] = score_func(y_test, y_pred)
                except Exception as e:
                    cv_test_scores[metric] = float("nan")

    # Combine results
    results = {"cv_train_scores": cv_train_scores, "cv_test_scores": cv_test_scores}
    return results


def get_models(
    random_state=1,
    cls=[
        "lasso",
        "ridge",
        "Elastic Net(Enet)",
        "gradient Boosting",
        "Random forest (rf)",
        "XGBoost (xgb)",
        "Support Vector Machine(svm)",
        "naive bayes",
        "Linear Discriminant Analysis (lda)",
        "adaboost",
        "DecisionTree",
        "KNeighbors",
        "Bagging",
    ],
):
    from sklearn.ensemble import (
        RandomForestClassifier,
        GradientBoostingClassifier,
        AdaBoostClassifier,
        BaggingClassifier,
    )
    from sklearn.svm import SVC
    from sklearn.linear_model import (
        LogisticRegression,
        Lasso,
        RidgeClassifierCV,
        ElasticNet,
    )
    from sklearn.naive_bayes import GaussianNB
    from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
    import xgboost as xgb
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.neighbors import KNeighborsClassifier

    res_cls = {}
    model_all = {
        "Lasso": LogisticRegression(
            penalty="l1", solver="saga", random_state=random_state
        ),
        "Ridge": RidgeClassifierCV(),
        "Elastic Net (Enet)": ElasticNet(random_state=random_state),
        "Gradient Boosting": GradientBoostingClassifier(random_state=random_state),
        "Random Forest (RF)": RandomForestClassifier(random_state=random_state),
        "XGBoost (XGB)": xgb.XGBClassifier(random_state=random_state),
        "Support Vector Machine (SVM)": SVC(kernel="rbf", probability=True),
        "Naive Bayes": GaussianNB(),
        "Linear Discriminant Analysis (LDA)": LinearDiscriminantAnalysis(),
        "AdaBoost": AdaBoostClassifier(random_state=random_state, algorithm="SAMME"),
        "DecisionTree": DecisionTreeClassifier(),
        "KNeighbors": KNeighborsClassifier(n_neighbors=5),
        "Bagging": BaggingClassifier(),
    }
    print("Using default models:")
    for cls_name in cls:
        cls_name = ips.strcmp(cls_name, list(model_all.keys()))[0]
        res_cls[cls_name] = model_all[cls_name]
        print(f"- {cls_name}")
    return res_cls


def get_features(
    X: Union[pd.DataFrame, np.ndarray],  # n_samples X n_features
    y: Union[pd.Series, np.ndarray, list],  # n_samples X n_features
    test_size: float = 0.2,
    random_state: int = 1,
    n_features: int = 10,
    fill_missing=True,
    rf_params: Optional[Dict] = None,
    rfe_params: Optional[Dict] = None,
    lasso_params: Optional[Dict] = None,
    ridge_params: Optional[Dict] = None,
    enet_params: Optional[Dict] = None,
    gb_params: Optional[Dict] = None,
    adaboost_params: Optional[Dict] = None,
    xgb_params: Optional[Dict] = None,
    dt_params: Optional[Dict] = None,
    bagging_params: Optional[Dict] = None,
    knn_params: Optional[Dict] = None,
    cls: list = [
        "lasso","ridge","Elastic Net(Enet)","gradient Boosting","Random forest (rf)","XGBoost (xgb)","Support Vector Machine(svm)",
        "naive bayes","Linear Discriminant Analysis (lda)","adaboost","DecisionTree","KNeighbors","Bagging"],
    metrics: Optional[List[str]] = None,
    cv_folds: int = 5,
    strict: bool = False,
    n_shared: int = 2,  # 只要有两个方法有重合,就纳入common genes
    use_selected_features: bool = True,
    plot_: bool = True,
    dir_save:str="./") -> dict:
    """
    Master function to perform feature selection and validate models.
    """
    from sklearn.compose import ColumnTransformer
    from sklearn.preprocessing import StandardScaler, OneHotEncoder

    # Ensure X and y are DataFrames/Series for consistency
    if isinstance(X, np.ndarray):
        X = pd.DataFrame(X)
    if isinstance(y, (np.ndarray, list)):
        y = pd.Series(y)

    # fill na
    if fill_missing:
        ips.df_fillna(data=X,method='knn',inplace=True,axis=0)
    if isinstance(y, str) and y in X.columns: 
        y_col_name=y
        y=X[y]
        y=ips.df_encoder(pd.DataFrame(y),method='dummy')
        X = X.drop(y_col_name,axis=1)
    else:
        y=ips.df_encoder(pd.DataFrame(y),method='dummy').values.ravel()
    y = y.loc[X.index]  # Align y with X after dropping rows with missing values in X
    y = y.ravel() if isinstance(y, np.ndarray) else y.values.ravel()

    if X.shape[0] != len(y):
        raise ValueError("X and y must have the same number of samples (rows).")

    # #! # Check for non-numeric columns in X and apply one-hot encoding if needed
    # Check if any column in X is non-numeric
    if any(not np.issubdtype(dtype, np.number) for dtype in X.dtypes):
        X = pd.get_dummies(X, drop_first=True)
    print(X.shape)

    # #!alternative:  # Identify categorical and numerical columns
    # categorical_cols = X.select_dtypes(include=["object", "category"]).columns
    # numerical_cols = X.select_dtypes(include=["number"]).columns

    # # Define preprocessing pipeline
    # preprocessor = ColumnTransformer(
    #     transformers=[
    #         ("num", StandardScaler(), numerical_cols),
    #         ("cat", OneHotEncoder(drop="first", handle_unknown="ignore"), categorical_cols),
    #     ]
    # )
    # # Preprocess the data
    # X = preprocessor.fit_transform(X)

    # Split data into training and test sets
    x_train, x_test, y_train, y_test = train_test_split(
        X, y, test_size=test_size, random_state=random_state
    )
    # Standardize features
    scaler = StandardScaler()
    x_train_scaled = scaler.fit_transform(x_train)
    x_test_scaled = scaler.transform(x_test)

    # Convert back to DataFrame for consistency
    x_train = pd.DataFrame(x_train_scaled, columns=x_train.columns)
    x_test = pd.DataFrame(x_test_scaled, columns=x_test.columns)

    rf_defaults = {"n_estimators": 100, "random_state": random_state}
    rfe_defaults = {"kernel": "linear", "n_features_to_select": n_features}
    lasso_defaults = {"alphas": np.logspace(-4, 4, 100), "cv": 10}
    ridge_defaults = {"alphas": np.logspace(-4, 4, 100), "cv": 10}
    enet_defaults = {"alphas": np.logspace(-4, 4, 100), "cv": 10}
    xgb_defaults = {
        "n_estimators": 100,
        "use_label_encoder": False,
        "eval_metric": "logloss",
        "random_state": random_state,
    }
    gb_defaults = {"n_estimators": 100, "random_state": random_state}
    adaboost_defaults = {"n_estimators": 50, "random_state": random_state}
    dt_defaults = {"max_depth": None, "random_state": random_state}
    bagging_defaults = {"n_estimators": 50, "random_state": random_state}
    knn_defaults = {"n_neighbors": 5}
    rf_params, rfe_params = rf_params or rf_defaults, rfe_params or rfe_defaults
    lasso_params, ridge_params = (
        lasso_params or lasso_defaults,
        ridge_params or ridge_defaults,
    )
    enet_params, xgb_params = enet_params or enet_defaults, xgb_params or xgb_defaults
    gb_params, adaboost_params = (
        gb_params or gb_defaults,
        adaboost_params or adaboost_defaults,
    )
    dt_params = dt_params or dt_defaults
    bagging_params = bagging_params or bagging_defaults
    knn_params = knn_params or knn_defaults

    cls_ = [
        "lasso",
        "ridge",
        "Elastic Net(Enet)",
        "Gradient Boosting",
        "Random Forest (rf)",
        "XGBoost (xgb)",
        "Support Vector Machine(svm)",
        "Naive Bayes",
        "Linear Discriminant Analysis (lda)",
        "adaboost",
    ]
    cls = [ips.strcmp(i, cls_)[0] for i in cls]

    # Lasso Feature Selection
    lasso_importances = (
        features_lasso(x_train, y_train, lasso_params)
        if "lasso" in cls
        else pd.DataFrame()
    )
    lasso_selected_features = (
        lasso_importances.head(n_features)["feature"].values if "lasso" in cls else []
    )
    # Ridge
    ridge_importances = (
        features_ridge(x_train, y_train, ridge_params)
        if "ridge" in cls
        else pd.DataFrame()
    )
    selected_ridge_features = (
        ridge_importances.head(n_features)["feature"].values if "ridge" in cls else []
    )
    # Elastic Net
    enet_importances = (
        features_enet(x_train, y_train, enet_params)
        if "Enet" in cls
        else pd.DataFrame()
    )
    selected_enet_features = (
        enet_importances.head(n_features)["feature"].values if "Enet" in cls else []
    )
    # Random Forest Feature Importance
    rf_importances = (
        features_rf(x_train, y_train, rf_params)
        if "Random Forest" in cls
        else pd.DataFrame()
    )
    top_rf_features = (
        rf_importances.head(n_features)["feature"].values
        if "Random Forest" in cls
        else []
    )
    # Gradient Boosting Feature Importance
    gb_importances = (
        features_gradient_boosting(x_train, y_train, gb_params)
        if "Gradient Boosting" in cls
        else pd.DataFrame()
    )
    top_gb_features = (
        gb_importances.head(n_features)["feature"].values
        if "Gradient Boosting" in cls
        else []
    )
    # xgb
    xgb_importances = (
        features_xgb(x_train, y_train, xgb_params) if "xgb" in cls else pd.DataFrame()
    )
    top_xgb_features = (
        xgb_importances.head(n_features)["feature"].values if "xgb" in cls else []
    )

    # SVM with RFE
    selected_svm_features = (
        features_svm(x_train, y_train, rfe_params) if "svm" in cls else []
    )
    # Naive Bayes
    selected_naive_bayes_features = (
        features_naive_bayes(x_train, y_train) if "Naive Bayes" in cls else []
    )
    # lda: linear discriminant analysis
    lda_importances = features_lda(x_train, y_train) if "lda" in cls else pd.DataFrame()
    selected_lda_features = (
        lda_importances.head(n_features)["feature"].values if "lda" in cls else []
    )
    # AdaBoost Feature Importance
    adaboost_importances = (
        features_adaboost(x_train, y_train, adaboost_params)
        if "AdaBoost" in cls
        else pd.DataFrame()
    )
    top_adaboost_features = (
        adaboost_importances.head(n_features)["feature"].values
        if "AdaBoost" in cls
        else []
    )
    # Decision Tree Feature Importance
    dt_importances = (
        features_decision_tree(x_train, y_train, dt_params)
        if "Decision Tree" in cls
        else pd.DataFrame()
    )
    top_dt_features = (
        dt_importances.head(n_features)["feature"].values
        if "Decision Tree" in cls
        else []
    )
    # Bagging Feature Importance
    bagging_importances = (
        features_bagging(x_train, y_train, bagging_params)
        if "Bagging" in cls
        else pd.DataFrame()
    )
    top_bagging_features = (
        bagging_importances.head(n_features)["feature"].values
        if "Bagging" in cls
        else []
    )
    # KNN Feature Importance via Permutation
    knn_importances = (
        features_knn(x_train, y_train, knn_params) if "KNN" in cls else pd.DataFrame()
    )
    top_knn_features = (
        knn_importances.head(n_features)["feature"].values if "KNN" in cls else []
    )

    #! Find common features
    common_features = ips.shared(
        lasso_selected_features,
        selected_ridge_features,
        selected_enet_features,
        top_rf_features,
        top_gb_features,
        top_xgb_features,
        selected_svm_features,
        selected_naive_bayes_features,
        selected_lda_features,
        top_adaboost_features,
        top_dt_features,
        top_bagging_features,
        top_knn_features,
        strict=strict,
        n_shared=n_shared,
        verbose=False
    )

    # Use selected features or all features for model validation
    x_train_selected = (
        x_train[list(common_features)] if use_selected_features else x_train
    )
    x_test_selected = x_test[list(common_features)] if use_selected_features else x_test

    if metrics is None:
        metrics = ["accuracy", "precision", "recall", "f1", "roc_auc"]

    # Prepare results DataFrame for selected features
    features_df = pd.DataFrame(
        {
            "type": ["Lasso"] * len(lasso_selected_features)
            + ["Ridge"] * len(selected_ridge_features)
            + ["Random Forest"] * len(top_rf_features)
            + ["Gradient Boosting"] * len(top_gb_features)
            + ["Enet"] * len(selected_enet_features)
            + ["xgb"] * len(top_xgb_features)
            + ["SVM"] * len(selected_svm_features)
            + ["Naive Bayes"] * len(selected_naive_bayes_features)
            + ["Linear Discriminant Analysis"] * len(selected_lda_features)
            + ["AdaBoost"] * len(top_adaboost_features)
            + ["Decision Tree"] * len(top_dt_features)
            + ["Bagging"] * len(top_bagging_features)
            + ["KNN"] * len(top_knn_features),
            "feature": np.concatenate(
                [
                    lasso_selected_features,
                    selected_ridge_features,
                    top_rf_features,
                    top_gb_features,
                    selected_enet_features,
                    top_xgb_features,
                    selected_svm_features,
                    selected_naive_bayes_features,
                    selected_lda_features,
                    top_adaboost_features,
                    top_dt_features,
                    top_bagging_features,
                    top_knn_features,
                ]
            ),
        }
    )

    #! Validate trained each classifier
    models = get_models(random_state=random_state, cls=cls)
    cv_train_results, cv_test_results = [], []
    for name, clf in models.items():
        if not x_train_selected.empty:
            cv_scores = validate_classifier(
                clf,
                x_train_selected,
                y_train,
                x_test_selected,
                y_test,
                metrics=metrics,
                cv_folds=cv_folds,
            )

            cv_train_score_df = pd.DataFrame(cv_scores["cv_train_scores"], index=[name])
            cv_test_score_df = pd.DataFrame(cv_scores["cv_test_scores"], index=[name])
            cv_train_results.append(cv_train_score_df)
            cv_test_results.append(cv_test_score_df)
    if all([cv_train_results, cv_test_results]):
        cv_train_results_df = (
            pd.concat(cv_train_results)
            .reset_index()
            .rename(columns={"index": "Classifier"})
        )
        cv_test_results_df = (
            pd.concat(cv_test_results)
            .reset_index()
            .rename(columns={"index": "Classifier"})
        )
        #! Store results in the main results dictionary
        results = {
            "selected_features": features_df,
            "cv_train_scores": cv_train_results_df,
            "cv_test_scores": rank_models(cv_test_results_df,plot_=plot_),
            "common_features": list(common_features),
        }
        if all([plot_,dir_save]):
            from datetime import datetime
            now_ = datetime.now().strftime("%y%m%d_%H%M%S")
            ips.figsave(dir_save+f"features{now_}.pdf")
    else:
        results = {
            "selected_features": pd.DataFrame(),
            "cv_train_scores": pd.DataFrame(),
            "cv_test_scores": pd.DataFrame(),
            "common_features": [],
        }
        print(f"Warning: 没有找到共同的genes, when n_shared={n_shared}")
    return results


#! # usage:
# # Get features and common features
# results = get_features(X, y)
# common_features = results["common_features"]
def validate_features(
    x_train: pd.DataFrame,
    y_train: pd.Series,
    x_true: pd.DataFrame,
    y_true: pd.Series,
    common_features: set = None,
    models: Optional[Dict[str, Any]] = None,
    metrics: Optional[list] = None,
    random_state: int = 1,
    smote: bool = False,
    n_jobs:int = -1,
    plot_: bool = True,
    class_weight: str = "balanced",
) -> dict:
    """
    Validate models using selected features on the validation dataset.

    Parameters:
    - x_train (pd.DataFrame): Training feature dataset.
    - y_train (pd.Series): Training target variable.
    - x_true (pd.DataFrame): Validation feature dataset.
    - y_true (pd.Series): Validation target variable.
    - common_features (set): Set of common features to use for validation.
    - models (dict, optional): Dictionary of models to validate.
    - metrics (list, optional): List of metrics to compute.
    - random_state (int): Random state for reproducibility.
    - plot_ (bool): Option to plot metrics (to be implemented if needed).
    - class_weight (str or dict): Class weights to handle imbalance.

    """
    from tqdm import tqdm
    # Ensure common features are selected
    common_features = ips.shared(common_features, x_train.columns, x_true.columns, strict=True,verbose=False)

    # Filter the training and validation datasets for the common features
    x_train_selected = x_train[common_features]
    x_true_selected = x_true[common_features]

    if not x_true_selected.index.equals(y_true.index):
        raise ValueError(
            "Index mismatch between validation features and target. Ensure data alignment."
        )

    y_true = y_true.loc[x_true_selected.index]

    # Handle class imbalance using SMOTE
    if smote:
        if (
            y_train.value_counts(normalize=True).max() < 0.8
        ):  # Threshold to decide if data is imbalanced
            smote = SMOTE(random_state=random_state)
            x_train_resampled, y_train_resampled = smote.fit_resample(
                x_train_selected, y_train
            )
        else:
            # skip SMOTE
            x_train_resampled, y_train_resampled = x_train_selected, y_train
    else:
        x_train_resampled, y_train_resampled = x_train_selected, y_train

    # Default models if not provided
    if models is None:
        models = {
            "Random Forest": RandomForestClassifier(
                class_weight=class_weight, random_state=random_state
            ),
            "SVM": SVC(probability=True, class_weight=class_weight),
            "Logistic Regression": LogisticRegression(
                class_weight=class_weight, random_state=random_state
            ),
            "Gradient Boosting": GradientBoostingClassifier(random_state=random_state),
            "AdaBoost": AdaBoostClassifier(
                random_state=random_state, algorithm="SAMME"
            ),
            "Lasso": LogisticRegression(
                penalty="l1", solver="saga", random_state=random_state
            ),
            "Ridge": LogisticRegression(
                penalty="l2", solver="saga", random_state=random_state
            ),
            "Elastic Net": LogisticRegression(
                penalty="elasticnet",
                solver="saga",
                l1_ratio=0.5,
                random_state=random_state,
            ),
            "XGBoost": xgb.XGBClassifier(eval_metric="logloss"
            ),
            "Naive Bayes": GaussianNB(),
            "LDA": LinearDiscriminantAnalysis(),
        }

    # Hyperparameter grids for tuning
    param_grids = {
        "Random Forest": {
            "n_estimators": [100, 200, 300, 400, 500],
            "max_depth": [None, 3, 5, 10, 20],
            "min_samples_split": [2, 5, 10],
            "min_samples_leaf": [1, 2, 4],
            "class_weight": [None, "balanced"],
        },
        "SVM": {
            "C": [0.01, 0.1, 1, 10, 100, 1000],
            "gamma": [0.001, 0.01, 0.1, "scale", "auto"],
            "kernel": ["linear", "rbf", "poly"],
        },
        "Logistic Regression": {
            "C": [0.01, 0.1, 1, 10, 100],
            "solver": ["liblinear", "saga", "newton-cg", "lbfgs"],
            "penalty": ["l1", "l2"],
            "max_iter": [100, 200, 300],
        },
        "Gradient Boosting": {
            "n_estimators": [100, 200, 300, 400, 500],
            "learning_rate": np.logspace(-3, 0, 4),
            "max_depth": [3, 5, 7, 9],
            "min_samples_split": [2, 5, 10],
        },
        "AdaBoost": {
            "n_estimators": [50, 100, 200, 300, 500],
            "learning_rate": np.logspace(-3, 0, 4),
        },
        "Lasso": {"C": np.logspace(-3, 1, 10), "max_iter": [100, 200, 300]},
        "Ridge": {"C": np.logspace(-3, 1, 10), "max_iter": [100, 200, 300]},
        "Elastic Net": {
            "C": np.logspace(-3, 1, 10),
            "l1_ratio": [0.1, 0.5, 0.9],
            "max_iter": [100, 200, 300],
        },
        "XGBoost": {
            "n_estimators": [100, 200],
            "max_depth": [3, 5, 7],
            "learning_rate": [0.01, 0.1, 0.2],
            "subsample": [0.8, 1.0],
            "colsample_bytree": [0.8, 1.0],
        },
        "Naive Bayes": {},
        "LDA": {"solver": ["svd", "lsqr", "eigen"]},
    }
    # Default metrics if not provided
    if metrics is None:
        metrics = [
            "accuracy",
            "precision",
            "recall",
            "f1",
            "roc_auc",
            "mcc",
            "specificity",
            "balanced_accuracy",
            "pr_auc",
        ]

    results = {}

    # Validate each classifier with GridSearchCV
    for name, clf in tqdm(
                models.items(),
                desc="for metric in metrics",
                colour="green",
                bar_format="{l_bar}{bar} {n_fmt}/{total_fmt}",
            ):
        print(f"\nValidating {name} on the validation dataset:")

        # Check if `predict_proba` method exists; if not, use CalibratedClassifierCV
        # 没有predict_proba的分类器，使用 CalibratedClassifierCV 可以获得校准的概率估计。此外，为了使代码更灵活，我们可以在创建分类器
        # 时检查 predict_proba 方法是否存在，如果不存在且用户希望计算 roc_auc 或 pr_auc，则启用 CalibratedClassifierCV
        if not hasattr(clf, "predict_proba"):
            print(
                f"Using CalibratedClassifierCV for {name} due to lack of probability estimates."
            )
            calibrated_clf = CalibratedClassifierCV(clf, method="sigmoid", cv="prefit")
        else:
            calibrated_clf = clf
        # Stratified K-Fold for cross-validation
        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)

        # Create GridSearchCV object
        gs = GridSearchCV(
            estimator=calibrated_clf,
            param_grid=param_grids[name],
            scoring="roc_auc",  # Optimize for ROC AUC
            cv=skf,  # Stratified K-Folds cross-validation
            n_jobs=n_jobs,
            verbose=1,
        )

        # Fit the model using GridSearchCV
        gs.fit(x_train_resampled, y_train_resampled)
        # Best estimator from grid search
        best_clf = gs.best_estimator_
        # Make predictions on the validation set
        y_pred = best_clf.predict(x_true_selected)
        # Calculate probabilities for ROC AUC if possible
        if hasattr(best_clf, "predict_proba"):
            y_pred_proba = best_clf.predict_proba(x_true_selected)[:, 1]
        elif hasattr(best_clf, "decision_function"):
            # If predict_proba is not available, use decision_function (e.g., for SVM)
            y_pred_proba = best_clf.decision_function(x_true_selected)
            # Ensure y_pred_proba is within 0 and 1 bounds
            y_pred_proba = (y_pred_proba - y_pred_proba.min()) / (
                y_pred_proba.max() - y_pred_proba.min()
            )
        else:
            y_pred_proba = None  # No probability output for certain models

        # Calculate metrics
        validation_scores = {}
        for metric in metrics:
            if metric == "accuracy":
                validation_scores[metric] = accuracy_score(y_true, y_pred)
            elif metric == "precision":
                validation_scores[metric] = precision_score(
                    y_true, y_pred, average="weighted"
                )
            elif metric == "recall":
                validation_scores[metric] = recall_score(
                    y_true, y_pred, average="weighted"
                )
            elif metric == "f1":
                validation_scores[metric] = f1_score(y_true, y_pred, average="weighted")
            elif metric == "roc_auc" and y_pred_proba is not None:
                validation_scores[metric] = roc_auc_score(y_true, y_pred_proba)
            elif metric == "mcc":
                validation_scores[metric] = matthews_corrcoef(y_true, y_pred)
            elif metric == "specificity":
                tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
                validation_scores[metric] = tn / (tn + fp)  # Specificity calculation
            elif metric == "balanced_accuracy":
                validation_scores[metric] = balanced_accuracy_score(y_true, y_pred)
            elif metric == "pr_auc" and y_pred_proba is not None:
                precision, recall, _ = precision_recall_curve(y_true, y_pred_proba)
                validation_scores[metric] = average_precision_score(
                    y_true, y_pred_proba
                )

        # Calculate ROC curve
        # https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html
        if y_pred_proba is not None:
            # fpr, tpr, roc_auc = dict(), dict(), dict()
            fpr, tpr, _ = roc_curve(y_true, y_pred_proba)
            lower_ci, upper_ci = cal_auc_ci(y_true, y_pred_proba,verbose=False)
            roc_auc = auc(fpr, tpr)
            roc_info = {
                "fpr": fpr.tolist(),
                "tpr": tpr.tolist(),
                "auc": roc_auc,
                "ci95": (lower_ci, upper_ci),
            }
            # precision-recall curve
            precision_, recall_, _ = precision_recall_curve(y_true, y_pred_proba)
            avg_precision_ = average_precision_score(y_true, y_pred_proba)
            pr_info = {
                "precision": precision_,
                "recall": recall_,
                "avg_precision": avg_precision_,
            }
        else:
            roc_info, pr_info = None, None
        results[name] = {
            "best_params": gs.best_params_,
            "scores": validation_scores,
            "roc_curve": roc_info,
            "pr_curve": pr_info,
            "confusion_matrix": confusion_matrix(y_true, y_pred),
        }

    df_results = pd.DataFrame.from_dict(results, orient="index")

    return df_results


#! usage validate_features()
# Validate models using the validation dataset (X_val, y_val)
# validation_results = validate_features(X, y, X_val, y_val, common_features)

# # If you want to access validation scores
# print(validation_results)
def plot_validate_features(res_val):
    """
    plot the results of 'validate_features()'
    """
    colors = plot.get_color(len(ips.flatten(res_val["pr_curve"].index)))
    if res_val.shape[0]>5:
        alpha=0
        figsize=[8,10]
        subplot_layout=[1,2]
        ncols=2
        bbox_to_anchor=[1.5,0.6]
    else:
        alpha=0.03
        figsize=[10,6]
        subplot_layout=[1,1]
        ncols=1
        bbox_to_anchor=[1,1]
    nexttile = plot.subplot(figsize=figsize)
    ax = nexttile(subplot_layout[0],subplot_layout[1])
    for i, model_name in enumerate(ips.flatten(res_val["pr_curve"].index)):
        fpr = res_val["roc_curve"][model_name]["fpr"]
        tpr = res_val["roc_curve"][model_name]["tpr"]
        (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"]
        mean_auc = res_val["roc_curve"][model_name]["auc"]
        plot_roc_curve(
            fpr,tpr,mean_auc,lower_ci,upper_ci,model_name=model_name,
            lw=1.5,color=colors[i],alpha=alpha,ax=ax)
    plot.figsets(sp=2,legend=dict(loc="upper right", ncols=ncols, fontsize=8, bbox_to_anchor=[1.5,0.6],markerscale=0.8))
    # plot.split_legend(ax,n=2, loc=["upper left", "lower left"],bbox=[[1,0.5],[1,0.5]],ncols=2,labelcolor="k",fontsize=8)

    ax = nexttile(subplot_layout[0],subplot_layout[1])
    for i, model_name in enumerate(ips.flatten(res_val["pr_curve"].index)):
        plot_pr_curve(
            recall=res_val["pr_curve"][model_name]["recall"],
            precision=res_val["pr_curve"][model_name]["precision"],
            avg_precision=res_val["pr_curve"][model_name]["avg_precision"],
            model_name=model_name,
            color=colors[i],lw=1.5,alpha=alpha,ax=ax)
    plot.figsets(sp=2,legend=dict(loc="upper right", ncols=1, fontsize=8, bbox_to_anchor=[1.5,0.5]))
    # plot.split_legend(ax,n=2, loc=["upper left", "lower left"],bbox=[[1,0.5],[1,0.5]],ncols=2,labelcolor="k",fontsize=8)
 
def plot_validate_features_single(res_val,figsize=None):
    if figsize is None:
        nexttile = plot.subplot(len(ips.flatten(res_val["pr_curve"].index)), 3)
    else:
        nexttile = plot.subplot(len(ips.flatten(res_val["pr_curve"].index)), 3,figsize=figsize)
    for model_name in ips.flatten(res_val["pr_curve"].index):
        fpr = res_val["roc_curve"][model_name]["fpr"]
        tpr = res_val["roc_curve"][model_name]["tpr"]
        (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"]
        mean_auc = res_val["roc_curve"][model_name]["auc"]

        # Plotting
        plot_roc_curve(fpr, tpr, mean_auc, lower_ci, upper_ci, ax=nexttile())
        plot.figsets(title=model_name, sp=2)

        plot_pr_curve(
            recall=res_val["pr_curve"][model_name]["recall"],
            precision=res_val["pr_curve"][model_name]["precision"],
            avg_precision=res_val["pr_curve"][model_name]["avg_precision"],
            model_name=model_name,
            ax=nexttile(),
        )
        plot.figsets(title=model_name, sp=2)

        # plot cm
        plot_cm(res_val["confusion_matrix"][model_name], ax=nexttile(), normalize=False)
        plot.figsets(title=model_name, sp=2)

def cal_auc_ci(y_true, y_pred, n_bootstraps=1000, ci=0.95, random_state=1,verbose=True):
    y_true = np.asarray(y_true)
    y_pred = np.asarray(y_pred)
    bootstrapped_scores = []
    if verbose:
        print("auroc score:", roc_auc_score(y_true, y_pred))
    rng = np.random.RandomState(random_state)
    for i in range(n_bootstraps):
        # bootstrap by sampling with replacement on the prediction indices
        indices = rng.randint(0, len(y_pred), len(y_pred))
        if len(np.unique(y_true[indices])) < 2:
            # We need at least one positive and one negative sample for ROC AUC
            # to be defined: reject the sample
            continue
        if isinstance(y_true, np.ndarray):
            score = roc_auc_score(y_true[indices], y_pred[indices])
        else:
            score = roc_auc_score(y_true.iloc[indices], y_pred.iloc[indices])
        bootstrapped_scores.append(score)
        # print("Bootstrap #{} ROC area: {:0.3f}".format(i + 1, score))
    sorted_scores = np.array(bootstrapped_scores)
    sorted_scores.sort()

    # Computing the lower and upper bound of the 90% confidence interval
    # You can change the bounds percentiles to 0.025 and 0.975 to get
    # a 95% confidence interval instead.
    confidence_lower = sorted_scores[int((1 - ci) * len(sorted_scores))]
    confidence_upper = sorted_scores[int(ci * len(sorted_scores))]
    if verbose:
        print(
        "Confidence interval for the score: [{:0.3f} - {:0.3}]".format(
            confidence_lower, confidence_upper
        )
    )
    return confidence_lower, confidence_upper


def plot_roc_curve(
    fpr=None,
    tpr=None,
    mean_auc=None,
    lower_ci=None,
    upper_ci=None,
    model_name=None,
    color="#FF8F00",
    lw=2,
    alpha=0.1,
    ci_display=True,
    title="ROC Curve",
    xlabel="1−Specificity",
    ylabel="Sensitivity",
    legend_loc="lower right",
    diagonal_color="0.5",
    figsize=(5, 5),
    ax=None,
    **kwargs,
):
    if ax is None:
        fig, ax = plt.subplots(figsize=figsize)
    if mean_auc is not None:
        model_name = "ROC curve" if model_name is None else model_name
        if ci_display:
            label = f"{model_name} (AUC = {mean_auc:.3f})\n95% CI: {lower_ci:.3f} - {upper_ci:.3f}"
        else:
            label = f"{model_name} (AUC = {mean_auc:.3f})"
    else:
        label = None

    # Plot ROC curve and the diagonal reference line
    ax.fill_between(fpr, tpr, alpha=alpha, color=color)
    ax.plot([0, 1], [0, 1], color=diagonal_color, clip_on=False, linestyle="--")
    ax.plot(fpr, tpr, color=color, lw=lw, label=label,clip_on=False, **kwargs)
    # Setting plot limits, labels, and title
    ax.set_xlim([-0.01, 1.0])
    ax.set_ylim([0.0, 1.0])
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.set_title(title)
    ax.legend(loc=legend_loc)
    return ax


# * usage: ml2ls.plot_roc_curve(fpr, tpr, mean_auc, lower_ci, upper_ci)
# for model_name in flatten(validation_results["roc_curve"].keys())[2:]:
#     fpr = validation_results["roc_curve"][model_name]["fpr"]
#     tpr = validation_results["roc_curve"][model_name]["tpr"]
#     (lower_ci, upper_ci) = validation_results["roc_curve"][model_name]["ci95"]
#     mean_auc = validation_results["roc_curve"][model_name]["auc"]

#     # Plotting
#     ml2ls.plot_roc_curve(fpr, tpr, mean_auc, lower_ci, upper_ci)
#     figsets(title=model_name)


def plot_pr_curve(
    recall=None,
    precision=None,
    avg_precision=None,
    model_name=None,
    lw=2,
    figsize=[5, 5],
    title="Precision-Recall Curve",
    xlabel="Recall",
    ylabel="Precision",
    alpha=0.1,
    color="#FF8F00",
    legend_loc="lower left",
    ax=None,
    **kwargs,
):
    if ax is None:
        fig, ax = plt.subplots(figsize=figsize)
    model_name = "PR curve" if model_name is None else model_name
    # Plot Precision-Recall curve
    ax.plot(
        recall,
        precision,
        lw=lw,
        color=color,
        label=(f"{model_name} (AUC={avg_precision:.2f})"),
        clip_on=False,
        **kwargs,
    )
    # Fill area under the curve
    ax.fill_between(recall, precision, alpha=alpha, color=color)

    # Customize axes
    ax.set_title(title)
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.set_xlim([-0.01, 1.0])
    ax.set_ylim([0.0, 1.0])
    ax.grid(False)
    ax.legend(loc=legend_loc)
    return ax


# * usage: ml2ls.plot_pr_curve()
# for md_name in flatten(validation_results["pr_curve"].keys()):
#     ml2ls.plot_pr_curve(
#         recall=validation_results["pr_curve"][md_name]["recall"],
#         precision=validation_results["pr_curve"][md_name]["precision"],
#         avg_precision=validation_results["pr_curve"][md_name]["avg_precision"],
#         model_name=md_name,
#         lw=2,
#         alpha=0.1,
#         color="r",
#     )


def plot_cm(
    cm,
    labels_name=None,
    thresh=0.8,
    axis_labels=None,
    cmap="Reds",
    normalize=True,
    xlabel="Predicted Label",
    ylabel="Actual Label",
    fontsize=12,
    figsize=[5, 5],
    ax=None,
):
    if ax is None:
        fig, ax = plt.subplots(figsize=figsize)

    cm_normalized = np.round(
        cm.astype("float") / cm.sum(axis=1)[:, np.newaxis] * 100, 2
    )
    cm_value = cm_normalized if normalize else cm.astype("int")
    # Plot the heatmap
    cax = ax.imshow(cm_normalized, interpolation="nearest", cmap=cmap)
    plt.colorbar(cax, ax=ax, fraction=0.046, pad=0.04)
    cax.set_clim(0, 100)

    # Define tick labels based on provided labels
    num_local = np.arange(len(labels_name)) if labels_name is not None else range(2)
    if axis_labels is None:
        axis_labels = labels_name if labels_name is not None else ["No", "Yes"]
    ax.set_xticks(num_local)
    ax.set_xticklabels(axis_labels)
    ax.set_yticks(num_local)
    ax.set_yticklabels(axis_labels)
    ax.set_ylabel(ylabel)
    ax.set_xlabel(xlabel)

    # Add TN, FP, FN, TP annotations specifically for binary classification (2x2 matrix)
    if labels_name is None or len(labels_name) == 2:
        # True Negative (TN), False Positive (FP), False Negative (FN), and True Positive (TP)
        #                 Predicted
        #                0   |   1
        #             ----------------
        #         0 |   TN   |  FP
        # Actual      ----------------
        #         1 |   FN   |  TP
        tn_label = "TN"
        fp_label = "FP"
        fn_label = "FN"
        tp_label = "TP"

        # Adjust positions slightly for TN, FP, FN, TP labels
        ax.text(
            0,
            0,
            (
                f"{tn_label}:{cm_normalized[0, 0]:.2f}%"
                if normalize
                else f"{tn_label}:{cm_value[0, 0]}"
            ),
            ha="center",
            va="center",
            color="white" if cm_normalized[0, 0] > thresh * 100 else "black",
            fontsize=fontsize,
        )
        ax.text(
            1,
            0,
            (
                f"{fp_label}:{cm_normalized[0, 1]:.2f}%"
                if normalize
                else f"{fp_label}:{cm_value[0, 1]}"
            ),
            ha="center",
            va="center",
            color="white" if cm_normalized[0, 1] > thresh * 100 else "black",
            fontsize=fontsize,
        )
        ax.text(
            0,
            1,
            (
                f"{fn_label}:{cm_normalized[1, 0]:.2f}%"
                if normalize
                else f"{fn_label}:{cm_value[1, 0]}"
            ),
            ha="center",
            va="center",
            color="white" if cm_normalized[1, 0] > thresh * 100 else "black",
            fontsize=fontsize,
        )
        ax.text(
            1,
            1,
            (
                f"{tp_label}:{cm_normalized[1, 1]:.2f}%"
                if normalize
                else f"{tp_label}:{cm_value[1, 1]}"
            ),
            ha="center",
            va="center",
            color="white" if cm_normalized[1, 1] > thresh * 100 else "black",
            fontsize=fontsize,
        )
    else:
        # Annotate cells with normalized percentage values
        for i in range(len(labels_name)):
            for j in range(len(labels_name)):
                val = cm_normalized[i, j]
                color = "white" if val > thresh * 100 else "black"
                ax.text(
                    j,
                    i,
                    f"{val:.2f}%",
                    ha="center",
                    va="center",
                    color=color,
                    fontsize=fontsize,
                )
    
    plot.figsets(ax=ax, 
                 boxloc="none"
                 )
    return ax

def rank_models(
    cv_test_scores,
    rm_outlier=False,
    metric_weights=None,
    plot_=True,
):
    """
    Selects the best model based on a multi-metric scoring approach, with outlier handling, optional visualization,
    and additional performance metrics.

    Parameters:
    - cv_test_scores (pd.DataFrame): DataFrame with cross-validation results across multiple metrics.
                                     Assumes columns are 'Classifier', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc'.
    - metric_weights (dict): Dictionary specifying weights for each metric (e.g., {'accuracy': 0.2, 'precision': 0.3, ...}).
                             If None, default weights are applied equally across available metrics.
                a. equal_weights(standard approch): 所有的metrics同等重要
                    e.g., {"accuracy": 0.2, "precision": 0.2, "recall": 0.2, "f1": 0.2, "roc_auc": 0.2}
                b. accuracy_focosed:  classification correctness (e.g., in balanced datasets), accuracy might be weighted more heavily.
                    e.g., {"accuracy": 0.4, "precision": 0.2, "recall": 0.2, "f1": 0.1, "roc_auc": 0.1}
                c. Precision and Recall Emphasis: In cases where false positives and false negatives are particularly important (such as
                    in medical applications or fraud detection), precision and recall may be weighted more heavily.
                    e.g., {"accuracy": 0.2, "precision": 0.3, "recall": 0.3, "f1": 0.1, "roc_auc": 0.1}
                d. F1-Focused: When balance between precision and recall is crucial (e.g., in imbalanced datasets)
                    e.g., {"accuracy": 0.2, "precision": 0.2, "recall": 0.2, "f1": 0.3, "roc_auc": 0.1}
                e. ROC-AUC Emphasis: In some cases, ROC AUC may be prioritized, particularly in classification tasks where class imbalance
                    is present, as ROC AUC accounts for the model's performance across all classification thresholds.
                    e.g., {"accuracy": 0.1, "precision": 0.2, "recall": 0.2, "f1": 0.3, "roc_auc": 0.3}

    - normalize (bool): Whether to normalize scores of each metric to range [0, 1].
    - visualize (bool): If True, generates visualizations (e.g., bar plot, radar chart).
    - outlier_threshold (float): The threshold to detect outliers using the IQR method. Default is 1.5.
    - cv_folds (int): The number of cross-validation folds used.

    Returns:
    - best_model (str): Name of the best model based on the combined metric scores.
    - scored_df (pd.DataFrame): DataFrame with an added 'combined_score' column used for model selection.
    - visualizations (dict): A dictionary containing visualizations if `visualize=True`.
    """
    from sklearn.preprocessing import MinMaxScaler
    import seaborn as sns
    import matplotlib.pyplot as plt
    from py2ls import plot

    # Check for missing metrics and set default weights if not provided
    available_metrics = cv_test_scores.columns[1:]  # Exclude 'Classifier' column
    if metric_weights is None:
        metric_weights = {
            metric: 1 / len(available_metrics) for metric in available_metrics
        }  # Equal weight if not specified
    elif metric_weights == "a":
        metric_weights = {
            "accuracy": 0.2,
            "precision": 0.2,
            "recall": 0.2,
            "f1": 0.2,
            "roc_auc": 0.2,
        }
    elif metric_weights == "b":
        metric_weights = {
            "accuracy": 0.4,
            "precision": 0.2,
            "recall": 0.2,
            "f1": 0.1,
            "roc_auc": 0.1,
        }
    elif metric_weights == "c":
        metric_weights = {
            "accuracy": 0.2,
            "precision": 0.3,
            "recall": 0.3,
            "f1": 0.1,
            "roc_auc": 0.1,
        }
    elif metric_weights == "d":
        metric_weights = {
            "accuracy": 0.2,
            "precision": 0.2,
            "recall": 0.2,
            "f1": 0.3,
            "roc_auc": 0.1,
        }
    elif metric_weights == "e":
        metric_weights = {
            "accuracy": 0.1,
            "precision": 0.2,
            "recall": 0.2,
            "f1": 0.3,
            "roc_auc": 0.3,
        }
    else:
        metric_weights = {
            metric: 1 / len(available_metrics) for metric in available_metrics
        }

    # Normalize weights if they don’t sum to 1
    total_weight = sum(metric_weights.values())
    metric_weights = {
        metric: weight / total_weight for metric, weight in metric_weights.items()
    }
    if rm_outlier:
        cv_test_scores_ = ips.df_outlier(cv_test_scores)
    else:
        cv_test_scores_=cv_test_scores

    # Normalize the scores of metrics if normalize is True
    scaler = MinMaxScaler()
    normalized_scores = pd.DataFrame(
        scaler.fit_transform(cv_test_scores_[available_metrics]),
        columns=available_metrics,
    )
    cv_test_scores_ = pd.concat(
        [cv_test_scores_[["Classifier"]], normalized_scores], axis=1
    )

    # Calculate weighted scores for each model
    cv_test_scores_["combined_score"] = sum(
        cv_test_scores_[metric] * weight for metric, weight in metric_weights.items()
    )
    top_models = cv_test_scores_.sort_values(by="combined_score", ascending=False)
    cv_test_scores = cv_test_scores.loc[top_models.index]
    top_models.reset_index(drop=True, inplace=True)
    cv_test_scores.reset_index(drop=True, inplace=True)

    if plot_:

        def generate_bar_plot(ax, cv_test_scores):
            ax = plot.plotxy(
                y="Classifier", x="combined_score", data=cv_test_scores, kind="bar"
            )
            plt.title("Classifier Performance")
            plt.tight_layout()
            return plt 

        nexttile = plot.subplot(2, 2, figsize=[10, 7])
        generate_bar_plot(nexttile(), top_models.dropna())
        plot.radar(
            ax=nexttile(projection="polar"),
            data=cv_test_scores.set_index("Classifier"),
            ylim=[0.5, 1],
            color=plot.get_color(10),
            alpha=0.05,
            circular=1,
        )
    return cv_test_scores


# # Example Usage:
# metric_weights = {
#     "accuracy": 0.2,
#     "precision": 0.3,
#     "recall": 0.2,
#     "f1": 0.2,
#     "roc_auc": 0.1,
# }
# cv_test_scores = res["cv_test_scores"].copy()
# best_model = rank_models(
#     cv_test_scores, metric_weights=metric_weights, normalize=True, plot_=True
# )

# figsave("classifier_performance.pdf")

def predict(
    x_train: pd.DataFrame,
    y_train: pd.Series,
    x_true: pd.DataFrame=None,
    y_true: Optional[pd.Series] = None,
    common_features: set = None,
    purpose: str = "classification",  # 'classification' or 'regression'
    cls: Optional[Dict[str, Any]] = None,
    metrics: Optional[List[str]] = None,
    random_state: int = 1,
    smote: bool = False,
    n_jobs:int  = -1,
    plot_: bool = True,
    dir_save:str="./",
    test_size:float=0.2,# specific only when x_true is None
    cv_folds:int=5,# more cv_folds 得更加稳定,auc可能更低
    cv_level:str="l",#"s":'low',"m":'medium',"l":"high"
    class_weight: str = "balanced",
    verbose:bool=False,
) -> pd.DataFrame:
    """ 
        第一种情况是内部拆分，第二种是直接预测，第三种是外部验证。
        Usage:
            (1). predict(x_train, y_train,...) 对 x_train 进行拆分训练/测试集，并在测试集上进行验证.
                predict 函数会根据 test_size 参数，将 x_train 和 y_train 拆分出内部测试集。然后模型会在拆分出的训练集上进行训练，并在测试集上验证效果。
            (2). predict(x_train, y_train, x_true,...)使用 x_train 和 y_train 训练并对 x_true 进行预测
                由于传入了 x_true，函数会跳过 x_train 的拆分，直接使用全部的 x_train 和 y_train 进行训练。然后对 x_true 进行预测，但由于没有提供 y_true，
                因此无法与真实值进行对比。
            (3). predict(x_train, y_train, x_true, y_true,...)使用 x_train 和 y_train 训练，并验证 x_true 与真实标签 y_true.
                predict 函数会在 x_train 和 y_train 上进行训练，并将 x_true 作为测试集。由于提供了 y_true，函数可以将预测结果与 y_true 进行对比，从而
                计算验证指标，完成对 x_true 的真正验证。
        trains and validates a variety of machine learning models for both classification and regression tasks. 
        It supports hyperparameter tuning with grid search and includes additional features like cross-validation, 
        feature scaling, and handling of class imbalance through SMOTE.

        Parameters:
            - x_train (pd.DataFrame):Training feature data, structured with each row as an observation and each column as a feature.
            - y_train (pd.Series):Target variable for the training dataset.
            - x_true (pd.DataFrame, optional):Test feature data. If not provided, the function splits x_train based on test_size.
            - y_true (pd.Series, optional):Test target values. If not provided, y_train is split into training and testing sets.
            - common_features (set, optional):Specifies a subset of features common across training and test data.
            - purpose (str, default = "classification"):Defines whether the task is "classification" or "regression". Determines which 
                metrics and models are applied.
            - cls (dict, optional):Dictionary to specify custom classifiers/regressors. Defaults to a set of common models if not provided.
            - metrics (list, optional):List of evaluation metrics (like accuracy, F1 score) used for model evaluation.
            - random_state (int, default = 1):Random seed to ensure reproducibility.
            - smote (bool, default = False):Applies Synthetic Minority Oversampling Technique (SMOTE) to address class imbalance if enabled.
            - n_jobs (int, default = -1):Number of parallel jobs for computation. Set to -1 to use all available cores.
            - plot_ (bool, default = True):If True, generates plots of the model evaluation metrics.
            - test_size (float, default = 0.2):Test data proportion if x_true is not provided.
            - cv_folds (int, default = 5):Number of cross-validation folds.
            - cv_level (str, default = "l"):Sets the detail level of cross-validation. "s" for low, "m" for medium, and "l" for high.
            - class_weight (str, default = "balanced"):Balances class weights in classification tasks.
            - verbose (bool, default = False):If True, prints detailed output during model training.
            - dir_save (str, default = "./"):Directory path to save plot outputs and results.

        Key Steps in the Function:
            Model Initialization: Depending on purpose, initializes either classification or regression models.
            Feature Selection: Ensures training and test sets have matching feature columns.
            SMOTE Application: Balances classes if smote is enabled and the task is classification.
            Cross-Validation and Hyperparameter Tuning: Utilizes GridSearchCV for model tuning based on cv_level.
            Evaluation and Plotting: Outputs evaluation metrics like AUC, confusion matrices, and optional plotting of performance metrics.
    """
    from tqdm import tqdm
    from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, ExtraTreesClassifier, ExtraTreesRegressor, BaggingClassifier, BaggingRegressor, AdaBoostClassifier, AdaBoostRegressor
    from sklearn.svm import SVC, SVR
    from sklearn.tree import DecisionTreeRegressor
    from sklearn.linear_model import LogisticRegression, ElasticNet, ElasticNetCV, LinearRegression, Lasso,RidgeClassifierCV, Perceptron, SGDClassifier
    from sklearn.neighbors import KNeighborsClassifier,KNeighborsRegressor
    from sklearn.naive_bayes import GaussianNB,BernoulliNB
    from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
    import xgboost as xgb
    import lightgbm as lgb
    import catboost as cb
    from sklearn.neural_network import MLPClassifier, MLPRegressor
    from sklearn.model_selection import GridSearchCV, StratifiedKFold, KFold
    from sklearn.discriminant_analysis import LinearDiscriminantAnalysis,QuadraticDiscriminantAnalysis
    from sklearn.preprocessing import PolynomialFeatures


    # 拼写检查
    purpose=ips.strcmp(purpose,['classification','regression'])[0]
    print(f"{purpose} processing...")
    # Default models or regressors if not provided
    if purpose == "classification":
        model_ = {
            "Random Forest": RandomForestClassifier(random_state=random_state, class_weight=class_weight),

            # SVC (Support Vector Classification) 
            "SVM": SVC(kernel="rbf",probability=True,class_weight=class_weight,random_state=random_state),

            # fit the best model without enforcing sparsity, which means it does not directly perform feature selection.
            "Logistic Regression": LogisticRegression(class_weight=class_weight, random_state=random_state),

            # Logistic Regression with L1 Regularization (Lasso)
            "Lasso Logistic Regression": LogisticRegression(penalty="l1", solver="saga", random_state=random_state),
            "Gradient Boosting": GradientBoostingClassifier(random_state=random_state),
            "XGBoost": xgb.XGBClassifier(eval_metric="logloss",random_state=random_state,),
            "KNN": KNeighborsClassifier(n_neighbors=5),
            "Naive Bayes": GaussianNB(),
            "Linear Discriminant Analysis": LinearDiscriminantAnalysis(),
            "AdaBoost":  AdaBoostClassifier(algorithm='SAMME', random_state=random_state),
            # "LightGBM": lgb.LGBMClassifier(random_state=random_state, class_weight=class_weight),
            "CatBoost": cb.CatBoostClassifier(verbose=0, random_state=random_state),
            "Extra Trees": ExtraTreesClassifier(random_state=random_state, class_weight=class_weight),
            "Bagging": BaggingClassifier(random_state=random_state),
            "Neural Network": MLPClassifier(max_iter=500, random_state=random_state),
            "DecisionTree": DecisionTreeClassifier(),
            "Quadratic Discriminant Analysis": QuadraticDiscriminantAnalysis(),
            "Ridge": RidgeClassifierCV(class_weight=class_weight, store_cv_results=True),
            "Perceptron": Perceptron(random_state=random_state),
            "Bernoulli Naive Bayes": BernoulliNB(),
            "SGDClassifier": SGDClassifier(random_state=random_state), 
        }
    elif purpose == "regression":
        model_ = {
            "Random Forest": RandomForestRegressor(random_state=random_state),
            "SVM": SVR(),# SVR (Support Vector Regression)
            # "Lasso": Lasso(random_state=random_state), # 它和LassoCV相同(必须要提供alpha参数),
            "LassoCV": LassoCV(cv=cv_folds, random_state=random_state),#LassoCV自动找出最适alpha,优于Lasso
            "Gradient Boosting": GradientBoostingRegressor(random_state=random_state),
            "XGBoost": xgb.XGBRegressor(eval_metric="rmse",random_state=random_state),
            "Linear Regression": LinearRegression(),
            "Lasso": Lasso(random_state=random_state),
            "AdaBoost": AdaBoostRegressor(random_state=random_state),
            # "LightGBM": lgb.LGBMRegressor(random_state=random_state),
            "CatBoost": cb.CatBoostRegressor(verbose=0, random_state=random_state),
            "Extra Trees": ExtraTreesRegressor(random_state=random_state),
            "Bagging": BaggingRegressor(random_state=random_state),
            "Neural Network": MLPRegressor(max_iter=500, random_state=random_state),
            "ElasticNet": ElasticNet(random_state=random_state),
            "Ridge": Ridge(), 
            "KNN":KNeighborsRegressor()
        }
    # indicate cls: 
    if ips.run_once_within(30):# 10 min
        print(f"supported models: {list(model_.keys())}")
    if cls is None:
        models=model_
    else:
        if not isinstance(cls, list):
            cls=[cls]
        models={}
        for cls_ in cls: 
            cls_ = ips.strcmp(cls_, list(model_.keys()))[0]
            models[cls_] = model_[cls_]
    if 'LightGBM' in models:
        x_train=ips.df_special_characters_cleaner(x_train)
        x_true=ips.df_special_characters_cleaner(x_true) if x_true is not None else None

    if isinstance(y_train, str) and y_train in x_train.columns: 
        y_train_col_name=y_train
        y_train=x_train[y_train]
        y_train=ips.df_encoder(pd.DataFrame(y_train),method='dummy')
        x_train = x_train.drop(y_train_col_name,axis=1)
    else:
        y_train=ips.df_encoder(pd.DataFrame(y_train),method='dummy').values.ravel()

    if x_true is None:
        x_train, x_true, y_train, y_true = train_test_split(
            x_train, 
            y_train, 
            test_size=test_size, 
            random_state=random_state, 
            stratify=y_train if purpose == "classification" else None
        )
        if isinstance(y_train, str) and y_train in x_train.columns: 
            y_train_col_name=y_train
            y_train=x_train[y_train]
            y_train=ips.df_encoder(pd.DataFrame(y_train),method='dummy')
            x_train = x_train.drop(y_train_col_name,axis=1)
        else:
            y_train=ips.df_encoder(pd.DataFrame(y_train),method='dummy').values.ravel()
    if y_true is not None:
        if isinstance(y_true, str) and y_true in x_true.columns: 
            y_true_col_name=y_true
            y_true=x_true[y_true]
            y_true=ips.df_encoder(pd.DataFrame(y_true),method='dummy')
            x_true = x_true.drop(y_true_col_name,axis=1)
        else:
            y_true=ips.df_encoder(pd.DataFrame(y_true),method='dummy').values.ravel()

    # to convert the 2D to 1D: 2D column-vector format (like [[1], [0], [1], ...]) instead of a 1D array ([1, 0, 1, ...]

    # y_train=y_train.values.ravel() if y_train is not None else None
    # y_true=y_true.values.ravel() if y_true is not None else None
    y_train = y_train.ravel() if isinstance(y_train, np.ndarray) else y_train.values.ravel()
    y_true = y_true.ravel() if isinstance(y_true, np.ndarray) else y_true.values.ravel()

    
    # Ensure common features are selected
    if common_features is not None:
        x_train, x_true = x_train[common_features], x_true[common_features]
    else:
        share_col_names = ips.shared(x_train.columns, x_true.columns,verbose=verbose)
        x_train, x_true =x_train[share_col_names], x_true[share_col_names]

    x_train, x_true = ips.df_scaler(x_train), ips.df_scaler(x_true)
    x_train, x_true = ips.df_encoder(x_train, method="dummy"), ips.df_encoder(
        x_true, method="dummy"
    )

    # Handle class imbalance using SMOTE (only for classification)
    if (
        smote
        and purpose == "classification"
        and y_train.value_counts(normalize=True).max() < 0.8
    ):
        from imblearn.over_sampling import SMOTE

        smote_sampler = SMOTE(random_state=random_state)
        x_train, y_train = smote_sampler.fit_resample(x_train, y_train)

    # Hyperparameter grids for tuning
    if cv_level in ["low",'simple','s','l']:
        param_grids = {
            "Random Forest": {
                "n_estimators": [100],  # One basic option
                "max_depth": [None, 10],
                "min_samples_split": [2],
                "min_samples_leaf": [1],
                "class_weight": [None],
            } if purpose == "classification" else {
                "n_estimators": [100],  # One basic option
                "max_depth": [None, 10],
                "min_samples_split": [2],
                "min_samples_leaf": [1],
                "max_features": [None],
                "bootstrap": [True],       # Only one option for simplicity
            },
            "SVM": {
                "C": [1],
                "gamma": ['scale'],
                "kernel": ['rbf'],
            },
            "Lasso": {
                "alpha": [0.1],
            },
            "LassoCV": {
                "alphas": [[0.1]],
            },
            "Logistic Regression": {
                "C": [1],
                "solver": ['lbfgs'],
                "penalty": ['l2'],
                "max_iter": [500],
            },
            "Gradient Boosting": {
                "n_estimators": [100],
                "learning_rate": [0.1],
                "max_depth": [3],
                "min_samples_split": [2],
                "subsample": [0.8],
            },
            "XGBoost": {
                "n_estimators": [100],
                "max_depth": [3],
                "learning_rate": [0.1],
                "subsample": [0.8],
                "colsample_bytree": [0.8],
            },
            "KNN": {
                "n_neighbors": [3],
                "weights": ['uniform'],
                "algorithm": ['auto'],
                "p": [2],
            } if purpose == 'classification' else {
                'n_neighbors': [3],
                'weights': ['uniform'],
                'metric': ['euclidean'],
                'leaf_size': [30],
                'p': [2],
            },
            "Naive Bayes": {
                "var_smoothing": [1e-9],
            },
            "SVR": {
                "C": [1],
                "gamma": ['scale'],
                "kernel": ['rbf'],
            },
            "Linear Regression": {
                "fit_intercept": [True],
            },
            "Extra Trees": {
                "n_estimators": [100],
                "max_depth": [None, 10],
                "min_samples_split": [2],
                "min_samples_leaf": [1],
            },
            "CatBoost": {
                "iterations": [100],
                "learning_rate": [0.1],
                "depth": [3],
                "l2_leaf_reg": [1],
            },
            "LightGBM": {
                "n_estimators": [100],
                "num_leaves": [31],
                "max_depth": [10],
                'min_data_in_leaf': [20],
                'min_gain_to_split': [0.01],
                'scale_pos_weight': [10],
            },
            "Bagging": {
                "n_estimators": [50],
                "max_samples": [0.7],
                "max_features": [0.7],
            },
            "Neural Network": {
                "hidden_layer_sizes": [(50,)],
                "activation": ["relu"],
                "solver": ["adam"],
                "alpha": [0.0001],
            },
            "Decision Tree": {
                "max_depth": [None, 10],
                "min_samples_split": [2],
                "min_samples_leaf": [1],
                "criterion": ["gini"],
            },
            "AdaBoost": {
                "n_estimators": [50],
                "learning_rate": [0.5],
            },
            "Linear Discriminant Analysis": {
                "solver": ["svd"],
                "shrinkage": [None],
            },
            "Quadratic Discriminant Analysis": {
                'reg_param': [0.0],
                'priors': [None],
                'tol': [1e-4],
            },
            "Ridge": {'class_weight': [None, 'balanced']} if purpose == "classification" else {
                'alpha': [0.1, 1, 10],
            },
            "Perceptron": {
                'alpha': [1e-3],
                'penalty': ['l2'],
                'max_iter': [1000],
                'eta0': [1.0],
            },
            "Bernoulli Naive Bayes": {
                'alpha': [0.1, 1, 10],
                'binarize': [0.0],
                'fit_prior': [True],
            },
            "SGDClassifier": {
                'eta0': [0.01],
                'loss': ['hinge'],
                'penalty': ['l2'],
                'alpha': [1e-3],
                'max_iter': [1000],
                'tol': [1e-3],
                'random_state': [random_state],
                'learning_rate': ['constant'],
            },
        }
    elif cv_level in ['high','advanced','h']:
        param_grids = { 
            "Random Forest": {
                            "n_estimators": [100, 200, 500, 700, 1000],
                            "max_depth": [None, 3, 5, 10, 15, 20, 30],
                            "min_samples_split": [2, 5, 10, 20],
                            "min_samples_leaf": [1, 2, 4],
                            "class_weight": [None, "balanced"] if purpose == "classification" else {},
                        } if purpose == "classification" else {
                            "n_estimators": [100, 200, 500, 700, 1000],
                            "max_depth": [None, 3, 5, 10, 15, 20, 30],
                            "min_samples_split": [2, 5, 10, 20],
                            "min_samples_leaf": [1, 2, 4],
                            "max_features": ['auto', 'sqrt', 'log2'],  # Number of features to consider when looking for the best split
                            "bootstrap": [True, False],             # Whether bootstrap samples are used when building trees
                            },
            "SVM": {
                    "C": [0.001, 0.01, 0.1, 1, 10, 100, 1000],
                    "gamma": ["scale", "auto", 0.001, 0.01, 0.1],
                    "kernel": ["linear", "rbf", "poly"],
                },
            "Logistic Regression": {
                                    "C": [0.001, 0.01, 0.1, 1, 10, 100, 1000],
                                    "solver": ["liblinear", "saga", "newton-cg", "lbfgs"],
                                    "penalty": ["l1", "l2", "elasticnet"],
                                    "max_iter": [100, 200, 300, 500],
                },
            "Lasso":{
                    "alpha": [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0],
                    "max_iter": [500, 1000, 2000, 5000],
                    "tol": [1e-4, 1e-5, 1e-6],
                    "selection": ["cyclic", "random"]
                },
            "LassoCV":{
                        "alphas": [[0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0]],
                        "max_iter": [500, 1000, 2000, 5000],
                        "cv": [3, 5, 10],
                        "tol": [1e-4, 1e-5, 1e-6]
                    },
            "Gradient Boosting": {
                                "n_estimators": [100, 200, 300, 400, 500, 700, 1000],
                                "learning_rate": [0.001, 0.01, 0.1, 0.2, 0.3, 0.5],
                                "max_depth": [3, 5, 7, 9, 15],
                                "min_samples_split": [2, 5, 10, 20],
                                "subsample": [0.8, 1.0],
                },
            "XGBoost": {
                        "n_estimators": [100, 200, 500, 700],
                        "max_depth": [3, 5, 7, 10],
                        "learning_rate": [0.01, 0.1, 0.2, 0.3],
                        "subsample": [0.8, 1.0],
                        "colsample_bytree": [0.8, 0.9, 1.0],
                },
            "KNN": {
                "n_neighbors": [1, 3, 5, 10, 15, 20],
                "weights": ["uniform", "distance"],
                "algorithm": ["auto", "ball_tree", "kd_tree", "brute"],
                "p": [1, 2],  # 1 for Manhattan, 2 for Euclidean distance
                } if purpose=='classification' else {
                    'n_neighbors': [3, 5, 7, 9, 11],  # Number of neighbors
                    'weights': ['uniform', 'distance'],  # Weight function used in prediction
                    'metric': ['euclidean', 'manhattan', 'minkowski'],  # Distance metric
                    'leaf_size': [20, 30, 40, 50],  # Leaf size for KDTree or BallTree algorithms
                    'p': [1, 2]  # Power parameter for the Minkowski metric (1 = Manhattan, 2 = Euclidean)
                    },
            "Naive Bayes": {
                "var_smoothing": [1e-10, 1e-9, 1e-8, 1e-7],
                },
            "AdaBoost": {
                        "n_estimators": [50, 100, 200, 300, 500],
                        "learning_rate": [0.001, 0.01, 0.1, 0.5, 1.0],
                },
            "SVR": {
                    "C": [0.01, 0.1, 1, 10, 100, 1000],
                    "gamma": [0.001, 0.01, 0.1, "scale", "auto"],
                    "kernel": ["linear", "rbf", "poly"],
                },
            "Linear Regression": {
                "fit_intercept": [True, False],
                },
            "Lasso":{
                "alpha": [0.001, 0.01, 0.1, 1.0, 10.0, 100.0],
                "max_iter": [1000, 2000]  # Higher iteration limit for fine-tuning
                },
            "Extra Trees": {
                "n_estimators": [100, 200, 500, 700, 1000],
                "max_depth": [None, 5, 10, 15, 20, 30],
                "min_samples_split": [2, 5, 10, 20],
                "min_samples_leaf": [1, 2, 4]
                },
            "CatBoost": {
                "iterations": [100, 200, 500],
                "learning_rate": [0.001, 0.01, 0.1, 0.2],
                "depth": [3, 5, 7, 10],
                "l2_leaf_reg": [1, 3, 5, 7, 10],
                "border_count": [32, 64, 128],
                },
            "LightGBM": {
                "n_estimators": [100, 200, 500, 700, 1000],
                "learning_rate": [0.001, 0.01, 0.1, 0.2],
                "num_leaves": [31, 50, 100, 200],
                "max_depth": [-1, 5, 10, 20, 30],
                "min_child_samples": [5, 10, 20],
                "subsample": [0.8, 1.0],
                "colsample_bytree": [0.8, 0.9, 1.0],
                },
            "Neural Network": {
                "hidden_layer_sizes": [(50,), (100,), (100, 50), (200, 100)],
                "activation": ["relu", "tanh", "logistic"],
                "solver": ["adam", "sgd", "lbfgs"],
                "alpha": [0.0001, 0.001, 0.01],
                "learning_rate": ["constant", "adaptive"],
                },
            "Decision Tree": {
                "max_depth": [None, 5, 10, 20, 30],
                "min_samples_split": [2, 5, 10, 20],
                "min_samples_leaf": [1, 2, 5, 10],
                "criterion": ["gini", "entropy"],
                "splitter": ["best", "random"],
                }, 
            "Linear Discriminant Analysis": {
                "solver": ["svd", "lsqr", "eigen"],
                "shrinkage": [None, "auto", 0.1, 0.5, 1.0],  # shrinkage levels for 'lsqr' and 'eigen'
                },
            'Ridge': {'class_weight': [None, 'balanced']} if purpose == "classification" else {
                'alpha': [0.1, 1, 10, 100, 1000],
                'solver': ['auto', 'svd', 'cholesky', 'lsqr', 'lbfgs'],
                'fit_intercept': [True, False],  # Whether to calculate the intercept
                'normalize': [True, False]  # If True, the regressors X will be normalized
                }
            }
    else: # median level
        param_grids = { 
            "Random Forest": {
                "n_estimators": [100, 200, 500],
                "max_depth": [None, 10, 20, 30],
                "min_samples_split": [2, 5, 10],
                "min_samples_leaf": [1, 2, 4],
                "class_weight": [None, "balanced"]
                } if purpose == "classification" else {
                        "n_estimators": [100, 200, 500],
                        "max_depth": [None, 10, 20, 30],
                        "min_samples_split": [2, 5, 10],
                        "min_samples_leaf": [1, 2, 4],
                        "max_features": ['auto', 'sqrt', 'log2'],  # Number of features to consider when looking for the best split
                        "bootstrap": [True, False],             # Whether bootstrap samples are used when building trees
                        },
            "SVM": {
                "C": [0.1, 1, 10, 100],  # Regularization strength
                "gamma": ['scale', 'auto'],  # Common gamma values
                "kernel": ['rbf', 'linear', 'poly'],
            },
            "Logistic Regression": {
                "C": [0.1, 1, 10, 100],  # Regularization strength
                "solver": ['lbfgs', 'liblinear', 'saga'],  # Common solvers
                "penalty": ['l2'],  # L2 penalty is most common
                "max_iter": [500, 1000, 2000],  # Increased max_iter for better convergence
            },
            "Lasso":{
                "alpha": [0.001, 0.01, 0.1, 1.0, 10.0, 100.0],
                "max_iter": [500, 1000, 2000]
            },
            "LassoCV":{
                "alphas": [[0.001, 0.01, 0.1, 1.0, 10.0, 100.0]],
                "max_iter": [500, 1000, 2000]
            },
            "Gradient Boosting": {
                "n_estimators": [100, 200, 500],
                "learning_rate": [0.01, 0.1, 0.2],
                "max_depth": [3, 5, 7],
                "min_samples_split": [2, 5, 10],
                "subsample": [0.8, 1.0],
            },
            "XGBoost": {
                "n_estimators": [100, 200, 500],
                "max_depth": [3, 5, 7],
                "learning_rate": [0.01, 0.1, 0.2],
                "subsample": [0.8, 1.0],
                "colsample_bytree": [0.8, 1.0],
            },
            "KNN": {
                "n_neighbors": [3, 5, 7, 10],
                "weights": ['uniform', 'distance'],
                "algorithm": ['auto', 'ball_tree', 'kd_tree', 'brute'],
                "p": [1, 2],
                } if purpose=='classification' else {
                    'n_neighbors': [3, 5, 7, 9, 11],  # Number of neighbors
                    'weights': ['uniform', 'distance'],  # Weight function used in prediction
                    'metric': ['euclidean', 'manhattan', 'minkowski'],  # Distance metric
                    'leaf_size': [20, 30, 40, 50],  # Leaf size for KDTree or BallTree algorithms
                    'p': [1, 2]  # Power parameter for the Minkowski metric (1 = Manhattan, 2 = Euclidean)
                    },
            "Naive Bayes": {
                "var_smoothing": [1e-9, 1e-8, 1e-7],
            },
            "SVR": {
                "C": [0.1, 1, 10, 100],
                "gamma": ['scale', 'auto'],
                "kernel": ['rbf', 'linear'],
            },
            "Linear Regression": {
                "fit_intercept": [True, False],
            },
            "Lasso": {
                "alpha": [0.1, 1.0, 10.0],
                "max_iter": [1000, 2000],  # Sufficient iterations for convergence
            },
            "Extra Trees": {
                "n_estimators": [100, 200, 500],
                "max_depth": [None, 10, 20, 30],
                "min_samples_split": [2, 5, 10],
                "min_samples_leaf": [1, 2, 4],
            },
            "CatBoost": {
                "iterations": [100, 200],
                "learning_rate": [0.01, 0.1],
                "depth": [3, 6, 10],
                "l2_leaf_reg": [1, 3, 5, 7],
            },
            "LightGBM": {
                "n_estimators": [100, 200, 500],
                "learning_rate": [0.01, 0.1],
                "num_leaves": [31, 50, 100],
                "max_depth": [-1, 10, 20],
                'min_data_in_leaf': [20],     # Minimum samples in each leaf
                'min_gain_to_split': [0.01],  # Minimum gain to allow a split
                'scale_pos_weight': [10],     # Address class imbalance
            }, 
            "Bagging": {
                "n_estimators": [10, 50, 100],
                "max_samples": [0.5, 0.7, 1.0],
                "max_features": [0.5, 0.7, 1.0],
            },
            "Neural Network": {
                "hidden_layer_sizes": [(50,), (100,), (100, 50)],
                "activation": ["relu", "tanh"],
                "solver": ["adam", "sgd"],
                "alpha": [0.0001, 0.001],
            },
            "Decision Tree": {
                "max_depth": [None, 10, 20],
                "min_samples_split": [2, 10],
                "min_samples_leaf": [1, 4],
                "criterion": ["gini", "entropy"],
            },
            "AdaBoost": {
                "n_estimators": [50, 100],
                "learning_rate": [0.5, 1.0],
            },
            "Linear Discriminant Analysis": {
                "solver": ["svd", "lsqr", "eigen"],
                "shrinkage": [None, "auto"],
            },            "Quadratic Discriminant Analysis":{
                'reg_param': [0.0, 0.1, 0.5, 1.0],  # Regularization parameter
                'priors': [None, [0.5, 0.5], [0.3, 0.7]],  # Class priors
                'tol': [1e-4, 1e-3, 1e-2]  # Tolerance value for the convergence of the algorithm
                }, 
            "Perceptron":{
                'alpha': [1e-4, 1e-3, 1e-2],  # Regularization parameter
                'penalty': ['l2', 'l1', 'elasticnet'],  # Regularization penalty
                'max_iter': [1000, 2000],  # Maximum number of iterations
                'eta0': [1.0, 0.1],  # Learning rate for gradient descent
                'tol': [1e-3, 1e-4, 1e-5],  # Tolerance for stopping criteria
                'random_state': [random_state]  # Random state for reproducibility
                },
            "Bernoulli Naive Bayes":{
                'alpha': [0.1, 1.0, 10.0],  # Additive (Laplace) smoothing parameter
                'binarize': [0.0, 0.5, 1.0],  # Threshold for binarizing the input features
                'fit_prior': [True, False]  # Whether to learn class prior probabilities
                },
            "SGDClassifier":{
                'eta0': [0.01, 0.1, 1.0],
                'loss': ['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron'],  # Loss function
                'penalty': ['l2', 'l1', 'elasticnet'],  # Regularization penalty
                'alpha': [1e-4, 1e-3, 1e-2],  # Regularization strength
                'l1_ratio': [0.15, 0.5, 0.85],  # L1 ratio for elasticnet penalty
                'max_iter': [1000, 2000],  # Maximum number of iterations
                'tol': [1e-3, 1e-4],  # Tolerance for stopping criteria
                'random_state': [random_state],  # Random state for reproducibility
                'learning_rate': ['constant', 'optimal', 'invscaling', 'adaptive'],  # Learning rate schedule
                },
            'Ridge': {'class_weight': [None, 'balanced']} if purpose == "classification" else {
                'alpha': [0.1, 1, 10, 100],
                'solver': ['auto', 'svd', 'cholesky', 'lsqr']  # Solver for optimization
            }
        }
 
    results = {}
    # Use StratifiedKFold for classification and KFold for regression
    cv = (
        StratifiedKFold(n_splits=cv_folds, shuffle=True, random_state=random_state)
        if purpose == "classification"
        else KFold(n_splits=cv_folds, shuffle=True, random_state=random_state)
    )

    # Train and validate each model
    for name, clf in tqdm(
                models.items(),
                desc="models",
                colour="green",
                bar_format="{l_bar}{bar} {n_fmt}/{total_fmt}",
            ):
        if verbose:
            print(f"\nTraining and validating {name}:")

        # Grid search with KFold or StratifiedKFold
        gs = GridSearchCV(
            clf,
            param_grid=param_grids.get(name, {}),
            scoring=(
                "roc_auc" if purpose == "classification" else "neg_mean_squared_error"
            ),
            cv=cv,
            n_jobs=n_jobs,
            verbose=verbose,
        )
        gs.fit(x_train, y_train)
        best_clf = gs.best_estimator_
        # make sure x_train and x_test has the same name
        x_true = x_true.reindex(columns=x_train.columns, fill_value=0) 
        y_pred = best_clf.predict(x_true)

        # y_pred_proba
        if hasattr(best_clf, "predict_proba"):
            y_pred_proba = best_clf.predict_proba(x_true)[:, 1]
        elif hasattr(best_clf, "decision_function"):
            # If predict_proba is not available, use decision_function (e.g., for SVM)
            y_pred_proba = best_clf.decision_function(x_true)
            # Ensure y_pred_proba is within 0 and 1 bounds
            y_pred_proba = (y_pred_proba - y_pred_proba.min()) / (
                y_pred_proba.max() - y_pred_proba.min()
            )
        else:
            y_pred_proba = None  # No probability output for certain models
            
        
        validation_scores = {}
        if y_true is not None:
            validation_scores = cal_metrics(y_true, y_pred, y_pred_proba=y_pred_proba, purpose=purpose, average="weighted") 

            # Calculate ROC curve
            # https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html
            if y_pred_proba is not None:
                # fpr, tpr, roc_auc = dict(), dict(), dict()
                fpr, tpr, _ = roc_curve(y_true, y_pred_proba)
                lower_ci, upper_ci = cal_auc_ci(y_true, y_pred_proba,verbose=False)
                roc_auc = auc(fpr, tpr)
                roc_info = {
                    "fpr": fpr.tolist(),
                    "tpr": tpr.tolist(),
                    "auc": roc_auc,
                    "ci95": (lower_ci, upper_ci),
                }
                # precision-recall curve
                precision_, recall_, _ = precision_recall_curve(y_true, y_pred_proba)
                avg_precision_ = average_precision_score(y_true, y_pred_proba)
                pr_info = {
                    "precision": precision_,
                    "recall": recall_,
                    "avg_precision": avg_precision_,
                }
            else:
                roc_info, pr_info = None, None
            if purpose=="classification":
                results[name] = {
                    "best_clf": gs.best_estimator_, 
                    "best_params": gs.best_params_,
                    "auc_indiv":[gs.cv_results_[f'split{i}_test_score'][gs.best_index_] for i in range(cv_folds)],
                    "scores": validation_scores,
                    "roc_curve": roc_info,
                    "pr_curve": pr_info,
                    "confusion_matrix": confusion_matrix(y_true, y_pred),
                    "predictions": y_pred.tolist(),
                    "predictions_proba": (
                        y_pred_proba.tolist() if y_pred_proba is not None else None
                    ),
                }
            else: # "regression"
                results[name] = {
                    "best_clf": gs.best_estimator_, 
                    "best_params": gs.best_params_,
                    "scores": validation_scores, # e.g., neg_MSE, R², etc. 
                    "predictions": y_pred.tolist(),
                    "predictions_proba": (
                        y_pred_proba.tolist() if y_pred_proba is not None else None
                    ),
                }

        else:
            results[name] = {
                "best_clf": gs.best_estimator_, 
                "best_params": gs.best_params_,
                "scores": validation_scores, 
                "predictions": y_pred.tolist(),
                "predictions_proba": (
                    y_pred_proba.tolist() if y_pred_proba is not None else None
                ),
            }

    # Convert results to DataFrame
    df_results = pd.DataFrame.from_dict(results, orient="index")

    # sort
    if y_true is not None and purpose=="classification":
        df_scores = pd.DataFrame(
                df_results["scores"].tolist(), index=df_results["scores"].index
            ).sort_values(by="roc_auc", ascending=False)
        df_results=df_results.loc[df_scores.index]

        if plot_:
            from datetime import datetime
            now_ = datetime.now().strftime("%y%m%d_%H%M%S")
            nexttile=plot.subplot(figsize=[12, 10])
            plot.heatmap(df_scores, kind="direct",ax=nexttile())
            plot.figsets(xangle=30)
            if dir_save:
                ips.figsave(dir_save+f"scores_sorted_heatmap{now_}.pdf")
            if df_scores.shape[0]>1:# draw cluster
                plot.heatmap(df_scores, kind="direct",cluster=True)
                plot.figsets(xangle=30)
                if dir_save:
                    ips.figsave(dir_save+f"scores_clus{now_}.pdf")
    if all([plot_, y_true is not None, purpose=='classification']):
        try:
            if len(models)>3:
                plot_validate_features(df_results)
            else:
                plot_validate_features_single(df_results,figsize=(12,4*len(models)))
            if dir_save:
                ips.figsave(dir_save+f"validate_features{now_}.pdf")
        except Exception as e:
            print(f"Error: 在画图的过程中出现了问题:{e}")
    return df_results


def cal_metrics(y_true, y_pred, y_pred_proba=None, purpose="regression", average="weighted"):
    """
    Calculate regression or classification metrics based on the purpose.
    
    Parameters:
    - y_true: Array of true values.
    - y_pred: Array of predicted labels for classification or predicted values for regression.
    - y_pred_proba: Array of predicted probabilities for classification (optional).
    - purpose: str, "regression" or "classification".
    - average: str, averaging method for multi-class classification ("binary", "micro", "macro", "weighted", etc.).
    
    Returns:
    - validation_scores: dict of computed metrics.
    """
    from sklearn.metrics import (
            mean_squared_error,
            mean_absolute_error,
            mean_absolute_percentage_error,
            explained_variance_score,
            r2_score,
            mean_squared_log_error,
            accuracy_score,
            precision_score,
            recall_score,
            f1_score,
            roc_auc_score,
            matthews_corrcoef,
            confusion_matrix,
            balanced_accuracy_score,
            average_precision_score,
            precision_recall_curve
        )
    validation_scores = {}

    if purpose == "regression":
        y_true = np.asarray(y_true)
        y_true = y_true.ravel()
        y_pred = np.asarray(y_pred) 
        y_pred = y_pred.ravel()
        # Regression metrics
        validation_scores = {
            "mse": mean_squared_error(y_true, y_pred),
            "rmse": np.sqrt(mean_squared_error(y_true, y_pred)),
            "mae": mean_absolute_error(y_true, y_pred),
            "r2": r2_score(y_true, y_pred),
            "mape": mean_absolute_percentage_error(y_true, y_pred),
            "explained_variance": explained_variance_score(y_true, y_pred),
            "mbd": np.mean(y_pred - y_true)  # Mean Bias Deviation
        }
        # Check if MSLE can be calculated
        if np.all(y_true >= 0) and np.all(y_pred >= 0):  # Ensure no negative values
            validation_scores["msle"] = mean_squared_log_error(y_true, y_pred)
        else:
            validation_scores["msle"] = "Cannot be calculated due to negative values"

    elif purpose == "classification":
        # Classification metrics
        validation_scores = {
            "accuracy": accuracy_score(y_true, y_pred),
            "precision": precision_score(y_true, y_pred, average=average),
            "recall": recall_score(y_true, y_pred, average=average),
            "f1": f1_score(y_true, y_pred, average=average),
            "mcc": matthews_corrcoef(y_true, y_pred),
            "specificity": None, 
            "balanced_accuracy": balanced_accuracy_score(y_true, y_pred)
        }

        # Confusion matrix to calculate specificity
        tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
        validation_scores["specificity"] = tn / (tn + fp) if (tn + fp) > 0 else 0  # Specificity calculation

        if y_pred_proba is not None: 
            # Calculate ROC-AUC
            validation_scores["roc_auc"] = roc_auc_score(y_true, y_pred_proba)  
            # PR-AUC (Precision-Recall AUC) calculation
            validation_scores["pr_auc"] = average_precision_score(y_true, y_pred_proba)
    else:
        raise ValueError("Invalid purpose specified. Choose 'regression' or 'classification'.")

    return validation_scores
 