"""
    @release_date  : $release_date
    @version       : $release_version
    @author        : Christos Matsingos, Ka Fu Man 
    
    This file is part of the TrIPP software
    (https://github.com/fornililab/TrIPP).
    Copyright (c) 2024 Christos Matsingos, Ka Fu Man and Arianna Fornili.

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, version 3.

    This program is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program. If not, see <http://www.gnu.org/licenses/>.
"""

import MDAnalysis as mda
import pandas as pd
from tripp._create_clustering_matrix_ import create_clustering_matrix
from tripp._clustering_kmedoids_ import kmedoids_clustering
from tripp._write_clustering_info_ import write_clustering_info
from sklearn.metrics import silhouette_score
import numpy as np
from tripp._calculate_rmsd_matrix_ import calculate_rmsd_matrix
from tripp._clustering_greedy_ import greedy_clustering
from tripp._clustering_dbscan_ import dbscan_clustering
from tripp._clustering_hdbscan_ import hdbscan_clustering
from tripp._pca_ import pca
from tripp._generate_clustering_summary_ import generate_clustering_summary
from tripp._determine_cluster_population_ratio_ import (
    determine_cluster_population_ratio,
)
import logging

class Clustering:
    """
    This class provides a way to extract representative structures from a trajectory
    using the clustering methods: kmedoids, greedy, or dbscan methods. Clustering
    is carried out using the pKa values and (optional) the buried ratio and 
    the relative position of selected residues as features.

    Parameters
    ----------
    topology_file: str
        Path to the topology file. The same formats allowed by MDAnalysis can be used.
    trajectory_file: str or dict
        When str, it is the path of the file containing the trajectory. The same
        formats allowed by MDAnalysis can be used. When dict, the clustering is performed using
        multiple trajectories specified in a dictionary, where trajectory names are
        used as key and file paths as values, e.g.: {'MD1' : 'file1 path', 'MD2' : 'file2 path', ...}.
        The same topology file is used for all trajectories.
    pka_file: str or list
        When str, it is the path of the CSV file with the pka values generated by the
        Trajectory class using trajectory_file and topology_file. 
        When list, it is a list of paths of the CSV pKa files for the
        trajectories in trajectory_file. In this case, the file paths need to be given in the 
        same order as the correspnding trajectories in trajectory_file.
    selections: list of str
        List of residues used for the clustering (selected with MDAnalysis selection syntax)
        For multi-chain systems, your topology and selection must include chain IDs.
    output_directory: str
        Directory where the output files will be saved.
    output_prefix: str
        Output file prefix
    buriedness_file: str or list, default=None
        When str, it is the path of the CSV file with the buried ratio values generated by the
        Trajectory class using trajectory_file and topology_file. 
        When list, it is a list of paths of the CSV buried ratio files for the
        trajectories in trajectory_file. In this case, the file paths need to be given in the 
        same order as the correspnding trajectories in trajectory_file.
        If None, the buried ratio values are not used for clustering.
    include_distances: bool, default=False
        If True, all the possible pairs of distances between charge centers in the selected 
        residues are used as additional features for the clustering alongside the pKa values.
    include_buriedness: bool, default=False
        If True, the buried ratio of the selected residues are used as additional features
        for the clustering alongside the pKa values.
    dimensionality_reduction: bool, befault=False
        If True, dimensionality reduction is performed on the input features using PCA. 
        The minimum number of principal components that explain at least 90% of the 
        total variance is used.
    """
    
    def __init__(
        self,
        topology_file,
        trajectory_file,
        pka_file,
        selections,
        output_directory,
        output_prefix,
        buriedness_file=None,
        include_distances=False,
        include_buriedness=False,
        dimensionality_reduction=False,
    ):
        self.logger = logging.getLogger()
        if self.logger.hasHandlers:
            for handler in self.logger.handlers[:]:
                self.logger.removeHandler(handler)
        logging.Formatter('%(message)s')
        self.logger.setLevel(logging.WARNING)
        self.topology_file = topology_file
        self.trajectory_file = trajectory_file
        self.pka_file = pka_file
        self.selections = selections
        self.output_directory = output_directory
        self.output_prefix = output_prefix
        self.buriedness_file = buriedness_file
        self.include_buriedness = include_buriedness
        self.include_distances = include_distances
        self._labels = None
        self._cluster_center_indices = None

        def make_pka_or_buriedness_df(file):
            if isinstance(file, str):
                df = pd.read_csv(file, index_col="Time [ps]")
                df["Trajectories"] = "Unnamed trajectory"

            elif isinstance(file, list):
                df_list = []
                for index, f in enumerate(file):
                    df = pd.read_csv(f, index_col="Time [ps]")
                    df["Trajectories"] = list(self.trajectory_file.keys())[index]
                    df_list.append(df)
                df = pd.concat(df_list)

            elif file is None:
                df = None

            return df

        self.pka_df = make_pka_or_buriedness_df(pka_file)
        self.buriedness_df = make_pka_or_buriedness_df(buriedness_file)

        (
            self.clustering_matrix,
            self.times,
            self.frames,
            self.trajectory_names,
            self.trajectory_dict,
        ) = create_clustering_matrix(
            self.trajectory_file,
            self.topology_file,
            self.pka_df,
            self.selections,
            self.include_distances,
            self.buriedness_df,
            self.include_buriedness,
        )

        if dimensionality_reduction:
            self.n_components, self.cummulative_variance, self.clustering_matrix = pca(
                self.clustering_matrix
            )

        elif not dimensionality_reduction:
            self.cummulative_variance = None
            self.n_components = None

    def kmedoids(
        self,
        automatic=False,
        n_clusters=None,
        metric="euclidean",
        method="alternate",
        init="k-medoids++",
        max_iter=300,
        random_state=None,
        max_clusters=20,
    ):
        """
        Performs clustering with K-Medoids.
        Parameters
        ----------
        automatic: bool, default=False
            If True, clustering is run with n_clusters ranging from 2 to 
            max_clusters. The silhouette score is determined 
            at each iteration and the best number of clusters is chosen 
            based on the highest average silhouette score.

        n_clusters: int, default=8
            Number of output clusters. Ignored if 
            automatic=True.

        metric, method, init, max_iter, random_state as found in
        sklearn_extra.cluster.KMedoids (https://scikit-learn-extra.readthedocs.io/en/stable/generated/sklearn_extra.cluster.KMedoids.html).

        max_clusters: int, default=20.
            Maximum number of clusters allowed when automatic=True.
        """

        clustering_method = "KMedoids"

        if automatic == False:
            if not n_clusters:
                raise Exception("n_clusters must be specified when automatic=False.") 
            labels, cluster_centers, medoid_indices, cluster_centers_trajectories = (
                kmedoids_clustering(
                    n_clusters=n_clusters,
                    metric=metric,
                    method=method,
                    init=init,
                    max_iter=max_iter,
                    random_state=random_state,
                    clustering_matrix=self.clustering_matrix,
                    frames=self.frames,
                    trajectory_names=self.trajectory_names,
                )
            )
            sil_score = round(silhouette_score(self.clustering_matrix, labels), 4)
            print(
                f"Clustering with {n_clusters} clusters produces an average silhouette score of {sil_score}."
            )
            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": [n_clusters],
                    "Average silhouette score": [sil_score],
                }
            )

        elif automatic == True:
            sil_scores = []
            cluster_nums = []
            for clusters in range(2, max_clusters + 1):
                (
                    labels,
                    cluster_centers,
                    medoid_indices,
                    cluster_centers_trajectories,
                ) = kmedoids_clustering(
                    n_clusters=clusters,
                    metric=metric,
                    method=method,
                    init=init,
                    max_iter=max_iter,
                    random_state=random_state,
                    clustering_matrix=self.clustering_matrix,
                    frames=self.frames,
                    trajectory_names=self.trajectory_names,
                )
                try:
                    sil_score = round(silhouette_score(self.clustering_matrix, labels), 4)
                    sil_scores.append(sil_score)
                    cluster_nums.append(clusters)
                    print(
                        f"Clustering with {clusters} clusters produces an average silhouette score of {sil_score}."
                    )
                except Exception as e:
                    print(f'Error found when evaluating silhouette score with {clusters} clusters so skipping this: {e}.')
                    continue
            sil_scores = np.array(sil_scores)
            best_cluster_n_index = np.argmax(sil_scores)
            sil_score = sil_scores[best_cluster_n_index]
            n_clusters = cluster_nums[best_cluster_n_index]
            labels, cluster_centers, medoid_indices, cluster_centers_trajectories = (
                kmedoids_clustering(
                    n_clusters=n_clusters,
                    metric=metric,
                    method=method,
                    init=init,
                    max_iter=max_iter,
                    random_state=random_state,
                    clustering_matrix=self.clustering_matrix,
                    frames=self.frames,
                    trajectory_names=self.trajectory_names,
                )
            )
            print(
                f"Best number of clusters identified at {n_clusters} with an average silhouette score of {sil_score}."
            )

            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": cluster_nums,
                    "Average silhouette score": sil_scores,
                }
            )
        
        summary = generate_clustering_summary(
            trajectory_file=self.trajectory_file,
            topology_file=self.topology_file,
            pka_file=self.pka_file,
            selections=self.selections,
            include_distances=self.include_distances,
            include_buriedness=self.include_buriedness,
            clustering_method=clustering_method,
            automatic=automatic,
            silhouette_scores=silhouette_scores,
            n_components=self.n_components,
            cummulative_variance=self.cummulative_variance,
            buriedness_file=self.buriedness_file,
        )
        self._labels, self._cluster_centers, self._cluster_center_indices, self._cluster_centers_trajectories  = (
        write_clustering_info(
            summary=summary,
            trajectory_dict=self.trajectory_dict,
            pka_df=self.pka_df,
            times=self.times,
            frames=self.frames,
            trajectory_names=self.trajectory_names,
            labels=labels,
            cluster_centers=cluster_centers,
            cluster_center_indices=medoid_indices,
            cluster_centers_trajectories=cluster_centers_trajectories,
            output_directory=self.output_directory,
            output_prefix=self.output_prefix,
            clustering_method=clustering_method,
            )
        )

    def greedy(
        self,
        automatic=False,
        max_cluster_population=0.95,
        cutoff=0.1,
        max_clusters=20,
        max_cutoffs=20,
    ):
        """
        This function implements the greedy clustering method
        as described in Micheletti et al.
        (DOI: 10.1002/1097-0134(20000901)40:4<662::aid-prot90>3.0.co;2-f).

        automatic: bool, default=False
            If True, clustering is run with a range of distance cutoffs defined by 
            np.arange(step, max_rmsd + step, step), where step = max_rmsd / max_cutoffs
            and max_rmsd is the maximum distance.
            The silhouette score is determined for each cutoff and the best cutoff is chosen 
            based on the highest silhouette score. Cutoffs that produce more than the 
            allowed maximum number of clusters (defined by max_clusters) or that produce 
            only one cluster are excluded.
        
        max_cluster_population: float, default=0.95
            Used when automatic=True. The maximum allowed population value for the most
            populated cluster. If the population of the most populated cluster exceeds 
            this value, the clustering is skipped.
        
        cutoff: float, default=0.1
            Distance cutoff used for clustering. Ignored if automatic=True.

        max_clusters: int, default=20
            Maximum number of clusters allowed when automatic=True.

        max_cutoffs: int, default=20
            Maximum number of distance cutoffs used when automatic=True.
        """
        clustering_method = "greedy"

        rmsd_matrix = calculate_rmsd_matrix(self.clustering_matrix, self.frames)

        if automatic == False:
            # Warning message for default values when automatic=False
            if cutoff == 0.1:
                print("Warning: 'automatic' is set to False and the default value for 'cutoff' (0.1) will be used. Consider setting this parameter explicitly for better results.") 
            (
                labels,
                cluster_centers,
                cluster_center_indices,
                cluster_centers_trajectories,
            ) = greedy_clustering(
                cutoff=cutoff,
                rmsd_matrix=rmsd_matrix,
                frames=self.frames,
                trajectory_names=self.trajectory_names,
            )
            sil_score = round(silhouette_score(self.clustering_matrix, labels), 4)
            print(
                f"Clustering with a cutoff of {cutoff} produces {len(set(labels))} clusters with an average silhouette score of {sil_score}."
            )
            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": [len(set(labels))],
                    "RMSD cutoff": [cutoff],
                    "Average silhouette score": [sil_score],
                }
            )

        elif automatic == True:
            sil_scores = []
            max_rmsd = np.max(rmsd_matrix)
            step = max_rmsd / max_cutoffs
            cutoffs = []
            cluster_nums = []
            for cutoff_i in np.arange(step, max_rmsd + step, step):
                cutoff_i = round(cutoff_i, 4)
                (
                    labels,
                    cluster_centers,
                    cluster_center_indices,
                    cluster_centers_trajectories,
                ) = greedy_clustering(
                    cutoff=cutoff_i,
                    rmsd_matrix=rmsd_matrix,
                    frames=self.frames,
                    trajectory_names=self.trajectory_names,
                )

                if len(labels) == len(set(labels)):
                    print(
                        f"A distance cutoff of {cutoff_i} produces no clusters. Moving on to next cutoff"
                    )

                elif len(set(labels)) == 1:
                    print(
                        f"A distance cutoff of {cutoff_i} produces only one cluster. Moving on to next cutoff."
                    )

                elif len(set(labels)) > max_clusters:
                    print(
                        f"A distance cutoff of {cutoff_i} produces more than {max_clusters} clusters. Moving on to next cutoff."
                    )

                elif (
                    determine_cluster_population_ratio(
                        labels=labels,
                        max_cluster_population=max_cluster_population,
                        clustering_method=clustering_method,
                    )
                    == True
                ):
                    print(
                        f"Clustering with a cutoff of {cutoff_i} results in the most populated cluster containing more than {max_cluster_population*100}% of frames. Moving on to next cutoff."
                    )

                else:
                    cutoffs.append(cutoff_i)
                    try: 
                        sil_score = round(silhouette_score(self.clustering_matrix, labels), 4)
                        sil_scores.append(sil_score)
                        cluster_nums.append(len(set(labels)))
                        print(f"Clustering with a cutoff of {cutoff_i} produces {len(set(labels))} clusters with an average silhouette score of {sil_score}.")
                    except Exception as e:
                        print(f'Error found when evaluating silhouette score with distance cutoff {cutoff_i} so skipping this: {e}.')
                        continue
                        

            sil_scores = np.array(sil_scores)
            best_cutoff_index = np.argmax(sil_scores)
            cutoff = cutoffs[best_cutoff_index]
            sil_score = sil_scores[best_cutoff_index]
            print(
                f"Best cutoff identified at {cutoff} with an average silhouette score of {sil_score}."
            )
            (
                labels,
                cluster_centers,
                cluster_center_indices,
                cluster_centers_trajectories,
            ) = greedy_clustering(
                cutoff=cutoff,
                rmsd_matrix=rmsd_matrix,
                frames=self.frames,
                trajectory_names=self.trajectory_names,
            )

            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": cluster_nums,
                    "Distance cutoff": cutoffs,
                    "Average silhouette score": sil_scores,
                }
            )
        
        summary = generate_clustering_summary(
            trajectory_file=self.trajectory_file,
            topology_file=self.topology_file,
            pka_file=self.pka_file,
            selections=self.selections,
            include_distances=self.include_distances,
            include_buriedness=self.include_buriedness,
            clustering_method=clustering_method,
            automatic=automatic,
            silhouette_scores=silhouette_scores,
            n_components=self.n_components,
            cummulative_variance=self.cummulative_variance,
            buriedness_file=self.buriedness_file,
        )
        self._labels, self._cluster_centers, self._cluster_center_indices, self._cluster_centers_trajectories  = (
            write_clustering_info(
            summary=summary,
            trajectory_dict=self.trajectory_dict,
            pka_df=self.pka_df,
            times=self.times,
            frames=self.frames,
            trajectory_names=self.trajectory_names,
            labels=labels,
            cluster_centers=cluster_centers,
            cluster_center_indices=cluster_center_indices,
            cluster_centers_trajectories=cluster_centers_trajectories,
            output_directory=self.output_directory,
            output_prefix=self.output_prefix,
            clustering_method=clustering_method,
            )
        )

    def dbscan(
        self,
        automatic=False,
        eps_range=(0.005, 0.505, 0.005),
        min_samples_range=(2, 11, 1),
        max_cluster_population=0.95,
        max_clusters=20,
        eps=0.5,
        min_samples=5,
        metric="euclidean",
        metric_params=None,
        algorithm="auto",
        leaf_size=30,
        p=None,
        n_jobs=None,
    ):
        """
        This function implements the DBSCAN method to do the clustering.

        automatic: bool, default=False
            If True, the clustering is run using multiple combinations of eps and 
            min_samples values. The silhouette score is determined for each combination and
            the combination with the highest silhouette score is selected. Combinations 
            that produce more than max_clusters or only one cluster are excluded.

        eps_range: tuple, default=(0.005, 0.505, 0.005)
            The range of epsilon values screened in the automatic grid search. Interpreted as 
            (start, stop, step), values are generated with np.arange(start, stop, step).

        min_samples_range: tuple, default=(2, 11, 1)
            The range of min_samples values screened in the automatic grid search. Interpreted as 
            (start, stop, step), values are generated with np.arange(start, stop, step).
        
        max_cluster_population: float, default=0.95
            Used when automatic=True. The maximum allowed population value for the most
            populated cluster. If the population of the most populated cluster exceeds 
            this value, the clustering is skipped.
            
        max_clusters: int, default=20
            Maximum number of clusters allowed when automatic=True.

        eps, min_samples, metric, metric_params, algorithm,
        leaf_size, p, and n_jobs as found in
        sklearn.cluster.DBSCAN (https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html).
        """

        clustering_method = "DBSCAN"

        if automatic == False:
            if eps == 0.5 and min_samples == 5:
                print("Warning: 'automatic' is set to False and the default values for 'eps' (0.5) and 'min_samples' (5) will be used. Consider setting these parameters explicitly for better results.") 
            (
                labels,
                cluster_centers,
                cluster_center_indices,
                cluster_centers_trajectories,
            ) = dbscan_clustering(
                eps=eps,
                min_samples=min_samples,
                metric=metric,
                metric_params=metric_params,
                algorithm=algorithm,
                leaf_size=leaf_size,
                p=p,
                n_jobs=n_jobs,
                clustering_matrix=self.clustering_matrix,
                frames=self.frames,
                find_centroid=True,
                trajectory_names=self.trajectory_names,
            )
            sil_score = round(silhouette_score(self.clustering_matrix, labels), 4)
            print(
                f"Clustering with parameters eps={eps} and min_samples={min_samples} produces {len(set(labels))} clusters with an average silhouette score of {sil_score}."
            )
            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": [len(set(labels))],
                    "Epsilon": [eps],
                    "Minimum samples": [min_samples],
                    "Average silhouette score": [sil_score],
                }
            )

        elif automatic == True:
            sil_scores = []
            params = []
            cluster_nums = []
            for eps in np.arange(eps_range[0], eps_range[1], eps_range[2]):
                eps = round(eps, 4)
                for min_samples in np.arange(
                    min_samples_range[0], min_samples_range[1], min_samples_range[2]
                ):
                    labels = dbscan_clustering(
                        eps=eps,
                        min_samples=min_samples,
                        metric=metric,
                        metric_params=metric_params,
                        algorithm=algorithm,
                        leaf_size=leaf_size,
                        p=p,
                        n_jobs=n_jobs,
                        clustering_matrix=self.clustering_matrix,
                        frames=self.frames,
                        find_centroid=False,
                        trajectory_names=self.trajectory_names,
                    )

                    if len(set(labels)) == len(labels):
                        print(
                            f"Clustering with parameters eps={eps} and min_samples={min_samples} produces no clusters. Moving on to next set of parameters"
                        )

                    elif len(set(labels)) <= 2:
                        print(
                            f"Clustering with parameters eps={eps} and min_samples={min_samples} produces only one cluster. Moving on to next set of parameters"
                        )

                    elif len(set(labels)) > max_clusters:
                        print(
                            f"Clustering with parameters eps={eps} and min_samples={min_samples} produces more than {max_clusters} clusters. Moving on to next set of parameters"
                        )

                    elif (
                        determine_cluster_population_ratio(
                            labels=labels,
                            max_cluster_population=max_cluster_population,
                            clustering_method=clustering_method,
                        )
                        == True
                    ):
                        print(
                            f"Clustering with parameters eps={eps} and min_samples={min_samples} results in the most populated cluster containing more than {max_cluster_population*100}% of frames. Moving on to next set of parameters."
                        )

                    else:
                        try:
                            sil_score = round(
                                silhouette_score(self.clustering_matrix, labels), 4
                            )
                            sil_scores.append(sil_score)
                            cluster_nums.append(len(set(labels)))
                            params.append([eps, min_samples])
                            print(
                                f"Clustering with parameters eps={eps} and min_samples={min_samples} produces {len(set(labels))} clusters with an average silhouette score of {sil_score}."
                            )
                        except Exception as e:
                            print(f'Error found when evaluating silhouette score with eps={eps} and min_samples={min_samples} so skipping this: {e}.')
                            continue

            sil_scores = np.array(sil_scores)
            best_params_index = np.argmax(sil_scores)
            best_eps, best_min_samples = tuple(params[best_params_index])
            sil_score = sil_scores[best_params_index]
            print(
                f"Best clustering parameters identified at eps={best_eps} and min_samples={best_min_samples} with an average silhouette score of {sil_score}."
            )
            (
                labels,
                cluster_centers,
                cluster_center_indices,
                cluster_centers_trajectories,
            ) = dbscan_clustering(
                eps=best_eps,
                min_samples=best_min_samples,
                metric=metric,
                metric_params=metric_params,
                algorithm=algorithm,
                leaf_size=leaf_size,
                p=p,
                n_jobs=n_jobs,
                clustering_matrix=self.clustering_matrix,
                frames=self.frames,
                find_centroid=True,
                trajectory_names=self.trajectory_names,
            )
            params = np.array(params)

            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": cluster_nums,
                    "Epsilon": params[:, 0],
                    "Minimum samples": params[:, 1],
                    "Average silhouette score": sil_scores,
                }
            )
        self._labels = labels
        self._cluster_center_indices = cluster_center_indices
        
        summary = generate_clustering_summary(
            trajectory_file=self.trajectory_file,
            topology_file=self.topology_file,
            pka_file=self.pka_file,
            selections=self.selections,
            include_distances=self.include_distances,
            include_buriedness=self.include_buriedness,
            clustering_method=clustering_method,
            automatic=automatic,
            silhouette_scores=silhouette_scores,
            n_components=self.n_components,
            cummulative_variance=self.cummulative_variance,
            buriedness_file=self.buriedness_file,
        )
        self._labels, self._cluster_centers, self._cluster_center_indices, self._cluster_centers_trajectories  = (
            write_clustering_info(
            summary=summary,
            trajectory_dict=self.trajectory_dict,
            pka_df=self.pka_df,
            times=self.times,
            frames=self.frames,
            trajectory_names=self.trajectory_names,
            labels=labels,
            cluster_centers=cluster_centers,
            cluster_center_indices=cluster_center_indices,
            cluster_centers_trajectories=cluster_centers_trajectories,
            output_directory=self.output_directory,
            output_prefix=self.output_prefix,
            clustering_method=clustering_method,
            )
        )
        
    def hdbscan(
        self,
        automatic=False,
        min_cluster_size_range=(20, 80, 5),
        min_samples_range=(20, 100, 5),
        max_cluster_population=0.95,
        max_clusters=20,
        min_cluster_size=5,
        min_samples=None,
        **kwargs):
        """
        This function implements the HDBSCAN method to do the clustering.

        automatic: bool, default=False
            If True, the clustering is run using various combinations of min_cluster_size and
            min_samples values. The silhouette score is determined for each combination and
            the combination with the highest silhouette score is selected. Combinations 
            that produce more than max_clusters or only one cluster are excluded.
        
        max_cluster_population: float, default=0.95
            Used when automatic=True. The maximum allowed population value for the most
            populated cluster. If the population of the most populated cluster exceeds 
            this value, the clustering is skipped.

        min_cluster_size_range: tuple, default=(20, 80, 5)
            The range of min_cluster_size screened in the automatic grid search. Interpreted as 
            (start, stop, step), values are generated with np.arange(start, stop, step).

        min_samples_range: tuple, default=(20, 100, 5)
            The range of min_samples value screened in the automatic grid search. Interpreted as 
            (start, stop, step), values are generated with np.arange(start, stop, step).

        max_clusters: int, default=2
            Maximum number of clusters allowed when automatic=True.

        kwargs:
            Other arguments allowed in HDBSCAN can be provided.
            (https://scikit-learn.org/stable/modules/generated/sklearn.cluster.HDBSCAN.html)
        """

        clustering_method = "HDBSCAN"

        if automatic == False:
            if max_cluster_population == 0.95 and min_cluster_size == 5 and not min_samples:
                print("Warning: 'automatic' is set to False and the default values for 'max_cluster_population' (0.95), 'min_cluster_size' (5), and 'min_samples' (None) will be used. Consider setting these parameters explicitly for better results.") 
            (
                labels,
                cluster_centers,
                cluster_center_indices,
                cluster_centers_trajectories,
            ) = hdbscan_clustering(
                min_cluster_size=min_cluster_size,
                min_samples=min_samples,
                clustering_matrix=self.clustering_matrix,
                frames=self.frames,
                find_centroid=True,
                trajectory_names=self.trajectory_names,
                **kwargs
            )
            sil_score = round(silhouette_score(self.clustering_matrix, labels), 4)
            print(
                f"Clustering with parameters min_cluster_size={min_cluster_size} and min_samples={min_samples} produces {len(set(labels))} clusters with an average silhouette score of {sil_score}."
            )
            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": [len(set(labels))],
                    "Minimum cluster size": [min_cluster_size],
                    "Minimum samples": [min_samples],
                    "Average silhouette score": [sil_score],
                }
            )

        elif automatic == True:
            sil_scores = []
            params = []
            cluster_nums = []
            for min_cluster_size in np.arange(min_cluster_size_range[0],
                                              min_cluster_size_range[1],
                                              min_cluster_size_range[2]):
                for min_samples in np.arange(
                    min_samples_range[0], 
                    min_samples_range[1], 
                    min_samples_range[2]
                ):
                    labels = hdbscan_clustering(
                        min_cluster_size=min_cluster_size,
                        min_samples=min_samples,
                        clustering_matrix=self.clustering_matrix,
                        frames=self.frames,
                        find_centroid=False,
                        trajectory_names=self.trajectory_names,
                    )

                    if len(set(labels)) == len(labels):
                        print(
                            f"Clustering with parameters min_cluster_size={min_cluster_size} and min_samples={min_samples} produces no clusters. Moving on to next set of parameters"
                        )

                    elif len(set(labels)) <= 2:
                        print(
                            f"Clustering with parameters min_cluster_size={min_cluster_size} and min_samples={min_samples} produces only one cluster. Moving on to next set of parameters"
                        )

                    elif len(set(labels)) > max_clusters:
                        print(
                            f"Clustering with parameters min_cluster_size={min_cluster_size} and min_samples={min_samples} produces more than {max_clusters} clusters. Moving on to next set of parameters"
                        )

                    elif (
                        determine_cluster_population_ratio(
                            labels=labels,
                            max_cluster_population=max_cluster_population,
                            clustering_method=clustering_method,
                        )
                        == True
                    ):
                        print(
                            f"Clustering with parameters min_cluster_size={min_cluster_size} and min_samples={min_samples} results in the most populated cluster containing more than {max_cluster_population*100}% of frames. Moving on to next st of parameters."
                        )

                    else:
                        try:
                            sil_score = round(
                                silhouette_score(self.clustering_matrix, labels), 4
                            )
                            sil_scores.append(sil_score)
                            cluster_nums.append(len(set(labels)))
                            params.append([min_cluster_size, min_samples])
                            print(
                                f"Clustering with parameters min_cluster_size={min_cluster_size} and min_samples={min_samples} produces {len(set(labels))} clusters with an average silhouette score of {sil_score}."
                            )
                        except Exception as e:
                            print(f'Error found when evaluating silhouette score with min_cluster_size={min_cluster_size} and min_samples={min_samples} so skipping this: {e}.')
                            continue

            sil_scores = np.array(sil_scores)
            best_params_index = np.argmax(sil_scores)
            best_min_cluster_size, best_min_samples = tuple(params[best_params_index])
            sil_score = sil_scores[best_params_index]
            print(
                f"Best clustering parameters identified at min_cluster_size={min_cluster_size} and min_samples={best_min_samples} with an average silhouette score of {sil_score}."
            )
            (
                labels,
                cluster_centers,
                cluster_center_indices,
                cluster_centers_trajectories,
            ) = hdbscan_clustering(
                min_cluster_size=best_min_cluster_size,
                min_samples=best_min_samples,
                clustering_matrix=self.clustering_matrix,
                frames=self.frames,
                find_centroid=True,
                trajectory_names=self.trajectory_names,
                **kwargs
            )
            params = np.array(params)

            silhouette_scores = pd.DataFrame(
                {
                    "Number of clusters": cluster_nums,
                    "Minimum cluster size": params[:, 0],
                    "Minimum samples": params[:, 1],
                    "Average silhouette score": sil_scores,
                }
            )
        
        summary = generate_clustering_summary(
            trajectory_file=self.trajectory_file,
            topology_file=self.topology_file,
            pka_file=self.pka_file,
            selections=self.selections,
            include_distances=self.include_distances,
            include_buriedness=self.include_buriedness,
            clustering_method=clustering_method,
            automatic=automatic,
            silhouette_scores=silhouette_scores,
            n_components=self.n_components,
            cummulative_variance=self.cummulative_variance,
            buriedness_file=self.buriedness_file,
        )
        self._labels, self._cluster_centers, self._cluster_center_indices, self._cluster_centers_trajectories  = (
            write_clustering_info(
            summary=summary,
            trajectory_dict=self.trajectory_dict,
            pka_df=self.pka_df,
            times=self.times,
            frames=self.frames,
            trajectory_names=self.trajectory_names,
            labels=labels,
            cluster_centers=cluster_centers,
            cluster_center_indices=cluster_center_indices,
            cluster_centers_trajectories=cluster_centers_trajectories,
            output_directory=self.output_directory,
            output_prefix=self.output_prefix,
            clustering_method=clustering_method,
            )
        )