# Copyright (C) 2019 Maxim Godzi, Anatoly Zaytsev, Dmitrii Kiselev
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.


import pandas as pd
import numpy as np
from collections import Counter
from sklearn.manifold import TSNE
from sklearn import decomposition
from sklearn import manifold


def _uni_counts_embedder(data, **kwargs):
    if 'index_col' not in kwargs:
        index_col = data.trajectory.retention_config['index_col']
    else:
        index_col = kwargs['index_col']
    if 'event_col' not in kwargs:
        event_col = data.trajectory.retention_config['event_col']
    else:
        event_col = kwargs['event_col']
    last_k = kwargs.get('last_k')
    if last_k is not None:
        data = data.groupby(index_col).tail(last_k)
    cv = data.groupby([index_col, event_col]).size().rename('event_count').reset_index()
    cv = cv.pivot(index=index_col, columns=event_col).fillna(0)
    cv.columns = cv.columns.levels[1]
    cv.columns.set_names(None, inplace=True)
    cv.index.set_names(None, inplace=True)
    setattr(cv.retention, 'datatype', 'features')
    return cv


def _ngram_agg(x, ngram_range):
    res = []
    shifts = []
    for i in range(ngram_range[0] - 1, ngram_range[1]):
        shifts.append(x.shift(i))
        res.extend(zip(*shifts))
    return Counter(res)


def counts_embedder(data, ngram_range=(1, 1), **kwargs):
    """
    Calculate ``index_col`` embedding (continuous vector form) by counting ``event_col`` appearance for each ``index_col``.

    Parameters
    --------
    data: pd.DataFrame
        Clickstream dataset.
    ngram_range: tuple, optional
        Range of ngrams to use in feature extraction. Default: ``(1, 1)``
    index_col: str, optional
        Name of custom index column, for more information refer to ``init_config``. For instance, if in config you have defined ``index_col`` as ``user_id``, but want to use function over sessions. By default the column defined in ``init_config`` will be used as ``index_col``.
    event_col: str, optional
        Name of custom event column, for more information refer to ``init_config``. For instance, you may want to aggregate some events or rename and use it as new event column. By default the column defined in ``init_config`` will be used as ``event_col``.
    last_k: int, optional
        Include only the last ``last_k`` events for each ``index_col``.
    wo_last: int, optional
        Exclude ``last_k`` events for each ``index_col``.

    Returns
    --------
    Vectorized dataframe with ``index_col`` as index and counts of ``event_col`` values as dataframe values.

    Return type
    -------
    pd.DataFrame
    """
    if max(ngram_range) == 1:
        return _uni_counts_embedder(data, **kwargs)
    if 'index_col' not in kwargs:
        index_col = data.trajectory.retention_config['index_col']
    else:
        index_col = kwargs['index_col']
    if 'event_col' not in kwargs:
        event_col = data.trajectory.retention_config['event_col']
    else:
        event_col = kwargs['event_col']
    last_k = kwargs.get('last_k')
    if last_k is not None:
        data = data.groupby(index_col).tail(last_k)
    wo_last = kwargs.get('wo_last_k')
    if wo_last is not None:
        bad_ids = data.groupby(index_col).tail(wo_last).index.values
        data = data[~data.index.isin(bad_ids)]
    cv = data.groupby(index_col)[event_col].apply(_ngram_agg, ngram_range=ngram_range).reset_index()
    cv = cv.pivot(index=index_col, columns='level_1', values=event_col).fillna(0)
    cv = cv.loc[:, [i for i in cv.columns if i[-1] == i[-1]]]
    cv.columns.set_names(None, inplace=True)
    cv.index.set_names(None, inplace=True)
    return cv


def frequency_embedder(data, ngram_range=(1, 1), **kwargs):
    """
    Similar to ``count_embedder()``, but normalizes events count over ``index_col``.

    Parameters
    -------
    data: pd.DataFrame
        Clickstream dataset.
    ngram_range: tuple, optional
        Range of ngrams to use in feature extraction. Default: ``(1, 1)``
    index_col: str, optional
        Name of custom index column, for more information refer to ``init_config``. For instance, if in config you have defined ``index_col`` as ``user_id``, but want to use function over sessions. By default the column defined in ``init_config`` will be used as ``index_col``.
    event_col: str, optional
        Name of custom event column, for more information refer to ``init_config``. For instance, you may want to aggregate some events or rename and use it as new event column. By default the column defined in ``init_config`` will be used as ``event_col``.

    Returns
    -------
    Dataframe with sessions vectorized by frequencies of events.

    Return type
    -------
    pd.DataFrame
    """
    cv = counts_embedder(data, ngram_range, **kwargs)
    freq = pd.DataFrame(
        cv.values / cv.values.sum(1).reshape(-1, 1),
        index=cv.index.values,
        columns=cv.columns.values,
    )
    setattr(freq.retention, 'datatype', 'features')
    return freq


def tfidf_embedder(data, ngram_range=(1, 1), **kwargs):
    """
    Similar to ``frequency_embedder()``, but normalizes event frequencies with inversed document frequency.

    Parameters
    --------
    data: pd.DataFrame
        Clickstream dataset.
    ngram_range: tuple, optional
        Range of ngrams to use in feature extraction. Default: ``(1, 1)``
    index_col: str, optional
        Name of custom index column, for more information refer to ``init_config``. For instance, if in config you have defined ``index_col`` as ``user_id``, but want to use function over sessions. By default the column defined in ``init_config`` will be used as ``index_col``.
    event_col: str, optional
        Name of custom event column, for more information refer to ``init_config``. For instance, you may want to aggregate some events or rename and use it as new event column. By default the column defined in ``init_config`` will be used as ``event_col``.

    Returns
    --------
    Dataframe with ``index_col`` vectorized by TF-IDF of events.

    Return type
    -------
    pd.DataFrame
    """
    tf = frequency_embedder(data, ngram_range, **kwargs)
    idf = np.log((tf.shape[0]) / ((tf > 0).sum(0) + 1e-20)).values
    tfidf = tf * idf
    setattr(tfidf.retention, 'datatype', 'features')
    return tfidf


def learn_tsne(data, **kwargs):
    """
    Calculates TSNE transformation for given matrix features.

    Parameters
    --------
    data: np.array
        Array of features.
    kwargs: optional
        Parameters for ``sklearn.manifold.TSNE()``

    Returns
    -------
    Calculated TSNE transform

    Return type
    -------
    np.ndarray
    """
    _tsne_filter = TSNE.get_params(TSNE)
    kwargs = {i: j for i, j in kwargs.items() if i in _tsne_filter}
    res = TSNE(random_state=0, **kwargs).fit_transform(data.values)
    return pd.DataFrame(res, index=data.index.values)


def get_manifold(data, manifold_type, **kwargs):
    """
    Reduces number of dimensions.

    Parameters
    ---------
    data: pd.DataFrame
        Dataframe with features for clustering indexed as in ``retention_config.index_col``
    manifold_type: str
        Name dimensionality reduction method from ``sklearn.decomposition`` and ``sklearn.manifold``
    kwargs: optional
        Parameters for ``sklearn.decomposition`` and ``sklearn.manifold`` methods.

    Returns
    --------
    pd.DataFrame with reduced dimensions.

    Return type
    --------
    pd.DataFrame
    """
    if hasattr(decomposition, manifold_type):
        man = getattr(decomposition, manifold_type)
    elif hasattr(manifold, manifold_type):
        man = getattr(manifold, manifold_type)
    else:
        raise ValueError(f'There is not such manifold {manifold_type}')
    tsvd = man(**{i: j for i, j in kwargs.items() if i in man.get_params(man)})
    res = tsvd.fit_transform(data)
    return pd.DataFrame(res, index=data.index)


def merge_features(features, metadata, meta_index_col=None, manifold_type=None, fillna=None, drop=False, **kwargs):
    """
    Adds metadata to TFIDF of trajectories. Eeduced if ``manifold_type`` is not ``None``.

    Parameters
    --------
    features: pd.DataFrame
        Dataframe with users` metadata.
    metadata: pd.DataFrame
        Dataframe with user or session properties or any other information you would like to extract as features (e.g. user properties, LTV values, etc.). Default: ``None``
    meta_index_col: str, optional
        Used when metadata is not ``None``. Name of column in ``metadata`` dataframe that contains the same ID as in ``index_col``, or if not defined, same as in retention_config (e.g ID of users or sessions). If ``None``, then index of metadata dataframe is used instead. Default: ``None``
    manifold_type: str, optional
        Name dimensionality reduction method from ``sklearn.decomposition`` and ``sklearn.manifold``. Default: ``None``
    fillna: optional
        Value for filling missing metadata for any ``index_col`` value. Default: ``None``
    drop: bool, optional
        If ``True``, then drops users which do not exist in ``metadata`` dataframe. Default: ``False``
    kwargs: optional
        Keyword arguments for ``sklearn.decomposition`` and ``sklearn.manifold`` methods.

    Returns
    -------
    Dataframe with trajectory features (possibly reduced) and users metadata.

    Return type
    -------
    pd.DataFrame
    """
    if manifold_type is not None:
        features = get_manifold(features, manifold_type, **kwargs)
    if meta_index_col is not None:
        metadata.index = metadata[meta_index_col].values
        metadata = metadata.drop(meta_index_col, 1)
    res = features.join(metadata, rsuffix='_meta',)
    if drop and (fillna is None):
        res = res[res.isnull().sum(1) == 0].copy()
    if fillna is not None:
        res = res.fillna(fillna)
    return res


def drop_equal_features(features, users, thres=0.1, **kwargs):
    """
    Drop nonzero features with equal counts of negative and positive user's

    :param features: array of features
    :param users: list of positive users (e.g. data.retention.get_positive_users())
    :param thres: threshold of dropping (e.g 0.1 means area of drop equals [0.9,1.1]
    :return: dropped features: array of features
    """
    feat_group = features.groupby(features.index.isin(users)).agg(lambda x: (x > 0).sum())
    feat_neg = feat_group.iloc[0]
    feat_pos = feat_group.iloc[1]
    feat_neg += 10**-6
    feat_div = feat_pos / feat_neg - 1.

    features_drop = features.drop(features.columns[feat_div.abs() < thres], axis=1)
    return features_drop
