#!/usr/bin/env python
# -*- coding: utf-8; -*-

# Copyright (c) 2021 Oracle and/or its affiliates.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/


from __future__ import print_function, absolute_import

import json
import re
import copy
from collections import defaultdict
from time import time

import numpy as np
import pandas as pd
import scipy as sp
from six import string_types
from sklearn.utils import Bunch
from sklearn.feature_extraction.text import CountVectorizer
from wordcloud import STOPWORDS

from ads.common import utils
from ads.common.card_identifier import card_identify
from ads.common.utils import JsonConverter

class TypedFeature(Bunch):
    def __init__(self, name, meta_data):
        assert "type" in meta_data
        self.meta_data = meta_data
        self.meta_data["feature_name"] = name
        super().__init__(**meta_data)

    @staticmethod
    def build(name, series):
        pass

    def __repr__(self):
        d = copy.deepcopy(self.meta_data)
        if "internal" in d:
            d.pop("internal")
        return json.dumps(d, indent=2, cls=JsonConverter)


class ContinuousTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        series = series.astype("float")

        non_null_series = series.loc[~series.isnull()]
        desc = non_null_series.describe()
        stats = {
            "mode": series.mode().iloc[0],
            "median": desc["50%"],
            "kurtosis": series.kurt(),
            "variance": series.var(),
            "skewness": series.skew(),
            "outlier_percentage": 100
            * np.count_nonzero(np.abs(sp.stats.zscore(non_null_series) >= 2.999))
            / series.size,
        }
        stats.update({k: v for k, v in desc.items()})

        return ContinuousTypedFeature(
            name,
            {
                "type": "continuous",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": stats,
                "internal": {"sample": series.head(5)},
            },
        )


class DiscreteTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)


class OrdinalTypedFeature(DiscreteTypedFeature):
    def __init__(self, name, meta_data):
        DiscreteTypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        nulls_removed = series.astype("category").loc[~series.isnull()]
        desc = nulls_removed.describe(include="all")
        value_counts = series.value_counts(ascending=False)
        x_min, x_max = np.nanmin(nulls_removed), np.nanmax(nulls_removed)

        stats = {
            "unique percentage": 100 * desc["unique"] / desc["count"],
            "x_min": x_min,
            "x_max": x_max,
            "mode": series.mode().iloc[0],
        }
        stats.update({k: v for k, v in desc.items()})

        return OrdinalTypedFeature(
            name,
            {
                "type": "ordinal",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": stats,
                "internal": {
                    "sample": series.head(5),
                    "unique": desc["unique"],
                    "counts": utils.truncate_series_top_n(
                        value_counts, n=min(16, len(value_counts))
                    ),
                    "high_cardinality": bool(desc["unique"] > 30),
                    "very_high_cardinality": bool(
                        desc["unique"] >= 0.95 * desc["count"]
                    ),
                },
            },
        )


class CategoricalTypedFeature(DiscreteTypedFeature):
    def __init__(self, name, meta_data):
        DiscreteTypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        desc = series.astype("category").loc[~series.isnull()].describe(include="all")
        value_counts = series.value_counts(ascending=False)
        if isinstance(desc["top"], string_types):
            mode = desc["top"] if len(desc["top"]) < 30 else desc["top"][:24] + "..."
        else:
            mode = desc["top"]

        stats = {
            "unique percentage": 100 * desc["unique"] / desc["count"],
            "mode": mode,
        }
        stats.update({k: v for k, v in desc.items()})

        return CategoricalTypedFeature(
            name,
            {
                "type": "categorical",
                "low_level_type": series.dtype.name,
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "stats": stats,
                "internal": {
                    "sample": series.sample(n=min(100, series.size)),
                    "unique": desc["unique"],
                    "counts": utils.truncate_series_top_n(
                        value_counts, n=min(16, len(value_counts))
                    ),
                    "high_cardinality": bool(desc["unique"] > 30),
                    "very_high_cardinality": bool(
                        desc["unique"] >= 0.95 * desc["count"]
                    ),
                },
            },
        )


class IPAddressTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        desc = series.astype("category").loc[~series.isnull()].describe()

        nulls_removed = series.loc[~series.isnull()]
        value_counts = nulls_removed.value_counts()

        return IPAddressTypedFeature(
            name,
            {
                "type": "ipaddress",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": {
                    "unique percentage": 100 * desc["unique"] / desc["count"],
                    "mode": series.mode().iloc[0],
                },
                "internal": {
                    "sample": nulls_removed.sample(n=min(100, nulls_removed.size)),
                    "counts": utils.truncate_series_top_n(
                        value_counts, n=min(16, len(value_counts))
                    ),
                    "unique": desc["unique"],
                },
            },
        )


class PhoneNumberTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        desc = series.astype("category").loc[~series.isnull()].describe()

        nulls_removed = list(series.loc[~series.isnull()])
        pat = re.compile(r"^(\+?\d{1,2}[\s-])?\(?(\d{3})\)?[\s.-]?\d{3}[\s.-]?\d{4}$")
        area_codes = [
            g.group(2)
            for g in filter(None, [pat.match(x) for x in nulls_removed])
            if g and len(g.groups()) >= 2
        ]
        value_counts = pd.Series(area_codes).value_counts()

        # resolve area codes
        return PhoneNumberTypedFeature(
            name,
            {
                "type": "phone",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": {
                    "unique percentage": 100 * desc["unique"] / desc["count"],
                    "mode": series.mode().iloc[0],
                },
                "internal": {
                    "sample": series.sample(n=min(100, series.size)),
                    "counts": utils.truncate_series_top_n(
                        value_counts, n=min(16, len(value_counts))
                    ),
                    "unique": desc["unique"],
                },
            },
        )


class GISTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series, samples):
        desc = series.astype("category").loc[~series.isnull()].describe()
        value_counts = series.value_counts(ascending=False)
        return GISTypedFeature(
            name,
            {
                "type": "gis",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": {
                    "observations": desc["count"],
                    "unique percentage": 100 * desc["unique"] / desc["count"]
                    # TODO mid point
                },
                "internal": {"sample": samples, "unique": desc["unique"]},
            },
        )


class AddressTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        nulls_removed = series.loc[~series.isnull()]
        mean_document_length = nulls_removed.str.len().mean()
        sampled_nulls_removed = nulls_removed.sample(n=min(len(nulls_removed), 1000))

        return AddressTypedFeature(
            name,
            {
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "type": "address",
                "low_level_type": series.dtype.name,
                "stats": {
                    "mean_character_count": mean_document_length,
                    "mean_word_length": nulls_removed.str.split().str.len().mean(),
                },
                "internal": {
                    "sample": series.sample(n=min(100, series.size)),
                    "word_frequencies": DocumentTypedFeature.vectorization(
                        name, sampled_nulls_removed, mean_document_length
                    ),
                },
            },
        )


class DocumentTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def corpus_processor(series):
        pat_punct = re.compile(r"[^a-zA-Z]", re.UNICODE + re.MULTILINE)
        pat_tags = re.compile(r"<.*?>", re.UNICODE + re.MULTILINE)
        pat_digits = re.compile(r"(\d|\W)+", re.UNICODE + re.MULTILINE)
        pat_splitter = re.compile(r"\s+", re.UNICODE + re.MULTILINE)
        for doc in series:
            # Remove punctuations, tags,  special characters and digits
            text = pat_punct.sub(" ", doc.lower())

            # remove tags
            text = pat_tags.sub("", text)

            # remove special characters and digits
            text = pat_digits.sub(" ", text)

            # Convert to list from string and return a generator expression
            yield " ".join(
                [
                    x
                    for x in pat_splitter.split(text)
                    if len(x) >= 3 and x not in STOPWORDS
                ]
            )

    @staticmethod
    def sub_vectorization(
        feature_name,
        series,
        min_df=0.0,
        max_df=1.0,
        min_tf=2,
    ):
        start = time()

        v1 = CountVectorizer(
            ngram_range=(1, 1),
            max_features=2000,
            decode_error="ignore",
            strip_accents="unicode",
            stop_words=None,
            min_df=min_df,
            max_df=max_df,
        )
        X1 = v1.fit_transform(DocumentTypedFeature.corpus_processor(series))
        unigrams = {
            k: int(v)
            for k, v in dict(
                zip(v1.get_feature_names(), np.asarray(X1.sum(axis=0)).ravel())
            ).items()
        }

        v2 = CountVectorizer(
            ngram_range=(2, 2),
            max_features=2000,
            decode_error="ignore",
            strip_accents="unicode",
            stop_words=None,
            min_df=min_df,
            max_df=max_df,
        )
        X2 = v2.fit_transform(DocumentTypedFeature.corpus_processor(series))
        bigrams = {
            k: int(v)
            for k, v in dict(
                zip(v2.get_feature_names(), np.asarray(X2.sum(axis=0)).ravel())
            ).items()
        }

        # drop unigrams that are prefixes of bigrams
        for k, v in bigrams.items():
            # take the key (k), split it to get first word and look for that word in unigrams
            for word in k.split():
                if word in unigrams:
                    # boost the bigram
                    bigrams[k] = v + unigrams[word]
                    del unigrams[word]

        unigrams.update(bigrams)

        ret = {k: v for k, v in unigrams.items() if v >= min_tf}
        return ret if ret else unigrams.items()

    @staticmethod
    def vectorization(feature_name, series, mean_document_length):
        # set the min_df and max_df as functions of the mean_document_length
        min_df = 0.1
        max_df = 0.8

        if mean_document_length > 5000:
            min_df = 0.2
            max_df = 0.7
        try:
            return DocumentTypedFeature.sub_vectorization(
                feature_name, series, min_df=min_df, max_df=max_df
            )
        except ValueError:
            return DocumentTypedFeature.sub_vectorization(
                feature_name, series, min_df=0.0, max_df=1.0
            )

    @staticmethod
    def build(name, series, is_cjk, is_html):

        internal = {"cjk": is_cjk, "html": is_html}

        if is_cjk:
            stats = {"mean_document_character_count": int(series.str.len().mean())}

            internal["sample"] = series.sample(n=min(100, series.size))

        else:
            nulls_removed = series.loc[~series.isnull()]
            if is_html:
                nulls_removed = nulls_removed.replace(
                    to_replace="<[^<]+?>", value=" ", regex=True
                ).replace("\n", " ", regex=True)

            internal["sample"] = nulls_removed.sample(n=min(100, nulls_removed.size))

            mean_document_length = nulls_removed.str.len().mean()
            sample_size = int(
                2e6 // mean_document_length
            )  # vectorize at most 2MB of text

            sampled_nulls_removed = nulls_removed.sample(
                n=min(len(nulls_removed), sample_size)
            )

            stats = {
                "mean_document_character_count": int(series.str.len().mean()),
                "mean_word_length": nulls_removed.str.split().str.len().mean(),
            }

            internal["word_frequencies"] = DocumentTypedFeature.vectorization(
                name, sampled_nulls_removed, mean_document_length
            )

        return DocumentTypedFeature(
            name,
            {
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "type": "document",
                "low_level_type": series.dtype.name,
                "stats": stats,
                "internal": internal,
            },
        )


class ZipcodeTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        max_threshold = 24
        suffix = ""

        series = series.loc[~series.isnull()].astype(str).str.slice(0, 5)

        for i in [5, 4, 3, 2, 1]:
            if series.nunique() <= max_threshold:
                break
            series = series.str[:-1]
            suffix += "*"

        series += suffix

        return ZipcodeTypedFeature(
            name,
            {
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "type": "zipcode",
                "low_level_type": series.dtype.name,
                "stats": {"mode": series.mode().iloc[0]},
                "internal": {
                    "sample": series.sample(n=min(100, series.size)),
                    "histogram": series.value_counts(),
                },
            },
        )


class UnknownTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        return UnknownTypedFeature(
            name,
            {
                "type": "unknown",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": {"mode": series.mode().iloc[0]},
                "internal": {"sample": series.sample(n=min(100, series.size))},
            },
        )


class ConstantTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        desc = series.astype("category").loc[~series.isnull()].describe()
        if "top" in desc:
            value = desc["top"]
            value_counts = utils.truncate_series_top_n(series.value_counts())
        else:
            value = "NaN"
            value_counts = {value: len(series)}
        return ConstantTypedFeature(
            name,
            {
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "type": "constant",
                "stats": {"value": value},
                "internal": {
                    "sample": series.sample(n=min(100, series.size)),
                    "counts": value_counts,
                },
            },
        )


class CreditCardTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        null_removed_series = series.loc[~series.isnull()]
        desc = null_removed_series.astype("category").describe()
        sampled_series = null_removed_series.sample(
            n=min(1000, len(null_removed_series))
        )

        d_scheme = defaultdict(int)

        # use counting method to build dict of credit card data
        for s in sampled_series:
            scheme = card_identify().identify_issue_network(s)
            d_scheme[scheme] += 1

        return CreditCardTypedFeature(
            name,
            {
                "type": "creditcard",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": {
                    "unique percentage": 100 * desc["unique"] / desc["count"],
                    "mode": desc["top"],
                },
                "internal": {
                    "sample": series.sample(n=min(100, series.size)),
                    "counts": dict(d_scheme),
                    "unique": desc["unique"],
                },
            },
        )


class DateTimeTypedFeature(TypedFeature):
    def __init__(self, name, meta_data):
        TypedFeature.__init__(self, name, meta_data)

    @staticmethod
    def build(name, series):
        desc = series.loc[~series.isnull()].describe()

        return DateTimeTypedFeature(
            name,
            {
                "type": "datetime",
                "missing_percentage": 100 * series.isna().sum() / series.size,
                "low_level_type": series.dtype.name,
                "stats": {
                    "unique percentage": 100 * desc["unique"] / desc["count"],
                    "first": desc["first"],
                    "last": desc["last"],
                    "mode": desc["top"],
                },
                "internal": {
                    "sample": series.sample(n=min(100, series.size)),
                    "unique": desc["unique"],
                },
            },
        )
