from datetime import datetime, date
import logging
import bson
import orjson
import csv
import zipfile
from qddate import DateParser

DEFAULT_DICT_SHARE = 10
SUPPORTED_FILE_TYPES = [
    "xls",
    "xlsx",
    "csv",
    "xml",
    "json",
    "jsonl",
    "yaml",
    "tsv",
    "sql",
    "bson",
]

DEFAULT_EMPTY_VALUES = [None, "", "None", "NaN", "-"]


DEFAULT_OPTIONS = {
    "encoding": "utf8",
    "delimiter": ",",
    "limit": 1000,
    "empty": DEFAULT_EMPTY_VALUES,
}


def get_file_type(filename):
    ext = filename.rsplit(".", 1)[-1].lower()
    if ext in SUPPORTED_FILE_TYPES:
        return ext
    return None


def get_option(options, name):
    """Returns value of the option"""
    if name in options.keys():
        return options[name]
    elif name in DEFAULT_OPTIONS.keys():
        return DEFAULT_OPTIONS[name]
    return None


def guess_int_size(i):
    if i < 255:
        return "uint8"
    if i < 65535:
        return "uint16"
    return "uint32"


def guess_datatype(s, qd):
    """Guesses type of data by string provided"""
    attrs = {"base": "str"}
    #    s = unicode(s)
    if s is None:
        return {"base": "empty"}
    elif isinstance(s, bool):
        return {"base": "bool"}
    elif isinstance(s, int):
        return {"base": "int"}
    elif isinstance(s, float):
        return {"base": "float"}
    elif isinstance(s, datetime):
        return {"base": "datetime"}
    elif isinstance(s, date):
        return {"base": "date"}
    elif type(s) != type(""):
        #        print((type(s)))
        return {"base": "typed"}
    #    s = s.decode('utf8', 'ignore')
    if s.isdigit():
        if s[0] == 0:
            attrs = {"base": "numstr"}
        else:
            attrs = {"base": "int", "subtype": guess_int_size(int(s))}
    else:
        try:
            i = float(s)
            attrs = {"base": "float"}
            return attrs
        except ValueError:
            pass
        if qd:
            is_date = False
            res = qd.match(s)
            if res:
                attrs = {"base": "date", "pat": res["pattern"]}
                is_date = True
            if not is_date:
                if len(s.strip()) == 0:
                    attrs = {"base": "empty"}
    return attrs


def dict_generator(indict, pre=None):
    pre = pre[:] if pre else []
    if isinstance(indict, dict):
        for key, value in list(indict.items()):
            if key == "_id":
                continue
            if isinstance(value, dict):
                #                print 'dgen', value, key, pre
                for d in dict_generator(value, pre + [key]):
                    yield d
            elif isinstance(value, list) or isinstance(value, tuple):
                for v in value:
                    if isinstance(v, dict):
                        #                print 'dgen', value, key, pre
                        for d in dict_generator(v, pre + [key]):
                            yield d
            #                    for d in dict_generator(v, [key] + pre):
            #                        yield d
            else:
                yield pre + [key, value]
    else:
        yield indict


class Analyzer:
    def __init__(self, nodates=True):
        if nodates:
            self.qd = None
        else:
            self.qd = DateParser(generate=True)
        pass

    def analyze(self, fromfile=None, itemlist=None, options=DEFAULT_OPTIONS):
        """Analyzes JSON file and produces stats"""
        if fromfile == None and itemlist == None:
            return None

        if "empty" not in options.keys():
            options["empty"] = DEFAULT_EMPTY_VALUES

        dictshare = get_option(options, "dictshare")
        if dictshare and dictshare.isdigit():
            dictshare = int(dictshare)
        else:
            dictshare = DEFAULT_DICT_SHARE

        profile = {"version": 1.0}
        fielddata = {}
        fieldtypes = {}

        #    data = json.load(open(profile['filename']))
        count = 0
        nfields = 0

        if fromfile:
            f_type = (
                get_file_type(fromfile)
                if options["format_in"] is None
                else options["format_in"]
            )
            if f_type not in ["jsonl", "bson", "csv"]:
                print("Only JSON lines (.jsonl), .csv and .bson files supported now")
                return

            if options["zipfile"]:
                z = zipfile.ZipFile(fromfile, mode="r")
                fnames = z.namelist()
                finfilename = fnames[0]
                if f_type == "bson":
                    infile = z.open(fnames[0], "rb")
                else:
                    infile = z.open(fnames[0], "r")
            else:
                finfilename = fromfile
                if f_type == "bson":
                    infile = open(fromfile, "rb")
                else:
                    infile = open(
                        fromfile, "r", encoding=get_option(options, "encoding")
                    )

            # Identify item list
            itemlist = []

            if f_type == "jsonl":
                for l in infile:
                    itemlist.append(orjson.loads(l))
            elif f_type == "csv":
                delimiter = get_option(options, "delimiter")
                reader = csv.DictReader(infile, delimiter=delimiter)
                for r in reader:
                    itemlist.append(r)
            elif f_type == "bson":
                bson_iter = bson.decode_file_iter(infile)
                for r in bson_iter:
                    itemlist.append(r)

        # process data items one by one
        logging.debug("Start processing %s" % (fromfile))
        for item in itemlist:
            count += 1
            dk = dict_generator(item)
            if count % 1000 == 0:
                logging.debug("Processing %d records of %s" % (count, fromfile))
            for i in dk:
                #            print(i)
                k = ".".join(i[:-1])
                if len(i) == 0:
                    continue
                if i[0].isdigit():
                    continue
                if len(i[0]) == 1:
                    continue
                v = i[-1]
                if k not in list(fielddata.keys()):
                    fielddata[k] = {
                        "key": k,
                        "uniq": {},
                        "n_uniq": 0,
                        "total": 0,
                        "share_uniq": 0.0,
                        "minlen": None,
                        "maxlen": 0,
                        "avglen": 0,
                        "totallen": 0,
                    }
                fd = fielddata[k]
                uniqval = fd["uniq"].get(v, 0)
                fd["uniq"][v] = uniqval + 1
                fd["total"] += 1
                if uniqval == 0:
                    fd["n_uniq"] += 1
                    fd["share_uniq"] = (fd["n_uniq"] * 100.0) / fd["total"]
                fl = len(str(v))
                if fd["minlen"] is None:
                    fd["minlen"] = fl
                else:
                    fd["minlen"] = fl if fl < fd["minlen"] else fd["minlen"]
                fd["maxlen"] = fl if fl > fd["maxlen"] else fd["maxlen"]
                fd["totallen"] += fl
                fielddata[k] = fd
                if k not in list(fieldtypes.keys()):
                    fieldtypes[k] = {"key": k, "types": {}}
                fd = fieldtypes[k]
                thetype = guess_datatype(v, self.qd)["base"]
                uniqval = fd["types"].get(thetype, 0)
                fd["types"][thetype] = uniqval + 1
                fieldtypes[k] = fd
        #        print count
        for k, v in list(fielddata.items()):
            fielddata[k]["share_uniq"] = (v["n_uniq"] * 100.0) / v["total"]
            fielddata[k]["avglen"] = v["totallen"] / v["total"]
        profile["count"] = count
        profile["num_fields"] = nfields
        finfields = {}
        for k, v in list(fielddata.items()):
            #            del v['uniq']
            fielddata[k] = v
        for fdk in list(fieldtypes.values()):
            fdt = list(fdk["types"].keys())
            if "empty" in fdt:
                del fdk["types"]["empty"]
            if len(list(fdk["types"].keys())) != 1:
                ftype = "str"
            else:
                ftype = list(fdk["types"].keys())[0]
            finfields[fdk["key"]] = ftype

        dictkeys = []
        dicts = {}
        #        print(profile)
        profile["fields"] = []
        for fd in list(fielddata.values()):
            #            print(fd)
            #            print(fd['key'])  # , fd['n_uniq'], fd['share_uniq'], fieldtypes[fd['key']]
            field = {"key": fd["key"], "is_uniq": 0 if fd["share_uniq"] < 100 else 1}
            profile["fields"].append(field)
            if fd["share_uniq"] < dictshare:
                dictkeys.append(fd["key"])
                dicts[fd["key"]] = {
                    "items": fd["uniq"],
                    "count": fd["n_uniq"],
                    "total": sum(fd["uniq"].values()),
                    "type": finfields[fd["key"]],
                }  # TODO: Shouldn't be "str" by default
        #            for k, v in fd['uniq'].items():
        #                print fd['key'], k, v
        profile["dictkeys"] = dictkeys

        profile["debug"] = {"fieldtypes": fieldtypes.copy(), "fielddata": fielddata}
        profile["fieldtypes"] = finfields
        table = []
        for fd in list(fielddata.values()):
            field = [
                fd["key"],
            ]
            field.append(finfields[fd["key"]])
            field.append(True if fd["key"] in dictkeys else False)
            field.append(False if fd["share_uniq"] < 100 else True)
            field.append(fd["n_uniq"])
            field.append(fd["share_uniq"])
            field.append(fd["minlen"])
            field.append(fd["maxlen"])
            field.append(fd["avglen"])
            tags = []
            if fd["share_uniq"] == 100:
                tags.append("uniq")
            allempty = 0
            if fd["key"] in dicts.keys():
                for key, value in dicts[fd["key"]]["items"].items():
                    if key in options["empty"]:
                        allempty += value
                if allempty == dicts[fd["key"]]["total"]:
                    tags.append("empty")
                else:
                    tags.append("dict")
            field.append(tags)
            table.append(field)
        return table

    def print(self, table):
        from tabulate import tabulate

        headers = [
            "key",
            "ftype",
            "is_dictkey",
            "is_uniq",
            "n_uniq",
            "share_uniq",
            "minlen",
            "maxlen",
            "avglen",
            "tags",
        ]
        print(tabulate(table, headers=headers))
