# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/6.2_desc.metrics.java.ipynb (unless otherwise specified).

__all__ = ['get_unicode', 'simulate_getting_dataframes_from_mongo', 'add_mccabe_metrics', 'create_parser_builds',
           'lang_keywords', 'find_class_nodes', 'find_method_nodes', 'find_field_names', 'find_string_in_text',
           'distinct_field_calls', 'calculate_lcom5', 'add_lcom5', 'flatten_lol', 'display_numeric_col_stats',
           'display_numeric_col_hist']

# Cell
# Imports
import pandas as pd
from numpy import mean, std
from statistics import median
from scipy.stats import sem, t
import lizard
import matplotlib.pyplot as plt
from tree_sitter import Language, Parser, Node
#Decoding files
import chardet
from bs4 import UnicodeDammit


# TODO: Remove when mongo call is implemented
import os

# Cell
def get_unicode(file_path):
    """Detects file encoding and returns unicode. Inspired by http://reinvantveer.github.io/2017/05/19/unicode-dammit.html"""
    with open(file_path, 'rb') as f:
        detection = chardet.detect(f.read())

    enc = detection["encoding"]
    if detection["encoding"] == "ascii":
        with open(file_path, encoding="ascii") as f:
            data = f.read()
    elif detection["encoding"] == "ISO-8859-9":
        with open(file_path, encoding="utf-8") as f:
            enc = "utf-8"
            data = f.read()
    else:
        try:
            # Try to open as non unicode file
            with open(file_path, encoding=detection["encoding"]) as f:
                data = f.read()
        except Exception as e:
            raise ValueError(f"Cannot return dictionary from empty or invalid csv file {file_path} due to {e}")

    if not data:
        raise ValueError(f"Cannot return dictionary from empty or invalid csv file {file_path}")

    return UnicodeDammit(data).unicode_markup, enc

# Cell
def simulate_getting_dataframes_from_mongo(folder_path):
    """Loads files from a specified folder into a pandas dataframe"""
    corpus_data = {"system": [], "name": [], "ground_truth": [], "contents": [], "encoding": []}
    for file in os.listdir(folder_path):
        if not os.path.isdir(os.path.join(folder_path, file)) and file != ".DS_Store":
            corpus_data["system"].append(None)
            corpus_data["name"].append(file)
            corpus_data["ground_truth"].append("src")
            contents, enc = get_unicode(os.path.join(folder_path, file))
            corpus_data['encoding'].append(enc)
            corpus_data['contents'].append(contents)
    corpus_df = pd.DataFrame(data = corpus_data)
    return corpus_df

# Cell
def add_mccabe_metrics(df, data_col, name_col):
    """Adds information about function length and cyclomatic complexity for classes to a dataframe"""
    num_funcs = []
    class_ccn = []
    avg_func_ccn = []
    avg_func_nloc = []
    for i in range(len(df)):
        file_num_funcs = []
        file_class_ccn = []
        file_avg_func_ccn = []
        file_avg_func_nloc = []
        metrics = lizard.analyze_file.analyze_source_code(df[name_col][i], df[data_col][i])
        class_dict = {}
        for func in metrics.function_list:
            class_name = '::'.join(func.name.split("::")[:-1])
            if class_name in class_dict:
                class_dict[class_name].append(func)
            else:
                class_dict[class_name] = [func]
        for class_key in class_dict:
            total_class_ccn = 0
            total_class_nloc = 0
            for func in class_dict[class_key]:
                total_class_ccn += func.cyclomatic_complexity
                total_class_nloc += func.length
            file_num_funcs.append(len(class_dict[class_key]))
            file_class_ccn.append(total_class_ccn)
            file_avg_func_ccn.append(total_class_ccn/len(class_dict[class_key]))
            file_avg_func_nloc.append(total_class_nloc/len(class_dict[class_key]))

        num_funcs.append(file_num_funcs)
        class_ccn.append(file_class_ccn)
        avg_func_ccn.append(file_avg_func_ccn)
        avg_func_nloc.append(file_avg_func_nloc)


    df["num_funcs"] = num_funcs
    df["class_ccn"] = class_ccn
    df["avg_func_ccn"] = avg_func_ccn
    df["avg_func_nloc"] = avg_func_nloc
    return df

# Cell
def create_parser_builds(path=None):
    """Creates a dictionary of tree-sitter parsers for select languages"""
    Language.build_library(
        # Store the library in the `build` directory
        'build/my-languages.so',

        # Include one or more languages
        [
            'tree-sitter-java'
        ]
    )

    JAVA_LANGUAGE = Language('build/my-languages.so', 'java')

    return {"java":JAVA_LANGUAGE}

# Cell
def lang_keywords():
    """Returns language specific keywords for a parser to find"""
    keyword_dict = {}
    keyword_dict["java"] = {"class": "class_declaration", "method":"method_declaration", "field_dec":"field_declaration", "field_name":"identifier"}
    return keyword_dict

# Cell
def find_class_nodes(root_node, key):
    """Recursively searches an AST for class nodes"""
    node_list = []
    def rec_class_search(node):
        if node.type == key["class"]:
            node_list.append(node)
        for child in node.children:
            rec_class_search(child)

    rec_class_search(root_node)
    return node_list

# Cell
def find_method_nodes(class_node, key):
    """Recursively searches an AST for method nodes"""
    node_list = []
    def rec_method_search(node):
        if node.type == key["method"]:
            node_list.append(node)
        if node.type != key["class"]:
            for child in node.children:
                rec_method_search(child)

    for node in class_node.children:
        rec_method_search(node)
    return node_list

# Cell
def find_field_names(class_node, file_bytes, key):
    """Finds the fields/attributes for a class AST"""
    class_fields = []

    def rec_name_search(node):
        if node.type == key["field_name"]:
            word = []
            for i in range(node.start_byte, node.end_byte):
                word.append(file_bytes[i])
            class_fields.append(word)
        else:
            for child in node.children:
                rec_name_search(child)

    def rec_field_search(node):
        if node.type == key["field_dec"]:
            rec_name_search(node)
        if node.type != key["class"]:
            for child in node.children:
                rec_field_search(child)

    for node in class_node.children:
        rec_field_search(node)

    return class_fields

# Cell
def find_string_in_text(node, pattern, file_bytes):
    """Counts the number of occurences of a byte-pattern array in a sample of code"""
    if len(node.children) > 0:
        count = 0
        for i in node.children:
            count += find_string_in_text(i, pattern, file_bytes)
        return count
    else:
        word = []
        for i in range(node.start_byte, node.end_byte):
            num_index_fails = 0
            try:
                word.append(file_bytes[i])
            except IndexError:
                num_index_fails += 1
        if(num_index_fails):
            print(f"INDEX ERROR ({num_index_fails} times)")
            print("Start byte:", node.start_byte, "End byte:", node.end_byte, "Word:", word)
        if word == pattern:
            return 1
        else:
            return 0

# Cell
def distinct_field_calls(class_node, field_names, file_bytes, key):
    """Recursively searches an AST for method nodes"""
    total_distinct_calls = []

    def rec_method_search(node):
        if node.type == key["method"]:
            distinct_method_field_calls = 0
            for field in field_names:
                if find_string_in_text(node, field, file_bytes):
                    distinct_method_field_calls += 1
            total_distinct_calls.append(distinct_method_field_calls)
        if node.type != key["class"]:
            for child in node.children:
                rec_method_search(child)

    for node in class_node.children:
        rec_method_search(node)

    return len(total_distinct_calls)

# Cell
def calculate_lcom5(tree, extension, file_bytes, name):
    """Parses the syntax tree of code to calculate the LCOM5 of its classes"""
    keyword_dict = lang_keywords()

    if extension not in keyword_dict:
        print(f"Tried to get LCOM5 of file with unsupported extension '.{extension}', 0 assigned to column.")
        return ["Undefined"]

    root_node = tree.root_node
    keywords = keyword_dict[extension]
    class_nodes = find_class_nodes(root_node, keywords)
    class_method_nodes = []
    class_field_names = []
    class_dfc = [] # Distinct field calls, as per the definition of LCOM5
    for node in enumerate(class_nodes):
        class_method_nodes.append(find_method_nodes(node[1], keywords))
        class_field_names.append(find_field_names(node[1], file_bytes, keywords))
        class_dfc.append(distinct_field_calls(node[1], class_field_names[node[0]], file_bytes, keywords))
    lcom5_list = []
    for j in range(len(class_nodes)):
        num_fields = len(class_field_names[j])
        num_meths = len(class_method_nodes[j])
        num_dac = class_dfc[j]
        numerator = num_dac - (num_meths*num_fields)
        denominator = num_fields - (num_meths*num_fields)
        if denominator == 0:
            lcom5_list.append("Undefined")
        else:
            lcom5_list.append(numerator/denominator)
    return lcom5_list

# Cell
def add_lcom5(df, col):
    """Adds a column with the LCOM5 of each class of each file to a dataframe"""
    lang_builds = create_parser_builds()
    parser = Parser()
    class_lcom5 = []

    for i in range(len(df)):
        ext = df["name"][i].split('.')[-1]
        parser.set_language(lang_builds[ext])
        enc = df["encoding"][i]
        tree = parser.parse(bytes(df["contents"][i], df["encoding"][i]))
        class_lcom5.append(calculate_lcom5(tree, ext, bytes(df["contents"][i], df["encoding"][i]), df["name"][i]))
    df["class_lcom5"] = class_lcom5

    return df

# Cell
def flatten_lol(list_list):
    """Takes in a list of lists and flattens it, returning a list of each entry"""
    flattened_list = []
    for sublist in list_list:
        for entry in sublist:
            flattened_list.append(entry)
    return flattened_list

# Cell
def display_numeric_col_stats(col, conf = 0.95, sig_figs = 4, clean=True, verbose_clean=False):
    """Computes statistical metrics about the entries in a dataframe column or list"""
    previous_length = len(col)
    numeric_types = [int, float, complex]
    if clean: col = [x for x in col if type(x) in numeric_types]
    if verbose_clean: print(f"Cleaning removed {previous_length - len(col)} non-numeric entries")

    if len(col) < 1:
        print("Error, data must contain at least one valid entry to display statistics")
        return

    print("Min =", round(min(col), sig_figs))
    print("Max =", round(max(col), sig_figs))
    print("Average =", round(mean(col), sig_figs))
    print("Median =", round(median(col), sig_figs))
    print("Standard Deviation =", round(std(col), sig_figs))

    n = len(col)
    m = mean(col)
    std_err = sem(col)
    h = std_err * t.ppf((1 + conf) / 2, n - 1)

    start = m - h
    end = m + h
    print(f"{conf} of data points fall between {round(start, sig_figs)} and {round(end, sig_figs)}")

# Cell
def display_numeric_col_hist(col, col_name="Metric", num_bins=20, clean=True, verbose_clean=False):
    """Displays a histogram with a customized number of bins for the data in a specified dataframe column or list"""
    previous_length = len(col)
    numeric_types = [int, float, complex]
    if clean: col = [x for x in col if type(x) in numeric_types]
    if verbose_clean: print(f"Cleaning removed {previous_length - len(col)} non-numeric entries")

    if len(col) < 1:
        print("Error, data must contain at least one valid entry to display histogram")
        return

    rng = max(col) - min(col)
    num = len(col)
    stnd_dev = std(col)

    plt.hist(col, num_bins, color="blue", alpha=0.5, edgecolor="black", linewidth=1.0)
    plt.title(col_name + " Histogram")
    plt.ylabel("Value  Range  Occurrences")
    plt.xlabel(col_name)
    plt.show()