#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Oct 28 18:37:20 2018

@author: gykovacs

This script contains all the functions and codes we used to produce the
tables in the paper. Before using the script, update the "results_path" variable
below to the path containing the cache file generated by the CacheAndValidate
object.
"""

import sys

# import SMOTE variants
import smote_variants as sv

# imbalanced databases
import imbalanced_databases as imbd

# some packages to transform the data
import numpy as np
import pandas as pd
import pickle

# path to the file generated by the CacheAndValidate object of the smote_variants package
results_path='/home/gykovacs/workspaces/smote/results.pickle'

# the thresholds used to categorize the datasets
ir_threshold= 9
n_min_threshold= 30
n_attr_threshold= 10

category_mapping= {'NR': 'noise removal',
                   'DR': 'dimension reduction',
                   'Clas': 'uses classifier',
                   'SCmp': 'componentwise sampling',
                   'SCpy': 'sampling by cloning',
                   'SO': 'ordinary sampling',
                   'M': 'memetic',
                   'DE': 'density estimation',
                   'DB': 'density based',
                   'Ex': 'extensive',
                   'CM': 'changes majority',
                   'Clus': 'uses clustering',
                   'BL': 'borderline',
                   'A': 'application'}

def tokenize_bibtex(entry):
    """
    Tokenize bibtex entry string
    Args:
        entry(str): string of a bibtex entry
    Returns:
        dict: the bibtex entry
    """
    start= entry.find('{') + 1
    token_start= start
    quote_level= 0
    brace_level= 0
    tokens= []
    for i in range(start, len(entry)):
        if entry[i] == '"':
            quote_level= quote_level + 1
        if entry[i] == '{':
            brace_level= brace_level + 1
        if entry[i] == '}':
            brace_level= brace_level - 1
        if (entry[i] == ',' and brace_level == 0) or (entry[i] == '}' and brace_level < 0) and quote_level % 2 == 0:
            tokens.append(entry[token_start:(i)])
            token_start= i + 1
    
    result= {}
    result['key']= tokens[0].strip()
    for i in range(1, len(tokens)):
        splitted= tokens[i].strip().split('=', 1)
        if len(splitted) == 2:
            key= splitted[0].strip().lower()
            value= splitted[1].strip()[1:-1]
            result[key]= value
            
    if 'year' not in result:
        print("No year attribute in %s" % result['key'])
        
    return result
            
def extract_bibtex_entry(string, types= ['@article', '@inproceedings', '@book', '@unknown']):
    """
    Extract bibtex entry from string
    Args:
        string (str): string to process
        types (list(str)): types of bibtex entries to find
    Returns:
        dict: the dict of the bibtex entry
    """
    lowercase= string.lower()
    for t in types:
        t= t.lower()
        i= lowercase.find(t)
        if i >= 0:
            num_brace= 0
            for j in range(i+len(t), len(string)):
                if string[j] == '{':
                    num_brace+= 1
                if string[j] == '}':
                    num_brace-= 1
                if num_brace == 0:
                    be= tokenize_bibtex(string[i:(j+1)])
                    be['type']= t
                    return be
    return {}

def oversampling_bib_lookup():
    """
    Creates a bibtex lookup table for oversampling techniques based on
    the bibtex entries in the source code.
    
    Returns:
        dict: a lookup table for bibtex entries
    """
    oversamplers= sv.get_all_oversamplers()
    if sv.NoSMOTE in oversamplers:
        oversamplers.remove(sv.NoSMOTE)
    
    oversampling_bibtex= {o.__name__: extract_bibtex_entry(o.__doc__) for o in oversamplers}
    
    return oversampling_bibtex

def oversampler_summary_table():
    """
    Creates the oversampler summary table.
    """
    oversamplers= sv.get_all_oversamplers()
    oversamplers.remove(sv.NoSMOTE)

    all_categories= [sv.OverSampling.cat_noise_removal,
                        sv.OverSampling.cat_dim_reduction,
                        sv.OverSampling.cat_uses_classifier,
                        sv.OverSampling.cat_sample_componentwise,
                        sv.OverSampling.cat_sample_ordinary,
                        sv.OverSampling.cat_sample_copy,
                        sv.OverSampling.cat_memetic,
                        sv.OverSampling.cat_density_estimation,
                        sv.OverSampling.cat_density_based,
                        sv.OverSampling.cat_extensive,
                        sv.OverSampling.cat_changes_majority,
                        sv.OverSampling.cat_uses_clustering,
                        sv.OverSampling.cat_borderline,
                        sv.OverSampling.cat_application]
    
    for o in oversamplers:
        sys.stdout.write(o.__name__ + " ")
        sys.stdout.write("& ")
        for i in range(len(all_categories)):
            if all_categories[i] in o.categories:
                sys.stdout.write("$\\times$ ")
            else:
                sys.stdout.write(" ")
            if i != len(all_categories)-1:
                sys.stdout.write("& ")
            else:
                print("\\\\")
    
    oversampling_bibtex= {o.__name__: extract_bibtex_entry(o.__doc__) for o in oversamplers}
    oversampling_years= {o.__name__: oversampling_bibtex[o.__name__]['year'] for o in oversamplers}
    
    oversamplers= sorted(oversamplers, key= lambda x: oversampling_years[x.__name__])
    
    cat_summary= []
    for o in oversamplers:
        cat_summary.append({'method': o.__name__.replace('_', '-') + ' (' + oversampling_years[o.__name__] + ')' + 'cite(' + oversampling_bibtex[o.__name__]['key'] + '))'})
        for a in all_categories:
            cat_summary[-1][a]= str(a in o.categories)
    
    pd.set_option('max_colwidth', 100)
    cat_summary= pd.DataFrame(cat_summary)
    cat_summary= cat_summary[['method'] + all_categories]
    cat_summary.index= np.arange(1, len(cat_summary) + 1)
    cat_summary_first= cat_summary.iloc[:int(len(cat_summary)/2+0.5)].reset_index()
    cat_summary_second= cat_summary.iloc[int(len(cat_summary)/2+0.5):].reset_index()

    cat_summary_second['index']= cat_summary_second['index'].astype(str)
    results= pd.concat([cat_summary_first, cat_summary_second], axis= 1)
    
    res= results.to_latex(index= False)
    res= res.replace('True', '$\\times$').replace('False', '')
    prefix= '\\begin{turn}{90}'
    postfix= '\\end{turn}'
    res= res.replace(' NR ', prefix + 'noise removal' + postfix)
    res= res.replace(' DR ', prefix + 'dimension reduction' + postfix)
    res= res.replace(' Clas ', prefix + 'uses classifier' + postfix)
    res= res.replace(' SCmp ', prefix + 'componentwise sampling' + postfix)
    res= res.replace(' SCpy ', prefix + 'sampling by cloning' + postfix)
    res= res.replace(' SO ', prefix + 'ordinary sampling' + postfix)
    res= res.replace(' M ', prefix + 'memetic' + postfix)
    res= res.replace(' DE ', prefix + 'density estimation' + postfix)
    res= res.replace(' DB ', prefix + 'density based' + postfix)
    res= res.replace(' Ex ', prefix + 'extensive' + postfix)
    res= res.replace(' CM ', prefix + 'changes majority' + postfix)
    res= res.replace(' Clus ', prefix + 'uses clustering' + postfix)
    res= res.replace(' BL ', prefix + 'borderline' + postfix)
    res= res.replace(' A ', prefix + 'application' + postfix)
    res= res.replace('index', '')
    res= res.replace('\\toprule', '')
    res= res.replace('cite(', '\\cite{')
    res= res.replace('))', '}')
    res= res.replace('\_', '_')
    res= res.replace('NaN', '')

    print(res)

def dataset_summary_table():
    """
    Creates the dataset summary table.
    """
    results= imbd.summary(include_citation= True, subset= 'study')
    
    num_features_upper_bound= 100
    len_upper_bound= 4000
    abalone19= results[results['name'] == 'abalone19']
    results= results[(results['len'] < len_upper_bound) & (results['encoded_n_attr'] < num_features_upper_bound)]
    results= results.extend(abalone19)
    
    citation_keys= results['citation'].apply(lambda x: tokenize_bibtex(x)['key'])
    citation_keys= citation_keys.apply(lambda x: '((' + x + '))')
    #results= results[['name', 'len', 'n_minority', 'encoded_n_attr', 'imbalance_ratio', 'imbalance_ratio_dist']]
    results= results[['name', 'len', 'n_minority', 'encoded_n_attr', 'imbalance_ratio']]
    results['name']= results['name'] + citation_keys
    #results.columns= ['name', 'n', 'n_min', 'n_attr', 'ir', 'idr']
    results.columns= ['name', 'n', 'n_min', 'n_attr', 'ir']
    results= results.sort_values('ir')
    results.index= np.arange(1, len(results) + 1)
    results['ir']= results['ir'].round(2)
    #results['idr']= results['idr'].round(2)
    res1= results.iloc[:int(len(results)/2)].reset_index()
    res2= results.iloc[int(len(results)/2):].reset_index()
    res_all= pd.concat([res1, res2], axis= 1)
    
    res= res_all.to_latex(index= False)
    res= res.replace('index', '')
    res= res.replace('\\toprule', '')
    res= res.replace('((', '\\cite{')
    res= res.replace('))', '}')
    
    print(res)

def top_score_and_classifier(score, databases= 'all', n_entries= 8):
    """
    Creates the table of top performers for a specific score
    Args:
        score (str): 'acc'/'gacc'/'f1'/'auc'/'brier'/'p_top20'
        databases (str): 'all'/'high_ir'/'low_ir'/'high_n_min'/'low_n_min'/'high_n_attr'/'low_n_attr'
        n_entries (int): number of entries to show
    """
    ascending= False
    if score == 'brier':
        ascending= True
    
    oversampling_bibtex= oversampling_bib_lookup()
    
    results= pickle.load(open(results_path, 'rb'))
    
    if databases == 'high_ir':
        results= results[results['imbalanced_ratio'] > ir_threshold]
    elif databases == 'low_ir':
        results= results[results['imbalanced_ratio'] <= ir_threshold]
    elif databases == 'high_n_min':
        results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) > n_min_threshold]
    elif databases == 'low_n_min':
        results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) <= n_min_threshold]
    elif databases == 'high_n_attr':
        results= results[results['db_n_attr'] > n_attr_threshold]
    elif databases == 'low_n_attr':
        results= results[results['db_n_attr'] <= n_attr_threshold]
    
    results_score= results[['db_name', 'classifier', 'sampler', score]]
    results_agg= results_score.groupby(by=['classifier', 'sampler']).agg({score: np.mean})
    results_agg= results_agg.reset_index()
    
    results_svm= results_agg[results_agg['classifier'] == 'CalibratedClassifierCV']
    results_dt= results_agg[results_agg['classifier'] == 'DecisionTreeClassifier']
    results_knn= results_agg[results_agg['classifier'] == 'KNeighborsClassifier']
    results_mlp= results_agg[results_agg['classifier'] == 'MLPClassifierWrapper']
    
    final_svm= results_svm[['sampler', score]].sort_values(by= score, ascending= ascending).iloc[:n_entries].reset_index(drop= True)
    final_dt= results_dt[['sampler', score]].sort_values(by= score, ascending= ascending).iloc[:n_entries].reset_index(drop= True)
    final_knn= results_knn[['sampler', score]].sort_values(by= score, ascending= ascending).iloc[:n_entries].reset_index(drop= True)
    final_mlp= results_mlp[['sampler', score]].sort_values(by= score, ascending= ascending).iloc[:n_entries].reset_index(drop= True)
    
    final_svm['sampler']= final_svm['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    final_dt['sampler']= final_dt['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    final_knn['sampler']= final_knn['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    final_mlp['sampler']= final_mlp['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    
    final= pd.concat([final_svm, final_dt, final_knn, final_mlp], axis= 1)
    baseline= results_agg[results_agg['sampler'] == 'NoSMOTE']
    smote= results_agg[results_agg['sampler'] == 'SMOTE']
    
    iterables= [['SVM', 'DT', 'kNN', 'MLP'], ['sampler', score]]
    index= pd.MultiIndex.from_product(iterables, names= ['classifier', ''])
    
    final.columns= index
    
    final.index= final.index + 1
    
    final= final.append(pd.DataFrame({final.columns[0]: 'SMOTE', final.columns[1]: smote[smote['classifier'] == 'CalibratedClassifierCV'][score].iloc[0],
                                      final.columns[2]: 'SMOTE', final.columns[3]: smote[smote['classifier'] == 'DecisionTreeClassifier'][score].iloc[0],
                                      final.columns[4]: 'SMOTE', final.columns[5]: smote[smote['classifier'] == 'KNeighborsClassifier'][score].iloc[0],
                                      final.columns[6]: 'SMOTE', final.columns[7]: smote[smote['classifier'] == 'MLPClassifierWrapper'][score].iloc[0]}, index=['baseline']))
    
    final= final.append(pd.DataFrame({final.columns[0]: 'no sampling', final.columns[1]: baseline[baseline['classifier'] == 'CalibratedClassifierCV'][score].iloc[0],
                                      final.columns[2]: 'no sampling', final.columns[3]: baseline[baseline['classifier'] == 'DecisionTreeClassifier'][score].iloc[0],
                                      final.columns[4]: 'no sampling', final.columns[5]: baseline[baseline['classifier'] == 'KNeighborsClassifier'][score].iloc[0],
                                      final.columns[6]: 'no sampling', final.columns[7]: baseline[baseline['classifier'] == 'MLPClassifierWrapper'][score].iloc[0]}, index=['baseline']))
    
    table= final.to_latex(float_format= lambda x: ('%.4f' % x).replace('0.', '.'))
    table= table.replace('cite(', '\\cite{')
    table= table.replace('))', '}')
    table= table.replace('\_', '_')
    print(table)
    
    return final

def top_results_by_scores(databases= 'all', n_entries= 8):
    """
    Creates a table summarizing the overall performances by scores
    Args:
        databases (str): 'all'/'high_ir'/'low_ir'/'high_n_min'/'low_n_min'/'high_n_attr'/'low_n_attr'
        n_entries (int): number of entries to show
    """
    results= pickle.load(open(results_path, 'rb'))
    
    oversampling_bibtex= oversampling_bib_lookup()
    
    if databases == 'high_ir':
        results= results[results['imbalanced_ratio'] > ir_threshold]
    elif databases == 'low_ir':
        results= results[results['imbalanced_ratio'] <= ir_threshold]
    elif databases == 'high_n_min':
        results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) > n_min_threshold]
    elif databases == 'low_n_min':
        results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) <= n_min_threshold]
    elif databases == 'high_n_attr':
        results= results[results['db_n_attr'] > n_attr_threshold]
    elif databases == 'low_n_attr':
        results= results[results['db_n_attr'] <= n_attr_threshold]
        
    results_agg= results.groupby(by='sampler').agg({'auc': np.mean, 'gacc': np.mean, 'f1': np.mean, 'p_top20': np.mean})
    results_agg= results_agg.reset_index()

    final_auc= results_agg[['sampler', 'auc']].sort_values(by= 'auc', ascending= False).iloc[:n_entries].reset_index(drop= True)
    final_auc['sampler']= final_auc['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    final_gacc= results_agg[['sampler', 'gacc']].sort_values(by= 'gacc', ascending= False).iloc[:n_entries].reset_index(drop= True)
    final_gacc['sampler']= final_gacc['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    final_f1= results_agg[['sampler', 'f1']].sort_values(by= 'f1', ascending= False).iloc[:n_entries].reset_index(drop= True)
    final_f1['sampler']= final_f1['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    final_ptop20= results_agg[['sampler', 'p_top20']].sort_values(by= 'p_top20', ascending= False).iloc[:n_entries].reset_index(drop= True)
    final_ptop20['sampler']= final_ptop20['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    
    final= pd.concat([final_auc, final_gacc, final_f1, final_ptop20], axis= 1)
    smote= results_agg[results_agg['sampler'] == 'SMOTE']
    baseline= results_agg[results_agg['sampler'] == 'NoSMOTE']

    iterables= [['AUC', 'GACC', 'F1', '% top20'], ['sampler', 'score']]
    index= pd.MultiIndex.from_product(iterables, names= ['score', ''])

    final.columns= index

    final.index= final.index + 1

    final= final.append(pd.DataFrame({final.columns[0]: 'SMOTE', final.columns[1]: smote['auc'].iloc[0],
                                     final.columns[2]: 'SMOTE', final.columns[3]: smote['gacc'].iloc[0],
                                     final.columns[4]: 'SMOTE', final.columns[5]: smote['f1'].iloc[0],
                                     final.columns[6]: 'SMOTE', final.columns[7]: smote['p_top20'].iloc[0]}, index= ['baseline']))

    final= final.append(pd.DataFrame({final.columns[0]: 'no sampling', final.columns[1]: baseline['auc'].iloc[0],
                           final.columns[2]: 'no sampling', final.columns[3]: baseline['gacc'].iloc[0],
                           final.columns[4]: 'no sampling', final.columns[5]: baseline['f1'].iloc[0],
                           final.columns[6]: 'no sampling', final.columns[7]: baseline['p_top20'].iloc[0],}, index= ['baseline']))
    
    final.columns= ['sampler', 'AUC', 'sampler', 'GAcc', 'sampler', 'F1', 'sampler', 'P20']

    table= final.to_latex(float_format= lambda x: ('%.4f' % x).replace('0.', '.'))
    table= table.replace('cite(', '\\cite{')
    table= table.replace('))', '}')
    table= table.replace('\_', '_')
    print(table)
    
    return final

def top_results_overall(databases= 'all', n_entries= 10):
    """
    Creates a table summarizing the overall performances
    Args:
        databases (str): 'all'/'high_ir'/'low_ir'/'high_n_min'/'low_n_min'/'high_n_attr'/'low_n_attr'
        n_entries (int): number of entries to show
    """
    results= pickle.load(open(results_path, 'rb'))
    
    oversampling_bibtex= oversampling_bib_lookup()
    
    if databases == 'high_ir':
        results= results[results['imbalanced_ratio'] > ir_threshold]
    elif databases == 'low_ir':
        results= results[results['imbalanced_ratio'] <= ir_threshold]
    elif databases == 'high_n_min':
        results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) > n_min_threshold]
    elif databases == 'low_n_min':
        results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) <= n_min_threshold]
    elif databases == 'high_n_attr':
        results= results[results['db_n_attr'] > n_attr_threshold]
    elif databases == 'low_n_attr':
        results= results[results['db_n_attr'] <= n_attr_threshold]
    
    results_agg= results.groupby(by='sampler').agg({'auc': np.mean, 'gacc': np.mean, 'f1': np.mean, 'p_top20': np.mean})
    results_agg= results_agg.reset_index()
    results_agg= results_agg[results_agg['sampler'] != 'NoSMOTE']
    
    results_rank= results_agg.rank(numeric_only= True, ascending= False)
    results_rank['sampler']= results_agg['sampler']
    results_rank['overall']= np.mean(results_rank[['auc', 'gacc', 'f1', 'p_top20']], axis= 1)
    results_rank.columns= ['rank_auc', 'rank_gacc', 'rank_f1', 'rank_ptop20', 'sampler', 'overall']
    results_agg['rank_auc']= results_rank['rank_auc']
    results_agg['rank_gacc']= results_rank['rank_gacc']
    results_agg['rank_f1']= results_rank['rank_f1']
    results_agg['rank_ptop20']= results_rank['rank_ptop20']
    results_agg['overall']= results_rank['overall']
    results_agg= results_agg.sort_values('overall')
    results_agg= results_agg[['sampler', 'overall', 'auc', 'rank_auc', 'gacc', 'rank_gacc', 'f1', 'rank_f1', 'p_top20', 'rank_ptop20']]
    results_agg= results_agg.reset_index(drop= True)
    results_agg.index= results_agg.index + 1
    final= results_agg.iloc[:n_entries]
    
    final['rank_auc']= final['rank_auc'].astype(int)
    final['rank_gacc']= final['rank_gacc'].astype(int)
    final['rank_f1']= final['rank_f1'].astype(int)
    final['rank_ptop20']= final['rank_ptop20'].astype(int)
    
    final['sampler']= final['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')

    table= final.to_latex(float_format= lambda x: ('%.4f' % x).replace(' 0.', '.'))
    table= table.replace('cite(', '\\cite{')
    table= table.replace('))', '}')
    table= table.replace('\_', '_')
    print(table)
    
    return final

def top_results_by_dataset_types(n_entries= 10):
    """
    Creates a table summarizing the overall performances by dataset types
    
    Args:
        n_entries (int): number of entries to show
    """
    results= pickle.load(open(results_path, 'rb'))
    
    oversampling_bibtex= oversampling_bib_lookup()
    
    def ranking(results, databases):
        if databases == 'high_ir':
            results= results[results['imbalanced_ratio'] > ir_threshold]
        elif databases == 'low_ir':
            results= results[results['imbalanced_ratio'] <= ir_threshold]
        elif databases == 'high_n_min':
            results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) > n_min_threshold]
        elif databases == 'low_n_min':
            results= results[results['db_size']/(1.0 + results['imbalanced_ratio']) <= n_min_threshold]
        elif databases == 'high_n_attr':
            results= results[results['db_n_attr'] > n_attr_threshold]
        elif databases == 'low_n_attr':
            results= results[results['db_n_attr'] <= n_attr_threshold]
        
        results_agg= results.groupby(by='sampler').agg({'auc': np.mean, 'gacc': np.mean, 'f1': np.mean, 'p_top20': np.mean})
        results_agg= results_agg.reset_index()
        results_agg= results_agg[results_agg['sampler'] != 'NoSMOTE']
        
        results_rank= results_agg.rank(numeric_only= True, ascending= False)
        results_rank['sampler']= results_agg['sampler']
        results_rank['overall']= np.mean(results_rank[['auc', 'gacc', 'f1', 'p_top20']], axis= 1)
        results_rank.columns= ['rank_auc', 'rank_gacc', 'rank_f1', 'rank_ptop20', 'sampler', 'overall']
        results_agg['rank_auc']= results_rank['rank_auc']
        results_agg['rank_gacc']= results_rank['rank_gacc']
        results_agg['rank_f1']= results_rank['rank_f1']
        results_agg['rank_ptop20']= results_rank['rank_ptop20']
        results_agg['overall']= results_rank['overall']
        results_agg= results_agg.sort_values('overall')
        results_agg= results_agg[['sampler', 'overall', 'auc', 'rank_auc', 'gacc', 'rank_gacc', 'f1', 'rank_f1', 'p_top20', 'rank_ptop20']]
        results_agg= results_agg.reset_index(drop= True)
        results_agg.index= results_agg.index + 1
        final= results_agg
        
        final['rank_auc']= final['rank_auc'].astype(int)
        final['rank_gacc']= final['rank_gacc'].astype(int)
        final['rank_f1']= final['rank_f1'].astype(int)
        final['rank_ptop20']= final['rank_ptop20'].astype(int)
        
        final['overall']= final['overall'].rank()
        
        return final[['sampler', 'overall']]
    
    results_all= ranking(results, databases= 'all')
    results_all['overall']= results_all['overall'].rank()
    results_high_ir= ranking(results, databases= 'high_ir')
    results_high_ir.columns= ['sampler', 'overall_high_ir']
    results_low_ir= ranking(results, databases= 'low_ir')
    results_low_ir.columns= ['sampler', 'overall_low_ir']
    results_high_n_min= ranking(results, databases= 'high_n_min')
    results_high_n_min.columns= ['sampler', 'overall_high_n_min']
    results_low_n_min= ranking(results, databases= 'low_n_min')
    results_low_n_min.columns= ['sampler', 'overall_low_n_min']
    results_high_n_attr= ranking(results, databases= 'high_n_attr')
    results_high_n_attr.columns= ['sampler', 'overall_high_n_attr']
    results_low_n_attr= ranking(results, databases= 'low_n_attr')
    results_low_n_attr.columns= ['sampler', 'overall_low_n_attr']
    
    final= results_all.merge(results_high_ir, on= 'sampler').merge(results_low_ir, on= 'sampler').merge(results_high_n_min, on= 'sampler').merge(results_low_n_min, on= 'sampler').merge(results_high_n_attr, on= 'sampler').merge(results_low_n_attr, on= 'sampler')
    final= final[(final['overall'] <= n_entries) | (final['overall_high_ir'] <= n_entries) | (final['overall_low_ir'] <= n_entries) | (final['overall_high_n_min'] <= n_entries) | (final['overall_low_n_min'] <= n_entries) | (final['overall_high_n_attr'] <= n_entries) | (final['overall_low_n_attr'] <= n_entries)]
    
    final['sampler']= final['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    final= final.reset_index(drop= True)
    final.index= final.index + 1

    table= final.to_latex(float_format= lambda x: ('%.0f' % x).replace(' 0.', '.'))
    table= table.replace('cite(', '\\cite{')
    table= table.replace('))', '}')
    table= table.replace('\_', '_')
    
    print(table)
    
    return final

def top_results_by_categories(percentile= 50):
    """
    Creates a table of the ranking of categories
    Args:
        percentile (int): the percentile of scores to be used
    """
    results= pickle.load(open(results_path, 'rb'))
    categories= ['NR', 'DR', 'Clas', 'SCmp', 'SCpy', 'SO', 'M', 'DE', 'DB', 'Ex', 'CM', 'Clus', 'BL', 'A']

    for i in categories:
        results[i]= results['sampler_categories'].apply(lambda x: i in eval(x))
    
    res_cat= {}
    for i in categories:
        res_cat[i]= results[results[i] == True]
    
    for r in res_cat:
        res_cat[r]= res_cat[r].groupby(by=['sampler']).agg({'auc': np.mean,
                                                           'f1': np.mean,
                                                           'gacc': np.mean,
                                                           'acc': np.mean,
                                                           'brier': np.mean,
                                                           'p_top20': np.mean})
    
    res= {}
    for i in categories:
        res[category_mapping[i]]= {'auc': np.percentile(res_cat[i]['auc'], percentile),
                       'f1': np.percentile(res_cat[i]['f1'], percentile),
                       'gacc': np.percentile(res_cat[i]['gacc'], percentile),
                       'acc': np.percentile(res_cat[i]['acc'], percentile),
                       'p_top20': np.percentile(res_cat[i]['p_top20'], percentile)}
    
    r= pd.DataFrame.from_dict(res).T
    
    r_auc= r['auc'].sort_values(ascending= False)
    r_gacc= r['gacc'].sort_values(ascending= False)
    r_f1= r['f1'].sort_values(ascending= False)
    r_ptop20= r['p_top20'].sort_values(ascending= False)
    
    r_auc= r_auc.reset_index()
    r_gacc= r_gacc.reset_index()
    r_f1= r_f1.reset_index()
    r_ptop20= r_ptop20.reset_index()
    
    final= pd.concat([r_auc, r_gacc, r_f1, r_ptop20], axis= 1, ignore_index= True)
    
    iterables= [['AUC', 'GACC', 'F1', '% top20'], ['attribute', 'score']]
    index= pd.MultiIndex.from_product(iterables, names= ['score', ''])

    final.columns= index

    final.index= final.index + 1
    
    table= final.to_latex(float_format= lambda x: ('%.4f' % x).replace('0.', '.'))
    table= table.replace('cite(', '\\cite{')
    table= table.replace('))', '}')
    table= table.replace('\_', '_')
    
    print(table)

def runtimes():
    """
    Creates a table of ranking the algorithms by average runtimes
    """
    results= pickle.load(open(results_path, 'rb'))
    
    oversampling_bibtex= oversampling_bib_lookup()
    
    results= results[results['sampler'] != 'NoSMOTE']
    
    results_agg= results.groupby('sampler').aggregate({'runtime': np.mean})
    results_sorted= results_agg.sort_values('runtime')
    
    results_sorted= results_sorted.reset_index()
    
    n= int(len(results_sorted)/3 + 0.5) + 1
    
    results_sorted.index= results_sorted.index + 1
    
    results_sorted['sampler']= results_sorted['sampler'].apply(lambda x: x.replace('_', '-') + ' cite(' + oversampling_bibtex[x]['key'] + '))')
    
    final= pd.concat([results_sorted.iloc[:n].reset_index(), results_sorted.iloc[n:2*n].reset_index(), results_sorted.iloc[2*n:3*n].reset_index()], axis= 1)
    
    table= final.to_latex(float_format= lambda x: ('%.4f' % x).replace(' 0.', ' .'), index= False)
    table= table.replace('cite(', '\\cite{')
    table= table.replace('))', '}')
    table= table.replace('\_', '_')
    
    print(table)

def create_documentation_page_os():
    oversamplers= sv.get_all_oversamplers()
    
    docs= "Oversamplers\n"
    docs= docs + "*"*len("Oversamplers") + "\n\n"
    
    for o in oversamplers:
        docs= docs + o.__name__ + "\n" + '-'*len(o.__name__) + "\n"
        docs= docs + "\n\n"
        docs= docs + "API\n"
        docs= docs + "^"*len("API") + "\n\n"
        docs= docs + ('.. autoclass:: smote_variants.%s' % o.__name__) + "\n"
        docs= docs + ('    :members:') + "\n"
        docs= docs + "\n"
        docs= docs + ('    .. automethod:: __init__')
        docs= docs + "\n\n"
        docs= docs + "Example\n"
        docs= docs + "^"*len("Example")
        docs= docs + "\n\n"
        docs= docs + ("    >>> oversampler= smote_variants.%s()\n" % o.__name__)
        docs= docs + "    >>> X_samp, y_samp= oversampler.sample(X, y)\n"
        docs= docs + "\n\n"
        docs= docs + ".. image:: figures/base.png" + "\n"
        docs= docs + (".. image:: figures/%s.png" % o.__name__) + "\n\n"
        docs= docs + o.__doc__.replace("\n    ", "\n")
    
    file= open("oversamplers.rst", "w")
    file.write(docs)
    file.close()
    
    return docs

def create_documentation_page_nf():
    noise_filters= sv.get_all_noisefilters()
    
    docs= "Noise filters and prototype selection\n"
    docs= docs + "*"*len("Noise filters and prototype selection") + "\n\n"
    
    for o in noise_filters:
        docs= docs + o.__name__ + "\n" + '='*len(o.__name__) + "\n"
        docs= docs + "\n\n"
        docs= docs + "API\n"
        docs= docs + "^"*len("API") + "\n\n"
        docs= docs + ('.. autoclass:: smote_variants.%s' % o.__name__) + "\n"
        docs= docs + ('    :members:') + "\n"
        docs= docs + "\n"
        docs= docs + ('    .. automethod:: __init__')
        docs= docs + "\n\n"
        docs= docs + "Example\n"
        docs= docs + "^"*len("Example")
        docs= docs + "\n\n"
        docs= docs + ("    >>> noise_filter= smote_variants.%s()\n" % o.__name__)
        docs= docs + "    >>> X_samp, y_samp= noise_filter.remove_noise(X, y)\n"
        docs= docs + "\n\n"
        docs= docs + ".. image:: figures/base.png" + "\n"
        docs= docs + (".. image:: figures/%s.png" % o.__name__) + "\n\n"
        docs= docs + o.__doc__.replace("\n    ", "\n")
    
    file= open("noise_filters.rst", "w")
    file.write(docs)
    file.close()
    
    return docs

def create_gallery_page():
    oversamplers= sv.get_all_oversamplers()
    noise_filters= sv.get_all_noisefilters()
    
    docs= "Gallery\n" + '*'*len('Gallery\n') + "\n\n"
    
    docs= docs + "In this page we demonstrate the output of various oversampling \
                    and noise removal techniques, using default parameters.\n\n"
    docs= docs + "For binary oversampling and noise removal, the figures can be reproduced by the ``ballpark_sample`` function using \
                    a built-in or user definied dataset:\n\n"
    docs= docs + ".. autofunction:: smote_variants.ballpark_sample\n\n"
    
    docs= docs + "For multiclass oversampling we have used the 'wine' dataset from \
                    ``sklearn.datasets``, which has 3 classes and many features, out \
                    which the first two coordinates have been used for visualization.\n\n"
    
    docs= docs + "Oversampling sample results\n"
    docs= docs + "="*len('Oversampling sample results\n') + "\n\n"
    
    docs= docs + "In the captions of the images some abbreviations \
                    referring to the operating principles are placed. Namely:\n\n"
    docs= docs + "    * NR: noise removal is involved\n"
    docs= docs + "    * DR: dimension reduction is applied\n"
    docs= docs + "    * Clas: some supervised classifier is used\n"
    docs= docs + "    * SCmp: sampling is carried out componentwise (attributewise)\n"
    docs= docs + "    * SCpy: sampling is carried out by copying instances\n"
    docs= docs + "    * SO: ordinary sampling (just like in SMOTE)\n"
    docs= docs + "    * M: memetic optimization is used\n"
    docs= docs + "    * DE: density estimation is used\n"
    docs= docs + "    * DB: density based - the sampling is based on a density of importance assigned to the instances\n"
    docs= docs + "    * Ex: the sampling is extensive - samples are added successively, not optimizing the holistic distribution of a given number of samples\n"
    docs= docs + "    * CM: changes majority - even majority samples can change\n"
    docs= docs + "    * Clus: uses some clustering technique\n"
    docs= docs + "    * BL: identifies and samples the neighborhoods of borderline samples\n"
    docs= docs + "    * A: developed for a specific application\n"
    
    docs= docs + "\n"
    docs= docs + ".. figure:: figures/base.png" + "\n\n\n"
    
    i= 0
    for o in oversamplers:
        docs= docs + (".. image:: figures/%s.png\n" % o.__name__)
        i= i + 1
        if i % 4 == 0:
            docs= docs + "\n"
    
    docs= docs + "Noise removal sample results\n"
    docs= docs + "="*len('Noise removal sample results\n') + "\n\n"
    
    docs= docs + ".. figure:: figures/base.png" + "\n\n\n"
    
    i= 0
    for n in noise_filters:
        docs= docs + (".. image:: figures/%s.png\n" % n.__name__)
        i= i + 1
        if i % 4 == 0:
            docs= docs + "\n"
            
    docs= docs + "Multiclass sample results\n"
    docs= docs + "="*len('Multiclass sample results\n') + "\n\n"
    
    docs= docs + ".. figure:: figures/multiclass-base.png" + "\n\n\n"
    
    oversamplers= [o for o in oversamplers if not sv.OverSampling.cat_changes_majority in o.categories and 'proportion' in o().get_params()]
    
    i= 0
    for o in oversamplers:
        docs= docs + (".. image:: figures/multiclass-%s.png\n" % o.__name__)
        i= i + 1
        if i % 4 == 0:
            docs= docs + "\n"
            
    file= open("gallery.rst", "w")
    file.write(docs)
    file.close()
    
    return docs

def generate_figures():
    oversamplers= sv.get_all_oversamplers()

    for o in oversamplers:
        sv.ballpark_sample(o(), img_file_base= 'figures/base.png', img_file_sampled= ('figures/%s.png' % o.__name__))
        
    noisefilters= sv.get_all_noisefilters()
    
    for n in noisefilters:
        sv.ballpark_sample(n(), img_file_base= 'figures/base.png', img_file_sampled= ('figures/%s.png' % n.__name__))

def generate_multiclass_figures():
    oversamplers= sv.get_all_oversamplers()
    oversamplers= [o for o in oversamplers if not sv.OverSampling.cat_changes_majority in o.categories and 'proportion' in o().get_params()]
    
    import sklearn.datasets as datasets
    
    dataset= datasets.load_wine()
    
    X= dataset['data']
    y= dataset['target']
    
    import matplotlib.pyplot as plt
    
    import sklearn.preprocessing as preprocessing
    
    ss= preprocessing.StandardScaler()
    
    X_ss= ss.fit_transform(X)
    
    def plot_and_save(X, y, filename, oversampler_name):
        plt.figure(figsize=(4, 3))
        plt.scatter(X[y == 0][:,0], X[y == 0][:,1], c='r', marker='o', label='class 0')
        plt.scatter(X[y == 1][:,0], X[y == 1][:,1], c='b', marker='P', label='class 1')
        plt.scatter(X[y == 2][:,0], X[y == 2][:,1], c='green', marker='x', label='class 2')
        plt.xlabel('feature 0')
        plt.ylabel('feature 1')
        plt.title(", ".join(["wine dataset", oversampler_name]))
        plt.savefig(filename)
        plt.show()
    
    plot_and_save(X, y, 'figures/multiclass-base.png', "No Oversampling")
    
    for o in oversamplers:
        print(o.__name__)
        mcos= sv.MulticlassOversampling(o())
        X_samp, y_samp= mcos.sample(X_ss, y)
        plot_and_save(ss.inverse_transform(X_samp), y_samp, "figures/multiclass-%s" % o.__name__, o.__name__)

def create_ranking_page():
    final= top_results_overall()
    
    from tabulate import tabulate
    
    docs= "Ranking\n" + "*"*len("Ranking") + "\n\n"
    docs= docs + "Based on a thorough evaluation using 104 imbalanced datasets, the following 10 techniques provide the highest performance in terms of the AUC, GAcc, F1 and P20 scores, in nearest neighbors, support vector machine, decision tree and multilayer perceptron based classification scenarios.\n"
    docs= docs + "For more details on the evaluation methodology, see our paper on the comparative study.\n\n"
    
    docs= docs + tabulate(final.values, final.columns, tablefmt="rst")
    docs= docs + "\n\n"


def generate_all_figures():
    generate_figures()
    generate_multiclass_figures()

def generate_doc_pages():
    create_documentation_page_os()
    create_documentation_page_nf()
    create_gallery_page()
    create_ranking_page()

#######################################
# rendering the 8 tables of the study #
#######################################

# rendering the table summarizing the oversampler techniques
oversampler_summary_table()

# rendering the table summarizing the datasets
dataset_summary_table()

# rendering the tables summarizing the results by scores and classifiers 
top_score_and_classifier('auc')
top_score_and_classifier('gacc')
top_score_and_classifier('f1')
top_score_and_classifier('p_top20')

# rendering the table summarizing the top results by score
top_results_by_scores()

# rendering the table summarizing the overall rankings
top_results_overall()

# rendering the table reporting results by dataset categories
top_results_by_dataset_types()

# rendering the table reporting results by oversampler categories
top_results_by_categories()

# rendering the table of runtimes
runtimes()

