/********************************************************************************************************************
*    Explainable outlier detection
*    
*    Tries to detect outliers by generating decision trees that attempt to predict the values of each column based on
*    each other column, testing in each branch of every tried split (if it meets some minimum criteria) whether there
*    are observations that seem too distant from the others in a 1-D distribution for the column that the split tries
*    to "predict" (will not generate a score for each observation).
*    Splits are based on gain, while outlierness is based on confidence intervals.
*    Similar in spirit to the GritBot software developed by RuleQuest research.
*    
*    
*    Copyright 2019 David Cortes.
*    
*    Written for C++11 standard and OpenMP 2.0 or later. Code is meant to be wrapped into scripting languages
*    such as R or Python.
*
*    This file is part of OutlierTree.
*
*    OutlierTree is free software: you can redistribute it and/or modify
*    it under the terms of the GNU General Public License as published by
*    the Free Software Foundation, either version 3 of the License, or
*    (at your option) any later version.
*
*    OutlierTree is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU General Public License for more details.
*
*    You should have received a copy of the GNU General Public License
*    along with OutlierTree.  If not, see <https://www.gnu.org/licenses/>.
********************************************************************************************************************/

/***********************
    Standard headers
************************/
#include <iostream>
#include <vector>
#include <memory>
#include <algorithm>
#include <numeric>
#include <math.h>
#include <stddef.h>
#include <limits.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#ifdef _OPENMP
    #include <omp.h>
#endif

/******************
  Short Functions
*******************/
#define extract_bit(number, bit) (((number) >> (bit)) & 1) /* https://stackoverflow.com/questions/2249731/how-do-i-get-bit-by-bit-data-from-an-integer-value-in-c */
#define pow2(n) ( ((size_t) 1) << (n) ) /* https://stackoverflow.com/questions/101439/the-most-efficient-way-to-implement-an-integer-based-power-function-powint-int */
#define avg_between(a, b) (((a) + (b)) * 0.5)
#define square(x) ((x) * (x))

#ifndef isnan
    #ifdef _isnan
        #define isnan _isnan
    #else
        #define isnan(x) ( (x) != (x) )
    #endif
#endif
#ifndef isinf
    #ifdef _finite
        #define isinf(x) (!_finite(x))
    #else
        #define isinf(x) ( (x) >= HUGE_VAL || (x) <= -HUGE_VAL )
    #endif
#endif
#define is_na_or_inf(x) (isnan(x) || isinf(x))

/* Functions for 'long double' types */
#ifndef sqrtl
    #define sqrtl sqrt
#endif
#ifndef logl
    #define logl log
#endif
#ifndef fminl
    #define fminl fmin
#endif
#ifndef fmaxl
    #define fmaxl fmax
#endif


/* Aliasing for compiler optimizations */
#ifndef restrict
    #ifdef __restrict
        #define restrict __restrict
    #elif defined(__restrict__)
        #define restrict __restrict__
    #else
        #define restrict 
    #endif
#endif

/* MSVC is stuck with an OpenMP version that's 19 years old at the time of writing and does not support unsigned iterators */
#ifdef _OPENMP
    #if _OPENMP < 200801 /* OpenMP < 3.0 */
        #define size_t_for long
    #else
        #define size_t_for size_t
    #endif
#else
    #define size_t_for size_t
#endif


/************************************************************
  Data types and structs that are returned from this module
*************************************************************/
typedef enum ColType {Numeric, Categorical, Ordinal, NoType} ColType;
typedef enum SplitType {
    LessOrEqual, Greater,  /* for numerical and ordinal */
    Equal, NotEqual,       /* will try to simplify to these post-hoc if possible */
    InSubset, NotInSubset, /* for categoricals */
    SingleCateg, SubTrees, /* one branch per category of a categorical column */
    IsNa, Root
} SplitType;
typedef enum ColTransf {NoTransf, Log, Exp} ColTransf; /* transformation to apply to numeric column */

/*    
*    1-d clusters that define homogeneous groups in which observations can be outliers.
*    Note that these are associated to a tree and define one extra condition from what
*    the tree already specifies. The branch they follow is stored in the cluster, unlike
*    for trees in which it's always left and right branch, as these get discarded more often.
*/
typedef struct Cluster {
    ColType   column_type = NoType;
    size_t col_num = 0; /* numer of the column by which its being split, the target column is given by index of the cluster vector */
    SplitType split_type = Root;
    double split_point = HUGE_VAL; /* numerical */
    std::vector<char> split_subset = std::vector<char>(); /* categorical */
    int split_lev = INT_MAX;    /* ordinal */
    bool has_NA_branch = false; /* this is in order to determine the best outlier cluster when it fits under more than 1 */

    size_t    cluster_size = 0;
    double    lower_lim = HUGE_VAL;                        /* numerical target column */
    double    upper_lim = -HUGE_VAL;                       /* numerical target column */
    double    perc_below = HUGE_VAL;                       /* numerical target column */
    double    perc_above = HUGE_VAL;                       /* numerical target column */
    double    display_lim_low = HUGE_VAL;                  /* numerical target column */
    double    display_lim_high = -HUGE_VAL;                /* numerical target column */
    double    display_mean = -HUGE_VAL;                    /* numerical target column */
    double    display_sd = -HUGE_VAL;                      /* numerical target column */
    std::vector<char> subset_common = std::vector<char>(); /* categorical or ordinal target column (=0 is common) */
    double    perc_in_subset = HUGE_VAL;                   /* categorical or ordinal target column */
    double    perc_next_most_comm = -HUGE_VAL;             /* categorical or ordinal target column */ /* TODO */

    double cluster_mean;              /* used to calculate outlier scores at prediction time */
    double cluster_sd;                /* used to calculate outlier scores at prediction time */
    std::vector<double> score_categ;  /* used to calculate outlier scores at prediction time */

    /* constructors in order to use C++'s vector emplace */

    /* full data (no conditions) */
    Cluster(ColType column_type, SplitType split_type)
    {
        this->column_type = column_type;
        this->split_type = split_type;
    }

    /* numerical split */
    Cluster(ColType column_type, size_t col_num, SplitType split_type, double split_point, bool has_NA_branch = false)
    {
        this->column_type = column_type;
        this->col_num = col_num;
        this->split_type = split_type;
        this->split_point = split_point;
        this->has_NA_branch = has_NA_branch;
    }

    /* categorical split */
    Cluster(ColType column_type, size_t col_num, SplitType split_type, char *split_subset, int ncat, bool has_NA_branch = false)
    {
        this->column_type = column_type;
        this->col_num = col_num;
        this->split_type = split_type;
        if (split_type != IsNa) this->split_subset.assign(split_subset, split_subset + ncat);
        this->split_subset.shrink_to_fit();
        this->has_NA_branch = has_NA_branch;
    }

    /* categorical split with only one level */
    Cluster(size_t col_num, int cat, int ncat, bool has_NA_branch = false)
    {
        this->column_type = Categorical;
        this->col_num = col_num;
        this->has_NA_branch = has_NA_branch;
        this->split_type = Equal;
        this->split_lev = cat;
    }

    /* ordinal split */
    Cluster(ColType column_type, size_t col_num, SplitType split_type, int split_lev, bool has_NA_branch = false)
    {
        this->column_type = column_type;
        this->col_num = col_num;
        this->split_type = split_type;
        this->split_lev = split_lev;
        this->has_NA_branch = has_NA_branch;
    }

    #ifdef Rcpp_hpp
    /* this is for serialization with Rcereal */
    template<class Archive>
    void serialize(Archive &archive)
    {
        archive(
                this->column_type,
                this->col_num,
                this->split_type,
                this->split_point,
                this->split_subset,
                this->split_lev,
                this->has_NA_branch,
                this->cluster_size,
                this->lower_lim,
                this->upper_lim,
                this->perc_below,
                this->perc_above,
                this->display_lim_low,
                this->display_lim_high,
                this->display_mean,
                this->display_sd,
                this->subset_common,
                this->perc_in_subset,
                this->perc_next_most_comm,
                this->cluster_mean,
                this->cluster_sd,
                this->score_categ
                );
    }
    Cluster() {};
    #endif
    
} Cluster;

/*    
*    Trees that host the aforementioned clusters. These work as follows:
*    - Each tree contains a split column and condition for splitting.
*    - The trees that follow them are specified in tree_left/right/NA.
*    - If the tree is dropped or not used, that branch gets an index of zero.
*    - The child tree will however remember which branch it took.
*    - At prediction time, the output will tell into which cluster and which tree
*      is each row an outlier (if they fall into any).
*    - The exact conditions are reconstructed by following the trees backwards
*      (i.e. first the cluster, then deepest tree, then follow parent tree until root).
*      This way, all the necessary information can be obtained without storing redundant
*      info, and without needing to reconstruct the conditions as the 'predict'
*      function is being called (which makes it easier to wrap into other languages).
*    - At prediction time, as the observation is passed down trees, all the clusters
*      in all those trees have to be tested for (so if a cluster is discarded, it can
*      keep only one branch of its split in the struct).
*    - As a side effect, in ordinal columns, the trees cannot be simplified to 'Equal'.
*    - All of this is ignored when using 'follow_all', in which case the trees work just
*      like the clusters, with an array 'all_branches' which contains all trees that have
*      to be follow from one particular tree.
*/
typedef struct ClusterTree {
    size_t parent = 0;              /* index in a vector */
    SplitType parent_branch = Root; /* this tree follows this branch in the split given by its parent */
    std::vector<size_t> clusters = std::vector<size_t>(); /* these clusters define additional splits */

    SplitType split_this_branch = Root;                        /* when using 'follow_all' */
    std::vector<size_t> all_branches = std::vector<size_t>();  /* when using 'follow_all' */

    ColType   column_type = NoType;
    size_t    col_num = 0;
    double    split_point = HUGE_VAL;
    std::vector<char> split_subset = std::vector<char>();
    int split_lev = INT_MAX;

    size_t tree_NA = 0;    /* binary splits */
    size_t tree_left = 0;  /* binary splits */
    size_t tree_right = 0; /* binary splits */
    std::vector<size_t> binary_branches = std::vector<size_t>(); /* multiple splits (single category or binarized categories) */

    ClusterTree(size_t parent, SplitType parent_branch)
    {
        this->parent = parent;
        this->parent_branch = parent_branch;
    }

    /* when using 'follow_all' */
    ClusterTree(size_t parent, size_t col_num, double split_point, SplitType split_this_branch)
    {
        this->parent = parent;
        this->col_num = col_num;
        this->column_type = Numeric;
        this->split_this_branch = split_this_branch;
        this->split_point = split_point;
    }

    ClusterTree(size_t parent, size_t col_num, int split_lev, SplitType split_this_branch)
    {
        this->parent = parent;
        this->col_num = col_num;
        this->column_type = Ordinal;
        this->split_this_branch = split_this_branch;
        this->split_lev = split_lev;
    }

    ClusterTree(size_t parent, size_t col_num, SplitType split_this_branch, char *split_subset, int ncat)
    {
        this->parent = parent;
        this->col_num = col_num;
        this->column_type = Categorical;
        if (split_this_branch != IsNa) {
            this->split_this_branch = split_this_branch;
            this->split_subset.assign(split_subset, split_subset + ncat);
            this->split_subset.shrink_to_fit();
        } else {
            this->split_this_branch = IsNa;
        }
    }

    ClusterTree(size_t parent, size_t col_num, int cat_chosen)
    {
        this->parent = parent;
        this->col_num = col_num;
        this->column_type = Categorical;
        this->split_this_branch = Equal;
        this->split_lev = cat_chosen;
    }

    #ifdef Rcpp_hpp
    /* this is for serialization with Rcereal */
    template<class Archive>
    void serialize(Archive &archive)
    {
        archive(
                this->parent,
                this->parent_branch,
                this->clusters,
                this->split_this_branch,
                this->all_branches,
                this->column_type,
                this->col_num,
                this->split_point,
                this->split_subset,
                this->split_lev,
                this->tree_NA,
                this->tree_left,
                this->tree_right,
                this->binary_branches
                );
    }
    ClusterTree() {};
    #endif

} ClusterTree;

/* these are needed for prediction time, and are thus returned from the function that fits the model */
typedef struct ModelOutputs {
    std::vector< std::vector<ClusterTree> > all_trees;  /* clusters in which observations can be outliers, required for prediction time */
    std::vector< std::vector<Cluster> > all_clusters;   /* decision trees that host the clusters, required for prediction time */
    std::vector<double> outlier_scores_final;   /* if an outlier is flagged, this indicates its score (lower is more outlier) as an upper probability bound */
    std::vector<size_t> outlier_clusters_final; /* if an outlier is flagged, this indicates the most suitable cluster in which to flag it as outlier */
    std::vector<size_t> outlier_columns_final;  /* if an outlier is flagged, this indicates the column that makes it an outlier */
    std::vector<size_t> outlier_trees_final;    /* if an outlier is flagged, this indicates the tree under which the cluster is found */
    std::vector<size_t> outlier_depth_final;    /* if an outlier is flagged, this indicates the split depth under which the cluster is found */
    std::vector<size_t> start_ix_cat_counts; /* this is to determine where to index the proportions */
    std::vector<long double> prop_categ;     /* this is just for statistics to show, it's not used for anything */
    std::vector<ColTransf> col_transf;       /* tells whether each numerical columns underwent log/exp transformations */
    std::vector<double> transf_offset;       /* value subtracted for log transform, mean subtracted for exp transform */
    std::vector<double> sd_div;              /* standard deviation with which exp-transformed columns were standardized */
    std::vector<int> ncat;      /* copied from the inputs, used to determine at prediction time if a category is out-of-range and skip */
    std::vector<int> ncat_ord;  /* copied from the inputs, used to determine at prediction time if a category is out-of-range and skip */
    size_t ncols_numeric;       /* copied from the inputs, used to determine at prediction time if a category is out-of-range and skip */
    size_t ncols_categ;         /* copied from the inputs, used to determine at prediction time if a category is out-of-range and skip */
    size_t ncols_ord;           /* copied from the inputs, used to determine at prediction time if a category is out-of-range and skip */
    std::vector<double> min_outlier_any_cl;             /* redundant info which speeds up prediction */
    std::vector<double> max_outlier_any_cl;             /* redundant info which speeds up prediction */
    std::vector<std::vector<bool>> cat_outlier_any_cl;  /* redundant info which speeds up prediction */
    size_t max_depth;                                   /* redundant info which speeds up prediction */


    #ifdef Rcpp_hpp
    /* this is for serialization with Rcereal */
    template<class Archive>
    void serialize(Archive &archive)
    {
        archive(
                this->all_trees,
                this->all_clusters,
                this->outlier_scores_final,
                this->outlier_clusters_final,
                this->outlier_columns_final,
                this->outlier_trees_final,
                this->outlier_depth_final,
                this->start_ix_cat_counts,
                this->prop_categ,
                this->col_transf,
                this->transf_offset,
                this->sd_div,
                this->ncat,
                this->ncat_ord,
                this->ncols_numeric,
                this->ncols_categ,
                this->ncols_ord,
                this->min_outlier_any_cl,
                this->max_outlier_any_cl,
                this->cat_outlier_any_cl,
                this->max_depth
                );
    }
    ModelOutputs() {};
    #endif

} ModelOutputs;

/*    
*    Note: the vectors with proportions in these structs are supposed to be all small numbers so 'long double' is an overkill for them
*    and does not make them translate into SIMD instructions in regular x86-64 CPUs, but if setting them as 'double' and then doing casts
*    from/between 'double' and the 'size_t' and 'long double's of other arrays (such as in function 'find_outlier_categories'), comparisons
*    such as '<=' will oftentimes fail even with small counts - this is an example that will fail when mixing the 3 types together:
*    >>>    (2 / (88+1)) * 0.5 <= (1 / 89) --> produces FALSE (right answer is TRUE)
*    All due to decimals (in that example) right of the 10th digit, and ends up creating categorical clusters that it should not create.
*    So don't change them back to regular 'double', or if necessary, change every 'long double' to 'double' too.
*/

/**************************************
  Prototypes from fit_model.cpp
  (This is the main module from which
   the model is generated)
***************************************/
bool fit_outliers_models(ModelOutputs &model_outputs,
                         double *restrict numeric_data,     size_t ncols_numeric,
                         int    *restrict categorical_data, size_t ncols_categ,   int *restrict ncat,
                         int    *restrict ordinal_data,     size_t ncols_ord,     int *restrict ncat_ord,
                         size_t nrows, char *restrict cols_ignore = NULL, int nthreads = 1,
                         bool categ_as_bin = true, bool ord_as_bin = true, bool cat_bruteforce_subset = false,
                         size_t max_depth = 3, double max_perc_outliers = 0.01, size_t min_size_numeric = 25, size_t min_size_categ = 75,
                         double min_gain = 1e-1, bool follow_all = false, double z_norm = 2.67, double z_outlier = 8.0);

typedef struct {
    
    std::vector<size_t> ix_arr;           /* indices from the target column */
    size_t st;                            /* chunk of the indices to take for current function calls */
    size_t end;                           /* chunk of the indices to take for current function calls */
    std::vector<double> outlier_scores;   /* these hold the model outputs for 1 column before combining them */
    std::vector<size_t> outlier_clusters; /* these hold the model outputs for 1 column before combining them */
    std::vector<size_t> outlier_trees;    /* these hold the model outputs for 1 column before combining them */
    std::vector<size_t> outlier_depth;    /* these hold the model outputs for 1 column before combining them */
    size_t target_col_num;                /* if categorical or ordinal, gets subtracted the number of numeric columns (used to index other arrays) */
    double sd_y;                          /* numerical only (standard deviation before splitting */
    long double base_info;                /* categorical and ordinal (information before splitting and before binarizing) */
    long double base_info_orig;           /* categorical and ordinal (information before splitting and after binarizing if needed) */
    bool log_transf;                      /* numerical - whether the target variable underwent a logarithmic transformation */
    bool exp_transf;                      /* numerical - whether the target variable underwent exponentiation on its Z values */
    double *target_numeric_col;           /* dynamic pointer */
    int    *target_categ_col;             /* dynamic pointer */
    std::vector<double> buffer_transf_y;  /* if applying logarithm or exponentiation, transformed values are stored here */
    std::vector<int> buffer_bin_y;        /* if binarizing, transformed values are stored here */
    std::vector<Cluster>     *clusters;   /* dynamic pointer, don't change to reference as it otherwise cannot be reassigned */
    std::vector<ClusterTree> *tree;       /* dynamic pointer, don't change to reference as it otherwise cannot be reassigned */
    bool   has_outliers;                  /* temporary variable from which the other two are updated */
    bool   lev_has_outliers;              /* whether the particular depth level has outliers (if so, wil remove them at the end before new split) */
    bool   col_has_outliers;              /* whether there's any outliers in the column (will later merge them into the outputs) */
    double left_tail;                     /* approximate value where a long left tail ends */
    double right_tail;                    /* approximate value where a long right tail ends */

    bool col_is_bin;                      /* whether the target categorical/ordinal column has 2 categories or has been forcibly binarized */
    long double *prop_small_this;         /* dynamic pointer */

    double orig_mean;                     /* value to reconstruct originals from exponentiated */
    double orig_sd;                       /* value to reconstruct originals from exponentiated */
    double log_minval;                    /* value to reconstruct originals from logarithms */
    double *orig_target_col;              /* column as it was before applying log/exp (dynamic pointer) */
    int *untransf_target_col;             /* column as it was before forcibly binarizing (dynamic pointer) */
    int *temp_ptr_x;                      /* dynamic pointer */

    std::vector<char> buffer_subset_categ_best;  /* categorical split that gave the best gain */
    double this_gain;                            /* buffer where to store gain */
    double this_split_point;                     /* numeric split threshold */
    int this_split_lev;                          /* ordinal split threshold */
    size_t this_split_ix;                        /* index at which the data is partitioned */
    size_t this_split_NA;                        /* index at which the non-NA values start */
    double best_gain;                            /* buffer where to store the info of the splitting column that produced the highest gain */
    ColType column_type_best;                    /* buffer where to store the info of the splitting column that produced the highest gain */
    double split_point_best;                     /* buffer where to store the info of the splitting column that produced the highest gain */
    int split_lev_best;                          /* buffer where to store the info of the splitting column that produced the highest gain */
    size_t col_best;                             /* buffer where to store the info of the splitting column that produced the highest gain */

    std::vector<long double> buffer_cat_sum;         /* buffer arrays where to allocate values required by functions and not used outside them */
    std::vector<long double> buffer_cat_sum_sq;      /* buffer arrays where to allocate values required by functions and not used outside them */
    std::vector<size_t>      buffer_crosstab;        /* buffer arrays where to allocate values required by functions and not used outside them */
    std::vector<size_t>      buffer_cat_cnt;         /* buffer arrays where to allocate values required by functions and not used outside them */
    std::vector<size_t>      buffer_cat_sorted;      /* buffer arrays where to allocate values required by functions and not used outside them */
    std::vector<char>        buffer_subset_categ;    /* buffer arrays where to allocate values required by functions and not used outside them */
    std::vector<char>        buffer_subset_outlier;  /* buffer arrays where to allocate values required by functions and not used outside them */
    
    bool drop_cluster;          /* for categorical and ordinal variables, not all clusters can flag observations as outliers, so those are not kept */
    bool already_split_main;    /* when binarizing categoricals/ordinals, avoid attempting the same split with numerical and ordinals that take the non-binarized data */
    bool target_col_is_ord;     /* whether the target column is ordinal (rest is the same as in categoricals) */
    int  ncat_this;             /* number of categories in the target column */

} Workspace;

/* info holders to shorten function call arguments */
typedef struct {
    bool categ_as_bin;
    bool ord_as_bin;
    bool cat_bruteforce_subset;
    size_t max_depth;
    double max_perc_outliers;
    size_t min_size_numeric;
    size_t min_size_categ;
    double min_gain;
    bool follow_all;
    double z_norm;
    double z_outlier;
    double z_tail;
    std::vector<long double> prop_small; /* this is not a parameter, but a shared array determined from the parameters and data */
} ModelParams;

/* Note: the vectors here are filled within the function that fits the model, while the pointers are passed from outside */
typedef struct {
    double  *restrict numeric_data;     size_t ncols_numeric;
    int     *restrict categorical_data; size_t ncols_categ;   int *restrict ncat;
    int     *restrict ordinal_data;     size_t ncols_ord;     int *restrict ncat_ord;
    size_t  nrows; size_t tot_cols; std::vector<char> has_NA; std::vector<char> skip_col; int max_categ;
    std::vector<size_t> cat_counts;
} InputData;


void process_numeric_col(std::vector<Cluster> &cluster_root,
                         std::vector<ClusterTree> &tree_root,
                         size_t target_col_num,
                         Workspace &workspace,
                         InputData &input_data,
                         ModelParams &model_params,
                         ModelOutputs &model_outputs);
void recursive_split_numeric(Workspace &workspace,
                             InputData &input_data,
                             ModelParams &model_params,
                             size_t curr_depth, bool is_NA_branch);
void process_categ_col(std::vector<Cluster> &cluster_root,
                       std::vector<ClusterTree> &tree_root,
                       size_t target_col_num, bool is_ord,
                       Workspace &workspace,
                       InputData &input_data,
                       ModelParams &model_params,
                       ModelOutputs &model_outputs);
void recursive_split_categ(Workspace &workspace,
                           InputData &input_data,
                           ModelParams &model_params,
                           size_t curr_depth, bool is_NA_branch);


/****************************************
  Prototypes from predict.cpp
  (This is the module from which
   new data can be flagged as outliers)
*****************************************/
typedef struct {
    double  *restrict numeric_data;
    int     *restrict categorical_data;
    int     *restrict ordinal_data;
    size_t nrows;
} PredictionData;

bool find_new_outliers(double *restrict numeric_data,
                       int    *restrict categorical_data,
                       int    *restrict ordinal_data,
                       size_t nrows, int nthreads, ModelOutputs &model_outputs);
bool follow_tree(ModelOutputs &model_outputs, PredictionData &prediction_data, size_t curr_tree, size_t curr_depth,
                 size_t_for row, size_t_for col, bool col_is_num, double num_val_this, int cat_val_this);
bool check_is_outlier_in_tree(std::vector<size_t> &clusters_in_tree, size_t curr_depth, size_t curr_tree,
                              ModelOutputs &model_outputs, PredictionData &prediction_data, size_t_for row, size_t_for col,
                              bool col_is_num, double num_val_this, int cat_val_this);


/****************************
  Prototypes from split.cpp
****************************/
#define SD_REG 1e-5 /* Regularization for standard deviation estimation */

typedef struct {
    size_t      cnt;
    long double sum;
    long double sum_sq;
} NumericBranch;

typedef struct {
    NumericBranch NA_branch    = {0, 0, 0};
    NumericBranch left_branch  = {0, 0, 0};
    NumericBranch right_branch = {0, 0, 0};
} NumericSplit;

typedef struct {
    size_t *restrict NA_branch;     /* array of counts of the target variable's categories */
    size_t *restrict left_branch;   /* array of counts of the target variable's categories */
    size_t *restrict right_branch;  /* array of counts of the target variable's categories */
    size_t ncat;                    /* number of categories/entries in the arrays above */
    size_t tot;                        /* size_NA +  size_left + size_right */
    size_t size_NA    = 0;
    size_t size_left  = 0;
    size_t size_right = 0;
} CategSplit;

void subset_to_onehot(size_t ix_arr[], size_t n_true, size_t n_tot, bool onehot[]);
size_t move_zero_count_to_front(size_t *restrict cat_sorted, size_t *restrict cat_cnt, size_t ncat_x);
void flag_zero_counts(char split_subset[], size_t buffer_cat_cnt[], size_t ncat_x);
double calc_sd(size_t cnt, long double sum, long double sum_sq);
double calc_sd(NumericBranch &branch);
double calc_sd(size_t ix_arr[], double x[], size_t st, size_t end);
double numeric_gain(NumericSplit &split_info, double tot_sd);
long double total_info(size_t categ_counts[], size_t ncat);
long double total_info(size_t categ_counts[], size_t ncat, size_t tot);
long double total_info(size_t *restrict ix_arr, int *restrict x, size_t st, size_t end, size_t ncat, size_t *restrict buffer_cat_cnt);
double categ_gain(CategSplit split_info, long double base_info);
double categ_gain(size_t *restrict categ_counts, size_t ncat, size_t *restrict ncat_col, size_t maxcat, long double base_info, size_t tot);
double categ_gain_from_split(size_t *restrict ix_arr, int *restrict x, size_t st, size_t st_non_na, size_t split_ix, size_t end,
                             size_t ncat, size_t *restrict buffer_cat_cnt, long double base_info);
void split_numericx_numericy(size_t *restrict ix_arr, size_t st, size_t end, double *restrict x, double *restrict y,
                             double sd_y, bool has_na, size_t min_size,
                             double *restrict gain, double *restrict split_point, size_t *restrict split_left, size_t *restrict split_NA);
void split_categx_numericy(size_t *restrict ix_arr, size_t st, size_t end, int *restrict x, double *restrict y, double sd_y,
                           bool x_is_ordinal, size_t ncat_x, size_t *restrict buffer_cat_cnt, long double *restrict buffer_cat_sum,
                           long double *restrict buffer_cat_sum_sq, size_t *restrict buffer_cat_sorted,
                           bool has_na, size_t min_size, double *gain, char *restrict split_subset, int *restrict split_point);
void split_numericx_categy(size_t *restrict ix_arr, size_t st, size_t end, double *restrict x, int *restrict y,
                           size_t ncat_y, long double base_info, size_t *restrict buffer_cat_cnt,
                           bool has_na, size_t min_size, double *restrict gain, double *restrict split_point,
                           size_t *restrict split_left, size_t *restrict split_NA);
void split_ordx_categy(size_t *restrict ix_arr, size_t st, size_t end, int *restrict x, int *restrict y,
                       size_t ncat_y, size_t ncat_x, long double base_info,
                       size_t *restrict buffer_cat_cnt, size_t *restrict buffer_crosstab, size_t *restrict buffer_ord_cnt,
                       bool has_na, size_t min_size, double *gain, int *split_point);
void split_categx_biny(size_t *restrict ix_arr, size_t st, size_t end, int *restrict x, int *restrict y,
                       size_t ncat_x, long double base_info,
                       size_t *restrict buffer_cat_cnt, size_t *restrict buffer_crosstab, size_t *restrict buffer_cat_sorted,
                       bool has_na, size_t min_size, double *gain, char *restrict split_subset);
void split_categx_categy_separate(size_t *restrict ix_arr, size_t st, size_t end, int *restrict x, int *restrict y,
                                  size_t ncat_x, size_t ncat_y, long double base_info,
                                  size_t *restrict buffer_cat_cnt, size_t *restrict buffer_crosstab,
                                  bool has_na, size_t min_size, double *gain);
void split_categx_categy_subset(size_t *restrict ix_arr, size_t st, size_t end, int *restrict x, int *restrict y,
                                size_t ncat_x, size_t ncat_y, long double base_info,
                                size_t *restrict buffer_cat_cnt, size_t *restrict buffer_crosstab, size_t *restrict buffer_split,
                                bool has_na, size_t min_size, double *gain, char *restrict split_subset);



/*******************************
  Prototypes from clusters.cpp
********************************/
#define calculate_max_outliers(n, perc) (  (n) * (perc) + (long double)2 * sqrtl( (n) * (perc) * ((long double)1 - perc) ) + (long double)1  )
#define z_score(x, mu, sd) (  ((x) - (mu)) / (sd)  )
#define chebyshyov_bound(sd) (1.0 / square(sd))

bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_t st, size_t end,
                              double *restrict outlier_scores, size_t *restrict outlier_clusters, size_t *restrict outlier_trees,
                              size_t *restrict outlier_depth, Cluster &cluster, std::vector<Cluster> &clusters, size_t cluster_num, size_t tree_num, size_t tree_depth,
                              bool is_log_transf, double log_minval, bool is_exp_transf, double orig_mean, double orig_sd,
                              double left_tail, double right_tail, double *restrict orig_x,
                              double max_perc_outliers, double z_norm, double z_outlier);
void define_categ_cluster_no_cond(int *restrict x, size_t *restrict ix_arr, size_t st, size_t end, size_t ncateg,
                                  double *restrict outlier_scores, size_t *restrict outlier_clusters, size_t *restrict outlier_trees,
                                  size_t *restrict outlier_depth, Cluster &cluster,
                                  size_t *restrict categ_counts, char *restrict is_outlier, double perc_next_most_comm);
bool define_categ_cluster(int *restrict x, size_t *restrict ix_arr, size_t st, size_t end, size_t ncateg,
                          double *restrict outlier_scores, size_t *restrict outlier_clusters, size_t *restrict outlier_trees,
                          size_t *restrict outlier_depth, Cluster &cluster, std::vector<Cluster> &clusters, size_t cluster_num, size_t tree_num, size_t tree_depth,
                          double max_perc_outliers, double z_norm, long double *restrict perc_threshold,
                          size_t *restrict buffer_categ_counts, long double *restrict buffer_categ_pct,
                          size_t *restrict buffer_categ_ix, char *restrict buffer_outliers, bool *restrict drop_cluster);
void simplify_when_equal_cond(std::vector<Cluster> &clusters, int ncat_ord[]);
void simplify_when_equal_cond(std::vector<ClusterTree> &trees, int ncat_ord[]);
#ifdef TEST_MODE_DEFINE
void prune_unused_trees(std::vector<ClusterTree> &trees);
#endif
bool check_tree_is_not_needed(ClusterTree &tree);
void calculate_cluster_minimums(ModelOutputs &model_outputs, size_t col);
void calculate_cluster_poss_categs(ModelOutputs &model_outputs, size_t col, size_t col_rel);


/**********************************
  Prototypes from cat_outlier.cpp
***********************************/
#define calculate_max_cat_outliers(n, perc, z_norm) ((long double)1 + ((n) * (perc) / z_norm)) /* Note: this is not anyhow probabilistic, nor based on provable bounds */
void find_outlier_categories(size_t categ_counts[], size_t ncateg, size_t tot, double max_perc_outliers,
                             long double perc_threshold[], size_t buffer_ix[], long double buffer_perc[],
                             double z_norm, char is_outlier[], bool *found_outliers, bool *new_is_outlier, double *next_most_comm);
bool find_outlier_categories_no_cond(size_t categ_counts[], size_t ncateg, size_t tot,
                                     char is_outlier[], double *next_most_comm);



/*********************************************
  Prototypes from misc.cpp and other structs
**********************************************/

/* an inefficient workaround for coding up option 'follow_all' */
typedef struct {
    double gain_restore;
    double gain_best_restore;
    double split_point_restore;
    int    split_lev_restore;
    std::vector<char> split_subset_restore;
    size_t ix1_restore;
    size_t ix2_restore;
    size_t ix3_restore;
    size_t ix4_restore;
    int *  temp_ptr_x;
    size_t col_best_restore;
    ColType col_type_best_rememer;
    double split_point_best_restore;
    int    split_lev_best_restore;
    std::vector<char> split_subset_best_restore;
    long double base_info_restore;
    long double base_info_orig_restore;
    double sd_y_restore;
    bool has_outliers_restore;
    bool lev_has_outliers_restore;
} RecursionState;


int calculate_category_indices(size_t start_ix_cat_counts[], int ncat[], size_t ncols, bool skip_col[], int max_categ = 0);
void calculate_all_cat_counts(size_t start_ix_cat_counts[], size_t cat_counts[], int ncat[],
                              int categorical_data[], size_t ncols, size_t nrows,
                              bool has_NA[], bool skip_col[], int nthreads);
void check_cat_col_unsplittable(size_t start_ix_cat_counts[], size_t cat_counts[], int ncat[],
                                size_t ncols, size_t min_conditioned_size, size_t nrows, bool skip_col[], int nthreads);
void calculate_lowerlim_proportion(long double *restrict prop_small, long double *restrict prop,
                                   size_t start_ix_cat_counts[], size_t cat_counts[],
                                   size_t ncols, size_t nrows, double z_norm, double z_tail);
void check_missing_no_variance(double numeric_data[], size_t ncols, size_t nrows, bool has_NA[], bool skip_col[], int nthreads);
void calc_central_mean_and_sd(size_t ix_arr[], size_t st, size_t end, double x[], size_t size_quarter, double *mean_central, double *sd_central);
void check_for_tails(size_t ix_arr[], size_t st, size_t end, double *restrict x,
                     double z_norm, double max_perc_outliers,
                     double *restrict buffer_x, double mean, double sd,
                     double *restrict left_tail, double *restrict right_tail,
                     bool *exp_transf, bool *log_transf);
size_t move_outliers_to_front(size_t ix_arr[], double outlier_scores[], size_t st, size_t end);
size_t move_NAs_to_front(size_t ix_arr[], double x[], size_t st, size_t end, bool inf_as_NA);
size_t move_NAs_to_front(size_t ix_arr[], int x[], size_t st, size_t end);
void divide_subset_split(size_t ix_arr[], double x[], size_t st, size_t end, double split_point, bool has_NA, size_t *split_NA, size_t *st_right);
void divide_subset_split(size_t ix_arr[], int x[], size_t st, size_t end, char subset_categ[], int ncat, bool has_NA, size_t *split_NA, size_t *st_right);
void divide_subset_split(size_t ix_arr[], int x[], size_t st, size_t end, int split_lev, bool has_NA, size_t *split_NA, size_t *st_right);
bool check_workspace_is_allocated(Workspace &workspace);
void allocate_thread_workspace(Workspace &workspace, size_t nrows, int max_categ);
void backup_recursion_state(Workspace &workspace, RecursionState &state_backup);
void restore_recursion_state(Workspace &workspace, RecursionState &state_backup);
void set_tree_as_numeric(ClusterTree &tree, double split_point, size_t col);
void set_tree_as_categorical(ClusterTree &tree, int ncat, char *split_subset, size_t col);
void set_tree_as_categorical(ClusterTree &tree, size_t col);
void set_tree_as_categorical(ClusterTree &tree, size_t col, int ncat);
void set_tree_as_ordinal(ClusterTree &tree, int split_lev, size_t col);
void forget_row_outputs(ModelOutputs &model_outputs);
void allocate_row_outputs(ModelOutputs &model_outputs, size_t nrows, size_t max_depth);
