import pandas as pd
import numpy as np
import os
from pandas.core.dtypes.common import is_numeric_dtype, is_integer_dtype
from scipy.special import legendre


def check_cat(col, cat):
    """
    Warns the user if he entered an invalid category for a column
    :param col: the column with the invalid category, will be either a string or a number depending on the user's input
    :param cat: the category of the column, given by the user
    :return:
    """
    if cat not in ['cross', 'cov']:
        raise ValueError('Invalid effect type for effect', col, '- should be one of \'cross\' or \'cov\'')


def save_matrix(name, sym=True):
    """
    Saves a matrix read from a file generated by BLUPF90 suite of programs as a numpy array
    The implementation determines the size of the matrix based on the values from the file. Given that the matrices
    used in our models are square and that they are invertible and thus can't have a line with only zero values, it
    means that the maximum index found as a line or column number in the file is also the size of the matrix
    :param name: the name of the file from which the matrix is read
    :param sym: whether the file from which the matrix is read stores it in a lower/upper triangular manner or not
    (generally, this is True, as most of our matrices of interest are symmetric)
    :return:
    """
    size = 0
    with open(name) as f:
        line = f.readline()
        while line:
            values = line.strip().split()
            size = max(size, int(values[0]), int(values[1]))
            line = f.readline()
    rez = np.zeros((size, size))
    with open(name) as f:
        line = f.readline()
        while line:
            values = line.strip().split()
            x, y, z = int(values[0]), int(values[1]), float(values[2])
            rez[x - 1, y - 1] = z
            if sym:
                rez[y - 1, x - 1] = z
            line = f.readline()
    return rez


def switch_to_no_inbreeding():
    """
    Used if inbreeding=False in a Renum object. Changes the line in renf90.par containing the string "add_an_upginb"
    to "add_animal" in order to not account for inbreeding
    :return:
    """

    with open('renf90.par') as f:
        renf_lines = f.readlines()
    new_renf_lines = list(map(lambda x: x.replace('add_an_upginb', 'add_animal'), renf_lines))
    with open('renf90.par', 'w') as f:
        f.writelines(new_renf_lines)


def remove_save_lines():
    """
    Removes saving options line from the parameter file, so they are not needlessly executed again
    :return: None
    """
    with open('renf90.par') as f:
        file_lines = f.readlines()
        no_save_lines = list(filter(lambda x: 'OPTION save' not in x, file_lines))
    with open('renf90.par', 'w') as f:
        f.writelines(no_save_lines)


class Renum:
    def __init__(self, data, animal_col, ped, inbreeding=False, genomic_data=None, use_blupf90_modules=True,
                 make_blupf90_files=False, trait_cols=None, fixed_effects=None, fixed_degree=None, random_degree=None,
                 lactation_col=None, dim_col=None, dim_range=None, res_variance=None, ag_variance=None,
                 pe_variance=None, export_A=False, export_Ainv=False, export_G=False, export_Ginv=False,
                 export_Hinv=False, export_A22=False, export_A22inv=False):
        """
        Class used to implement the renumbering and reordering of pedigree. For now, it only supports a renumf90
        wrapper
        :param data: dataframe containing animals' phenotypic data
        :param animal_col: the column in data containing the ids of the animal - can be given as a string or as a number
        :param ped: dataframe containing animals' pedigree data - it is mandatory that it has 3 columns, the first one
        being the animals' ID, the second one the ID of the sire and the third one the ID of the dam
        :param inbreeding: boolean parameter which tells whether inbreeding should be accounted for or not
        :param genomic_data: dataframe containing animals' genotypic data, if available
        :param use_blupf90_modules: whether or not to use renumf90 wrapper
        :param make_blupf90_files: unused yet, if True, a native implementation would generate files similar to those
        generated by renumf90
        :param trait_cols: the columns in data associated to traits - the columns are given as an Iterable object and
        can be either strings or numbers
        :param fixed_effects: the fixed effects, which are given as an Iterable. Fixed effects can refer to one column
        and can be either categorical or covariate, or refer to concatenation of multiple columns (which will be
        considered categorical). They can also be nested, either one column is nested based on another one, or each
        column for a concatenated effect is nested based on one column
        :param fixed_degree: used for test day model, represents the degree of the fixed Legendre polynomial
        :param random_degree: used for test day model, represents the degree of the random Legendre polynomial
        :param lactation_col: used for test day model, represents the column in data which contains the lactation for
        each record, can be given as a string or as a number
        :param dim_col: used for test day model, represents the column in data which contains the DIM for each record,
        can be given as a string or as a number
        :param dim_range: used for test day model, a pair of numbers specifying the DIM interval (for example, (5, 310))
        :param res_variance: initial residual variance, if it exists
        :param ag_variance: initial additive genetic variance, if it exists
        :param pe_variance: initial permanent genetic variance, if it exists
        :param export_A: unused yet, determines whether the A matrix should be saved or not
        :param export_Ainv: unused yet, determines whether the inverse of the A matrix should be saved or not
        :param export_G: unused yet, determines whether the G matrix should be saved or not
        :param export_Ginv: unused yet, determines whether the inverse of the G matrix should be saved or not
        :param export_Hinv: unused yet, determines whether the inverse of the H matrix should be saved or not
        :param export_A22: unused yet, determines whether the A22 matrix should be saved or not
        :param export_A22inv: unused yet, determines whether the inverse of the A22 matrix should be saved or not
        """
        self.data = data
        self.trait_cols = trait_cols
        self.traits_count = len(trait_cols)

        # Ensure that fixed_effects is an Interable with size equal to the number of fixed_effects
        self.fixed_effects = [] if fixed_effects is None else fixed_effects
        self.fixed_count = len(self.fixed_effects)

        # Fixed and random degree are -1 if not given in order to follow the convention that they should be at least
        # equal to 0 if given
        self.fixed_degree = -1 if fixed_degree is None else fixed_degree
        self.random_degree = -1 if random_degree is None else random_degree

        # Maximum between fixed degree and random degree, useful for knowing how many columns based on Legendre
        # polynomials should be created
        self.degree = max(self.fixed_degree, self.random_degree)
        self.res_variance = res_variance
        self.new_data = pd.DataFrame()

        # Variable used for counting how many nested effects have been added in the renumf90 file at a given moment
        # It is needed in order to know what values to write in the renumf90 file and also how to order columns for the
        # phenotypes data which will be given to renumf90
        self.nested_count = 0

        # Indices of traits that will be used in order to create the renumf90 file and will also determine the positions
        # of the columns associated to each trait in the phenotypic data given to renumf90
        self.trait_cols_idx = []
        self.lactation_col = lactation_col
        self.dim_col = dim_col
        self.dim_range = dim_range

        # Converts the animal ID column to a 1-indexed integer position
        self.animal_col = self.__get_col__(animal_col)

        # True if permanent effects will be needed (for test day or repeatability model), False otherwise
        self.has_perm = False
        self.ped = ped
        self.new_ped = None
        self.genomic_data = genomic_data
        self.ag_variance = ag_variance
        self.pe_variance = pe_variance
        self.inbreeding = inbreeding
        self.snp_data = None
        self.A = None
        self.Ainv = None
        self.G = None
        self.Ginv = None
        self.Hinv = None
        self.A22 = None
        self.A22inv = None
        self.animal_to_code = {}
        self.code_to_animal = {}
        self.sires = None
        self.dams = None
        self.export_A = export_A
        self.export_Ainv = export_Ainv
        self.export_A22 = export_A22
        self.export_A22inv = export_A22inv
        self.export_G = export_G
        self.export_Ginv = export_Ginv
        self.export_Hinv = export_Hinv
        self.export_genomic = (self.export_A22 or self.export_A22inv or self.export_G or self.export_Ginv
                               or self.export_Hinv)

        if use_blupf90_modules:
            # Stores the lines that will be written in the renumf90 file
            self.file_lines = ['DATAFILE\n', 'pheno.txt\n', 'TRAITS\n']

            # It is not allowed to have a valid fixed_degree or random_degree, while not providing at least one of the
            # lactation column or the DIM column. At the same time, both the lactation column and the DIM column should
            # be provided, or none of them should be
            if (self.fixed_degree >= 0 or self.random_degree >= 0) and (lactation_col is None or dim_col is None):
                raise ValueError("Need both a lactation column and a DIM column in order to use Legendre polynomials")
            if (lactation_col is not None and dim_col is None) or (lactation_col is None and dim_col is not None):
                raise ValueError("Can't have a lactation column without a DIM column or viceversa")

            self.__check_traits__()
            self.__check_lactation_col__()
            self.__check_dim_col__()
            self.__check_pure_gblup__()
            self.__add_traits_line__()
            self.file_lines.append('FIELDS_PASSED TO OUTPUT\n')
            self.file_lines.append('\n')
            self.file_lines.append('WEIGHT(S)\n')
            self.file_lines.append('\n')
            self.file_lines.append('RESIDUAL_VARIANCE\n')
            self.__add_res_variance__()
            self.__add_fixed_effects__()
            self.__add_legendre_polynomials__()
            self.__add_traits_to_renumf90_pheno__()
            self.__add_animal_effect__()
            self.__add_perm_effect__()
            self.__add_pedigree__()
            self.__add_inbreeding__()
            self.__add_random_regression__()
            self.__add_variances__()
            self.__add_size_constraint__()

            self.new_data.astype(str).replace('0.0', '0').to_csv('pheno.txt', sep=' ', header=False, index=False)

            # We will compute A using preGSf90. The approach will be to actually compute an A22 matrix which will be
            # made by all the animals. In order to do this, we need to know which animals will be kept by renumbering,
            # in order to have consistent results. Thus, we will run renumf90 once and use the renaddXX.ped file to
            # determine which animals are part of the renumbered pedigree. Then, we will build a dummy renumf90 file
            # using dummy SNPs for each animal in the pedigree and running preGSf90 will compute the A matrix
            if ped is not None and (self.export_A or self.export_Ainv):
                with open('genrenf90.par', 'w') as f:
                    f.writelines(self.file_lines)
                os.system('renumf90 genrenf90.par')
                self.dummy_lines.extend(['SNP_FILE\n', 'dummy_snps.txt\n', 'OPTION no_quality_control\n',
                                         'OPTION saveAscii\n'])
                if self.export_A:
                    self.dummy_lines.append('OPTION saveA22\n')
                if self.export_Ainv:
                    self.dummy_lines.append('OPTION saveA22Inverse\n')
                self.__add_animal_codes__()
                self.__add_dummy_snps__()
                with open('genrenf90.par', 'w') as f:
                    f.writelines(self.dummy_lines)
                os.system('renumf90 genrenf90.par')
                if not inbreeding:
                    switch_to_no_inbreeding()
                os.system('preGSf90 renf90.par')

                if self.export_A:
                    self.__save_A__()
                if self.export_Ainv:
                    self.__save_Ainv__()

            # Finally, we add actual genomic data in the renumf90 files if data is available and then we run the final
            # iteration of renumf90.
            self.__add_genomic__()
            with open('genrenf90.par', 'w') as f:
                f.writelines(self.file_lines)
            if self.genomic_data is None or (not self.export_genomic):
                os.system('renumf90 genrenf90.par')
                if not inbreeding:
                    switch_to_no_inbreeding()
            if not self.animal_to_code:
                self.__add_animal_codes__()

            if self.ped is not None:
                self.new_ped = self.ped.copy()[self.ped.iloc[:, 0].isin(self.animal_to_code)]
                self.new_ped.iloc[:, 0] = self.ped.iloc[:, 0].map(self.animal_to_code)

            remove_save_lines()

            self.__read_inbreeding__()

    def __get_col__(self, col):
        """
        Returns a 1-indexed value for a given column
        :param col: the column to be converted, can be either a string or an index
        :return:
        """
        if type(col) == str:
            return self.data.columns.get_loc(col) + 1
        return col

    def __check_traits__(self):
        """
        Checking that each trait column contains only numeric values
        :return: None
        """
        for col in self.trait_cols:
            mod_col = self.__get_col__(col)
            if not is_numeric_dtype(self.data.iloc[:, mod_col - 1]):
                raise ValueError("Trait", col, "is not numeric")
            self.trait_cols_idx.append(mod_col)

    def __check_lactation_col__(self):
        """
        Ensuring that the lactation column only has strictly positive integer values, if it exists. Sets the maximum
        lactation number, by convention it is considered to be 1 when there are no actual lactations used (when animal
        model is used)
        :return: None
        """
        if self.lactation_col is not None:
            mod_col = self.__get_col__(self.lactation_col)
            if not is_integer_dtype(self.data.iloc[:, mod_col - 1]):
                raise ValueError("Lactation column should only have integer values")
            if self.data.iloc[:, mod_col - 1].min() <= 0:
                raise ValueError("Lactation column values should be higher than 0")
            self.lactation_dim = self.data.iloc[:, mod_col - 1].max()
        else:
            self.lactation_dim = 1

    def __check_dim_col__(self):
        """
        If DIM col exists, it is ensured that it only has positive integer values. It is also checked that the DIM range
        is valid (as in the first element is smaller than the second one)
        :return: None
        """
        if self.dim_col is not None:
            mod_col = self.__get_col__(self.dim_col)
            if not is_integer_dtype(self.data.iloc[:, mod_col - 1]):
                raise ValueError("DIM column should only have integer values")
            data_col = self.data.iloc[:, mod_col - 1]
            if data_col.min() <= 0:
                raise ValueError("DIM column values should be higher than 0")
            if self.dim_range is None:
                self.dim_range = (data_col.min(), data_col.max())
            else:
                if self.dim_range[0] > self.dim_range[1]:
                    raise ValueError("DIM range should have the first element smaller than the second one")

    def __add_traits_line__(self):
        """
        Add traits in the renumf90 file: the indices of the traits will be the ones coming after the fixed effects and
        the legendre polynomials. We also have to take into account the fact that the number of actual traits given to
        renumf90 will be multiplied by the number of lactations in our data
        :return: None
        """
        self.file_lines.append(' '.join(map(str, range(self.fixed_count + self.degree + 2, self.fixed_count +
                                                       self.degree + 1 + self.traits_count * self.lactation_dim
                                                       + 1))) + '\n')

    def __add_res_variance__(self):
        """
        If initial residual variance is given, than it is checked that it only contains numerical values and that it
        has the expected shape. Otherwise, the default residual variance with 1 on the main diagonal and 0.1 outside of
        it is given to renumf90
        :return:
        """
        if self.res_variance is None:
            self.file_lines.extend([' '.join(map(str, [0.1] * i + [1.0] + [0.1]
                                                 * (self.traits_count * self.lactation_dim - i - 1))) + '\n' for i in
                                    range(self.traits_count * self.lactation_dim)])
        else:
            if self.res_variance.shape != (self.traits_count * self.lactation_dim,
                                           self.traits_count * self.lactation_dim):
                raise ValueError('Invalid residual variance dimensions: should be',
                                 (self.traits_count * self.lactation_dim, self.traits_count * self.lactation_dim),
                                 'but they are instead', self.res_variance.shape)
            if not is_numeric_dtype(self.res_variance):
                raise ValueError('Residual variance should only contain numerical values')
            self.file_lines.extend([' '.join(map(str, row)) + '\n' for row in self.res_variance])

    def __add_fixed_effects__(self):
        count = 1
        nested_count = 1
        for effect in self.fixed_effects:
            if len(effect) == 1:
                # If len is 1, it means we have an Iterable with categorical effects which should be concatenated into
                # one fixed effect
                all_cols = [self.__get_col__(effect_col) - 1 for effect_col in effect[0]]
                self.new_data[count] = self.data.iloc[:, all_cols].astype(str).apply('_'.join, axis=1)
                self.file_lines.extend((['EFFECT\n', (str(count) + ' ') * self.traits_count * self.lactation_dim
                                         + 'cross alpha\n']))
            if len(effect) == 2:
                # If len is 2, we can either have a list of effects to be concatenated, each effect being nested, or
                # one effect, given by the column and its category. However, we need to check that this is not a string,
                # as that can also be an Iterable
                if hasattr(effect[0], '__iter__') and type(effect[0]) != str:
                    # If the first element is an Iterable, then we should also check that the second element is an
                    # Iterable and if it is, we should also make sure that it has the same length with the first
                    # element. If these conditions are met, then all columns, main and nested, are converted into one
                    # fixed categorical effect. If the second element is not an Iterable, it should be a column which
                    # nests all the other columns in the first element. All columns and the nested one will be
                    # concatenated into one fixed effect
                    tmp_data = pd.DataFrame()
                    if hasattr(effect[1], '__iter__'):
                        if len(effect[0]) != len(effect[1]):
                            raise ValueError('Number of columns to combine and number of nested columns if more than',
                                             'one should be the same')
                        idx = 0
                        for col, nested in zip(effect[0], effect[1]):
                            mod_col = self.__get_col__(col)
                            mod_nested = self.__get_col__(nested)
                            tmp_data[idx] = self.data.iloc[:, [mod_col, mod_nested]].astype(str).apply('_'.join, axis=1)
                            idx += 1
                    else:
                        all_cols = [self.__get_col__(effect_col) - 1 for effect_col in effect[0]]
                        tmp_data[0] = self.data.iloc[:, all_cols].astype(str).apply('_'.join, axis=1)
                        tmp_data[1] = self.data.iloc[:, self.__get_col__(effect[1]) - 1]
                    self.new_data[count] = tmp_data.apply('_'.join, axis=1)
                    self.file_lines.extend((['EFFECT\n', (str(count) + ' ') * self.traits_count * self.lactation_dim
                                             + 'cross alpha\n']))
                else:
                    # For the case when the first element is not an Iterable, we must have a normal fixed effect given
                    # by column and category. We check that the category is valid and we also perform additional checks
                    # such as verifying that the column contains only numerical values if cat is 'cov' (covariate)
                    col, cat = effect
                    check_cat(col, cat)
                    mod_col = self.__get_col__(col)
                    if is_numeric_dtype(self.data.iloc[:, mod_col - 1]):
                        self.file_lines.extend((['EFFECT\n', (str(count) + ' ') * self.traits_count
                                                 * self.lactation_dim + cat + (' numer' if cat == 'cross' else '')
                                                 + '\n']))
                    else:
                        if cat == 'cov':
                            raise ValueError('Covariate effect', col, 'has non-numeric values')
                        self.file_lines.extend(
                            ['EFFECT\n', (str(count) + ' ') * self.traits_count * self.lactation_dim + cat
                             + ' alpha\n'])
                    self.new_data[count] = self.data.iloc[:, mod_col - 1]
            elif len(effect) == 4:
                # If len is 4, then we have a column of a certain category nested by another column of certain category.
                # We check that if col is covariate, it only contains numerical values and we also check in this case
                # that nested is a covariate and contains only numerical values. This is done in order to comply with
                # BLUPF90 requirements. We also have to place the correct index for nested columns, as they are supposed
                # to be numbered after the actual fixed effects, Legendre coefficients and traits.
                col, cat, nested, nested_cat = effect
                check_cat(col, cat)
                check_cat(nested, nested_cat)

                mod_col = self.__get_col__(col)
                mod_nested = self.__get_col__(nested)
                if cat == 'cross':
                    self.new_data[count] = self.data.iloc[:, [mod_col - 1, mod_nested - 1]].astype(str).apply('_'.join,
                                                                                                              axis=1)
                    self.file_lines.extend(['EFFECT\n', (str(count) + ' ') * self.traits_count * self.lactation_dim
                                            + 'cross alpha\n'])
                else:
                    if not is_numeric_dtype(self.data.iloc[:, mod_col - 1]):
                        raise ValueError('Covariate effect', col, 'has non-numeric values')
                    if nested_cat == 'cov' and not is_numeric_dtype(self.data.iloc[:, mod_nested - 1]):
                        raise ValueError('Covariate nested effect', nested, 'has non-numeric values')
                    self.file_lines.extend(['EFFECT\n', (str(count) + ' ') * self.traits_count * self.lactation_dim
                                            + cat + (' numer\n' if cat == 'cross' else ' \n'), 'NESTED\n',
                                            (str(self.fixed_count + self.degree + 1 + nested_count) + ' ')
                                            * self.traits_count * self.lactation_dim + ' ' + nested_cat
                                            + (' \n' if nested_cat == 'cov' else 'numer\n'
                                            if is_numeric_dtype(self.data.iloc[:, mod_nested - 1]) else ' alpha\n')])
                    self.new_data[count] = self.data.iloc[:, mod_col - 1]
                    self.new_data[self.fixed_count + self.degree + 1 + nested_count] = self.data.iloc[
                                                                                       :, mod_nested - 1]
                    nested_count += 1
                    self.nested_count += 1
            else:
                raise ValueError('Invalid effect declaration')
            count += 1

    def __add_legendre_polynomials__(self):
        if self.degree >= 0:
            dim_values = self.data.iloc[:, self.__get_col__(self.dim_col) - 1]
            scaled_dim_col = -1 + 2 * (dim_values - dim_values.min()) / (dim_values.max() - dim_values.min())
            for i in range(self.degree + 1):
                self.new_data[self.fixed_count + i + 1] = legendre(i)(scaled_dim_col) * np.sqrt(i + 0.5)
                if i <= self.fixed_degree:
                    # Legendre fixed coefficients have their indices right after the fixed effects
                    self.file_lines.extend(['EFFECT\n', (str(self.fixed_count + i + 1) + ' ') * self.traits_count
                                            * self.lactation_dim + 'cov\n'])

    def __add_traits_to_renumf90_pheno__(self):
        """
        Adds traits in the dataframe that will be used to create the phenotype file used by renumf90. If we don't have
        a test day model, the column is simply added. However, if test day model is used, we must be able to
        differentiate between lactations. To do that, we will have a number of columns equal to the number of lactations
        which will contain for the ith column the observations associated to the ith lactation and the rest of the
        values will be 0
        :return: None
        """
        count = 1
        for i in range(self.lactation_dim):
            for idx in self.trait_cols_idx:
                trait_data = self.data.iloc[:, idx - 1]
                if self.lactation_col is None:
                    self.new_data[self.fixed_count + self.degree + 1 + self.nested_count + count] = trait_data
                else:
                    self.new_data[self.fixed_count + self.degree + 1 + self.nested_count + count]\
                        = trait_data * (self.data.iloc[:, self.__get_col__(self.lactation_col) - 1] == i + 1)
                count += 1

    def __add_animal_effect__(self):
        """
        Adds animal effect in renumf90 file. The animal effect will be ordered after the fixed effects, fixed Legendre
        polynomials, nested columns and traits, hence the index used for the animal effect
        :return: None
        """
        col_idx = self.fixed_count + self.degree + 1 + self.nested_count + self.traits_count * self.lactation_dim + 1
        self.new_data[col_idx] = self.data.iloc[:, self.animal_col - 1]
        self.file_lines.extend(['EFFECT\n', (str(col_idx) + ' ') * self.traits_count * self.lactation_dim
                                + 'cross alpha\n', 'RANDOM\n', 'animal\n'])

    def __add_perm_effect__(self):
        """
        Checks whether we have multiple records for an animal (test day model or repeatability model) or not and adds
        a permament effect to renumf90 if needed
        :return: None
        """
        if self.lactation_dim > 1 or self.data.iloc[:, self.animal_col - 1].value_counts().max() > 1:
            self.file_lines.extend(['OPTIONAL\n', 'pe\n'])
            self.has_perm = True

    def __add_pedigree__(self):
        if self.ped is not None:
            self.ped.to_csv('ped.txt', sep=' ', header=False, index=False)
            self.file_lines.extend(['FILE\n', 'ped.txt\n', 'FILE_POS\n', '1 2 3 0 0\n'])
        self.__add_snps__()
        if self.ped is not None:
            self.file_lines.extend(['PED_DEPTH\n', '100\n'])

    def __add_snps__(self):
        """
        If SNP data is available, we need to process it. It is mandatory that the first column contains the animals'
        IDs. We check that the other columns only contain values of 0, 1 and 2. Then, we create a dataframe which will
        contain the IDs and the concatenated SNPs, as blupf90 programs require, and we will write them in a snps.txt
        file according to blupf90 requirements. We also save a copy of all current lines which will be written in the
        renumf90 file in order to be able to use these dummy lines to create a file which will be used to find out the
        values of A matrix or its inverse
        :return:
        """
        self.dummy_lines = self.file_lines.copy()
        if self.genomic_data is not None:
            self.snp_data = pd.DataFrame()
            for col in self.genomic_data.columns[1:]:
                genomic_col = self.genomic_data[col]
                if (not is_integer_dtype(genomic_col)) or genomic_col.min() < 0 or genomic_col.max() > 2:
                    raise ValueError('Invalid SNPs values - should only be 0, 1 or 2')
            self.snp_data[0] = self.genomic_data.iloc[:, 0].astype(str)
            self.snp_data[1] = self.genomic_data.iloc[:, 1:].astype(str).apply(''.join, axis=1)
            self.file_lines.extend(['SNP_FILE\n', 'snps.txt\n'])
            self.__write_snps_to_file__()

    def __add_inbreeding__(self):
        if self.inbreeding:
            self.file_lines.extend(['INBREEDING\n', 'pedigree\n'])

    def __add_random_regression__(self):
        """
        The random Legendre polynomials columns will be ordered so that they can be found after the fixed effects, which
        explains the used range
        :return: None
        """
        if self.random_degree >= 0:
            self.file_lines.extend(['RANDOM_REGRESSION\n', 'data\n', 'RR_POSITION\n',
                                    ' '.join(map(str, range(self.fixed_count + 1,
                                                            self.fixed_count + self.random_degree + 2))) + '\n'])

    def __add_variances__(self):
        """
        Writes the additive genetic and permanent effect covariance matrices elements in strings which will represent
        lines in the renumf90 file

        The dimension of the additive genetic and permanent effect covariance matrices depends on the number of
        lactations, the number of traits, and the degree of the random Legendre polynomials, that is for the test day
        model. However, thanks to our constraints, we can generalize the result to the animal model as well, because
        in that case we will have lactation_dim = 1 and random_degree = -1, and thus the dimension will be the number of
        traits x the number of traits, which is correct. If either the additive or permanent initial variances are
        given, we need to check that their values are numeric and also that their dimensions are the proper ones.
        Otherwise, for any of them that is not given, we will use a default matrix with 1 on the main diagonal and 0.1
        otherwise.
        :return:
        """
        dim = self.lactation_dim * self.traits_count * (max(self.random_degree, 0) + 1)
        self.file_lines.append('(CO)VARIANCES\n')
        if self.ag_variance is None:
            self.file_lines.extend([' '.join(map(str, [0.1] * i + [1.0] + [0.1] * (dim - i - 1))) + '\n' for i in
                                    range(dim)])
        else:
            if self.ag_variance.shape != (dim, dim):
                raise ValueError('Invalid genetic additive variance dimensions: should be', (dim, dim),
                                 'but they are instead', self.ag_variance.shape)
            if not is_numeric_dtype(self.ag_variance):
                raise ValueError('Additive genetic variance should only contain numerical values')
            self.file_lines.extend([' '.join(map(str, row)) + '\n' for row in self.ag_variance])
        if self.has_perm:
            self.file_lines.append('(CO)VARIANCES_PE\n')
            if self.pe_variance is None:
                self.file_lines.extend([' '.join(map(str, [0.1] * i + [1.0] + [0.1] * (dim - i - 1))) + '\n' for i in
                                        range(dim)])
            else:
                if self.pe_variance.shape != (dim, dim):
                    raise ValueError('Invalid permanent variance dimensions: should be', (dim, dim),
                                     'but they are instead', self.ag_variance.shape)
                if not is_numeric_dtype(self.pe_variance):
                    raise ValueError('Permanent variance should only contain numerical values')
                self.file_lines.extend([' '.join(map(str, row)) + '\n' for row in self.pe_variance])

    def __add_size_constraint__(self):
        """
        Adds a constraint of how long an alphanumeric element in the given data can be. As the RENUMF90 default is 20
        and this sometimes may not be enough, we need to add one manually
        :return:
        """
        max_size = self.new_data.astype(str).agg([len]).max().max()
        self.file_lines.append('OPTION alpha_size ' + str((max_size // 10 + 1) * 10) + '\n')

    def __add_genomic__(self):
        """
        After adding genomic data, we can run an interation of renumf90 followed by preGSf90. This allows us to save
        matrices of interest such as A22, G, inverse of A22, inverse of G and inverse of H. The options added ensure
        that this matrices will be stored in text files, from where we will read them
        :return: None
        """
        if self.genomic_data is not None:
            if self.export_genomic:
                self.file_lines.append('OPTION saveAscii\n')
            if self.ped is not None:
                if self.export_Hinv:
                    self.file_lines.append('OPTION saveHinv\n')
                if self.export_A22:
                    self.file_lines.append('OPTION saveA22\n')
                if self.export_A22inv:
                    self.file_lines.append('OPTION saveA22Inverse\n')
            if self.export_G:
                self.file_lines.append('OPTION saveG\n')
            if self.export_Ginv:
                self.file_lines.append('OPTION saveGInverse\n')
            self.file_lines.extend(['OPTION thrStopCorAG 0.0\n', 'OPTION no_quality_control\n'])
            if self.ped is None:
                self.file_lines.extend(['OPTION AlphaBeta 0.99 0.01\n', 'OPTION tunedG 0\n'])
            if self.export_genomic:
                with open('genrenf90.par', 'w') as f:
                    f.writelines(self.file_lines)
                os.system('renumf90 genrenf90.par')
                if not self.inbreeding:
                    switch_to_no_inbreeding()
                os.system('preGSf90 renf90.par')
                if self.ped is not None:
                    if self.export_A22:
                        self.__save_A22__()
                    if self.export_A22inv:
                        self.__save_A22inv__()
                if self.export_G:
                    self.__save_G__()
                if self.export_Ginv:
                    self.__save_Ginv__()
                if self.ped is not None and self.export_Hinv:
                    self.__save_Hinv__()

    def __write_snps_to_file__(self):
        """
        SNPs are written in a file respecting blupf90 restrictions to have SNPs aligned, thus in order to ensure this,
        we need to know the maximum length of an animal ID (let it be MAX) and for each SNPs row, we will add
        MAX - len(current_animal_ID) + 1 spaces between the ID and the SNPs (+ 1 is needed in order for the longest IDs
        to be separated by at least one space from their SNPs as well)
        :return: None
        """
        if self.snp_data is not None:
            max_name_len = self.snp_data[0].map(str).agg([len]).max().max()
            with open('snps.txt', 'w') as f:
                for idx in self.snp_data.index:
                    name = self.snp_data.loc[idx, 0]
                    snps = self.snp_data.loc[idx, 1]
                    f.write(name + ' ' * (max_name_len - len(name) + 1) + snps + '\n')

    def __save_A__(self):
        self.A = save_matrix('A22')

    def __save_Ainv__(self):
        self.Ainv = save_matrix('A22i')

    def __save_A22__(self):
        self.A22 = save_matrix('A22')

    def __save_A22inv__(self):
        self.A22inv = save_matrix('A22i')

    def __save_G__(self):
        self.G = save_matrix('G')

    def __save_Ginv__(self):
        self.Ginv = save_matrix('Gi')

    def __save_Hinv__(self):
        self.Hinv = save_matrix('Hinv.txt')

    def __add_animal_codes__(self):
        """
        Associate animals' original IDs to their code given by renumf90. We find the renaddXX.ped file (IT IS NECESSARY
        THAT MULTIPLE RUNS OF RENUMF90 ARE NOT MIXED TOGETHER OUTSIDE RUNNING THIS APP, OTHERWISE WE MAY END UP WITH TWO
        DIFFERENT RENADDXX.PED FILES AND THIS METHOD MAY CHOOSE THE WRONG ONE). After the file is found, all animals IDs
        from the renumbered pedigree are stored. In particular, two more dictionaries will be created and stored as
        fields: animal_to_code, which has animal IDs as keys and renumbered indices as values, and code_to_animal,
        which has renumbered indices as keys and animal IDs as values.
        :return: None
        """
        ped_file = [name for name in os.listdir('.') if name.startswith('renadd')][0]
        self.animal_to_code = {}
        self.code_to_animal = {}
        self.animals = []
        with open(ped_file) as f:
            lines = f.readlines()
            self.animal_count = len(lines)
            self.sires = np.zeros(self.animal_count).astype(int)
            self.dams = np.zeros(self.animal_count).astype(int)
            for line in lines:
                values = line.strip().split()
                self.animal_to_code[values[-1]] = int(values[0])
                self.code_to_animal[int(values[0])] = values[-1]
                self.animals.append(values[-1])
                self.sires[int(values[0]) - 1] = int(values[1])
                self.dams[int(values[0]) - 1] = int(values[2])

    def __add_dummy_snps__(self):
        """
        Construction of the dummy SNPs used to compute A matrix. Computing A matrix is done by actually computing an
        A22 matrix, where all the animals from the pedigree have genotypes. It is assumed that
        renumf90 was run previously (which is true, since we only use this method in __init__, just after a renumf90
        run). The dummy SNPs dataframe is then created. For this, 10 values of 5 are used for each animal's SNPs.
        We use 10 because blupf90 programs can't run with very few SNPs. Values of 5 represent missing genotypes, which
        is actually the case. Finally, the dummy SNPs are converted to a file respecting blupf90 programs requirements,
        just like in the __write_snps_to_file__ method.
        :return: None
        """

        self.dummy_snp = pd.DataFrame()
        self.dummy_snp[0] = self.animals
        self.dummy_snp[1] = ['5' * 10] * len(self.dummy_snp[0])
        self.dummy_snp[0] = self.dummy_snp[0].astype(str)
        max_name_len = self.dummy_snp[0].map(str).agg([len]).max().max()
        with open('dummy_snps.txt', 'w') as f:
            for idx in self.dummy_snp.index:
                name = self.dummy_snp.loc[idx, 0]
                snps = self.dummy_snp.loc[idx, 1]
                f.write(name + ' ' * (max_name_len - len(name) + 1) + snps + '\n')

    def __check_pure_gblup__(self):
        """
        When there is no pedigree given, we need to check if conditions for pure GBLUP are met. That means that we need
        to have genomic data available and it also means that all animals in the phenotypic data have to be present in
        the genomic data
        :return: None
        """
        if self.ped is None:
            if self.genomic_data is None:
                raise ValueError('It is not possible to perform analysis with neither pedigree of genomic data. At '
                                 + 'least one must be provided')
            if not set(self.data.iloc[:, self.__get_col__(self.animal_col) - 1]).issubset(set(
                    self.genomic_data.iloc[:, 0])):
                raise ValueError('There are animal with records, but no genotypes. In order to perform pure GBLUP, '
                                 + 'genotypes of all animals records need to be provided.')

    def __read_inbreeding__(self):
        """
        If pedigree is provided and inbreeding is set to True, the inbreeding coefficients will be read from the
        renf90.inb file
        :return: None
        """
        self.inbreeding_coefficients = np.zeros(self.animal_count)
        if self.ped is not None and self.inbreeding:
            with open('renf90.inb') as f:
                for line in f.readlines():
                    values = line.strip().split()
                    self.inbreeding_coefficients[int(values[-1]) - 1] = float(values[1])
