# FUNCTIONS FROM INDUSTRIAL DATA SCIENCE WORKFLOW (IDSW) PACKAGE
# Extract, transform, and load (ETL) data

# Marco Cesar Prado Soares, Data Scientist Specialist @ Bayer Crop Science LATAM
# marcosoares.feq@gmail.com
# marco.soares@bayer.com
from dataclasses import dataclass


class spc_chart_assistant:
            
    # Initialize instance attributes.
    # define the Class constructor, i.e., how are its objects:
    def __init__(self, assistant_startup = True, keep_assistant_on = True):
                
        import os
        
        # If the user passes the argument, use them. Otherwise, use the standard values.
        # Set the class objects' attributes.
        # Suppose the object is named assistant. We can access the attribute as:
        # assistant.assistant_startup, for instance.
        # So, we can save the variables as objects' attributes.
        self.assistant_startup = assistant_startup
        self.keep_assistant_on = keep_assistant_on
        # Base Github directory containing the assistant images to be downloaded:
        self.base_git_dir = "https://github.com/marcosoares-92/img_examples_guides/raw/main"
        # Create a new folder to store the images in local environment, 
        # if the folder do not exists:
        self.new_dir = "tmp"
        
        os.makedirs(self.new_dir, exist_ok = True)
        # exist_ok = True creates the directory only if it does not exist.
        
        self.last_img_number = 18 # number of the last image on the assistant
        self.numbers_to_end_assistant = (3, 4, 7, 9, 10, 13, 15, 16, 19, 20, 21, 22)
        # tuple: cannot be modified
        # 3: 'g', 4: 't', 7: 'i_mr', 9: 'std_error', 10: '3s', 13: 'x_bar_s'
        # 15: 'std_error' (grouped), 16: '3s' (grouped), 19: 'p', 20: 'np',
        # 21: 'c', 22: 'u'
        self.screen_number = 0 # start as zero
        self.file_to_fetch = ''
        self.img_url = ''
        self.img_local_path = ''
        # to check the class attributes, use the __dict__ method. Examples:
        ## object.__dict__ will show all attributes from object
                
    # Define the class methods.
    # All methods must take an object from the class (self) as one of the parameters
    
    def download_assistant_imgs (self):
                
        import os
        import shutil # component of the standard library to move or copy files.
        from html2image import Html2Image
                
        # Start the html object
        html_img = Html2Image()
                
        for screen_number in range(0, (self.last_img_number + 1)):
                
            # ranges from 0 to (last_img_number + 1) - 1 = last_img_number
            # convert the screen number to string to create the file name:
            
            # Update the attributes:
            self.file_to_fetch = "cc_s" + str(screen_number) + ".png"
            self.img_url = os.path.join(self.base_git_dir, self.file_to_fetch)
            
            # Download the image:
            # pypi.org/project/html2image/
            img = html_img.screenshot(url = self.img_url, save_as = self.file_to_fetch, size = (500, 500))
            # If size is omitted, the image is downloaded in the low-resolution default.
            # save_as must be a file name, a path is not accepted.
            # Make the output from the method equals to a variable eliminates its verbosity
                    
            # Create the new path for the image (local environment):
            self.img_local_path = os.path.join(self.new_dir, self.file_to_fetch)
            # Move the image files to the new paths:
            # use shutil.move(source, destination) method to move the files:
            # pynative.com/python-move-files
            # docs.python.org/3/library/shutil.html
            shutil.move(self.file_to_fetch, self.img_local_path)
            # Notice that file_to_fetch attribute still stores a file name like 'cc_s0.png'
        
        # Now, all images for the assistant were downloaded and stored in the temporary
        # folder. So, let's start the two boolean variables to initiate it and run it:
        self.assistant_startup = True 
        # attribute to start the assistant in the first screen
        self.keep_assistant_on = True
        # attribute to maintain the assistant working
        
        return self

    def delete_assistant_imgs (self):
                
        import os
        # Now, that the user closed the assistant, we can remove the downloaded files 
        # (delete them) from the notebook's workspace.
                
        # The os.remove function deletes a file or directory specified.
        for screen_number in range(0, (self.last_img_number + 1)):
                    
            self.file_to_fetch = "cc_s" + str(screen_number) + ".png"
            self.img_local_path = os.path.join(self.new_dir, self.file_to_fetch)
            os.remove(self.img_local_path)
                
        # Now that the files were removed, check if the tmp folder is empty:
        size = os.path.getsize(self.new_dir)
        # os.path.getsize returns the total size in Bytes from a folder or a file.
                
        # Get the list of sub-folders, files or subdirectories (the content) from the folder:
        list_of_contents = os.listdir(self.new_dir)
        # doc.python.org/3/library/os.html
        # It returns a list of strings representing the paths of each file or directory 
        # in the analyzed folder.
                
        # If the size is 0 and the length of the list_of_contents is also zero (i.e., there is no
        # previous sub-directory created), then remove the directory:
        if ((size == 0) & (len(list_of_contents) == 0)):
            
            os.rmdir(self.new_dir)

    def print_screen_legend (self):
        
        if (self.screen_number == 0):
            
            print("The control chart is a line graph showing a measure (y-axis) over time (x-axis).")
            
            print("In contrast to the run chart, the central line of the control chart represents the (weighted) mean, rather than the median.")
            print("Additionally, two lines representing the upper and lower control limits are shown.\n")
            print("The control limits represent the boundaries of the so-called common cause variation, which is inherent to the process.")
            print("Walther A. Shewhart, who invented the control chart, described two types of variation: chance-cause variation and assignable-cause variation.")
            print("These were later renamed to common-cause and special-cause variation.\n")
            
            print("Common-cause variation:")
            print("Is present in any process.")
            print("It is caused by phenomena that are always present within the system.")
            print("It makes the process predictable (within limits).")
            print("Common-cause variation is also called random variation or noise.\n")
                    
            print("Special-cause variation:")
            print("Is present in some processes.")
            print("It is caused by phenomena that are not normally present in the system.")
            print("It makes the process unpredictable.")
            print("Special-cause variation is also called non-random variation or signal.\n")
                    
            print("It is important to notice that neither common, nor special-cause variation is in itself 'good' or 'bad'.")
            print("A stable process may function at an unsatisfactory level; and an unstable process may be moving in the right direction.")
            print("On the other hand, the end goal of improvement is always to achieve a stable process functioning at a satisfactory level.\n")
                    
            print("Control chart limits:")
            print("The control limits, also called sigma limits, are usually placed at ±3 standard deviations from the central line.")
            print("So, the standard deviation is estimated as the common variation of the process of interest.")
            print("This variation depends on the theoretical distribution of data.")
            print("It is a beginner's mistake to simply calculate the standard deviation of all the data points.")
            print("This procedure would include both the common and special-cause variation in the calculus.")
            print("Since the calculations of control limits depend on the type of data (distribution), many types of control charts have been developed for specific purposes.")
        
        elif (self.screen_number == 1):
                    
            print("CHARTS FOR RARE EVENTS\n")
            print("ATTENTION: Due not previously group data in this case. Since events are rare, they are likely to be eliminated during aggregation.\n")
                    
            print("G-chart for units produced between (rare) defectives or defects;")
            print("or total events between successive rare occurrences:\n")
            print("When defects or defectives are rare and the subgroups are small, C, U, and P-charts become useless.")
            print("That is because most subgroups will have no defects.")
                    
            print("Example: if 8% of discharged patients have a hospitals-acquired pressure ulcer, and the average weekly number of discharges in a small department is 10, we would, on average, expect to have less than one pressure ulcer per week.")
            print("Instead, we could plot the number of discharges between each discharge of a patient with one or more pressure ulcers.\n")
            print("The number of units between defectives is modelled by the geometric distribution.")
            print("So, the G-control chart plots counting of occurrence by number; time unit; or timestamp.\n")
                    
            print("In the example of discharged patients: the indicator is the number of discharges between each of these rare cases.")
            print("Note that the first patient with pressure ulcer is missing from the chart.")
            print("It is due to the fact that we do not know how many discharges there had been before the first patient with detected pressure ulcer.\n")
            print("The central line of the G-chart is the theoretical median of the distribution")
            print("median = mean × 0.693")
                    
            print("Since the geometric distribution is highly skewed, the median is a better representation of the process center.")
            print("Also, notice that the G-chart rarely has a lower control limit.\n")
                    
            print("T-chart for time between successive rare events:\n")
            print("Like the G-chart, the T-chart is a rare event chart.")
            print("Instead of displaying the number of cases between events (defectives), this chart represents the time between successive rare events.\n")
            print("Since time is a continuous variable, the T-chart belongs with the other charts for measure numeric data.")
            print("Then, T-chart plots the timedelta (e.g. number of days between occurrences) by the measurement, time unit, or timestamp.")
                
        elif (self.screen_number == 2):
            
            print("A quality characteristic that is measured on a numerical scale is called a variable.")
            print("Examples: length or width, temperature, and volume.\n")
            
            print("The Shewhart control charts are widely used to monitor the mean and variability of variables.")
            print("On the other hand, many quality characteristics can be expressed in terms of a numerical measurement.")
                    
            print("For example: the diameter of a bearing could be measured with a micrometer and expressed in millimeters.\n")
            print("A single measurable quality characteristic, such as a dimension, weight, or volume, is a variable.")
            print("Control charts for variables are used extensively, and are one of the primary tools used in the analize and control steps of DMAIC.")
            
            print("Many quality characteristics cannot be conveniently represented numerically, though.")
            print("In such cases, we usually classify each item inspected as either conforming or nonconforming to the specifications on that quality characteristic.")
            print("The terminology defective or nondefective is often used to identify these two classifications of product.")
            print("More recently, this terminology conforming and nonconforming has become popular.")        
            print("Quality characteristics of this type are called attributes.\n")
            
            print("Control Charts for Nonconformities (defects):")
            print("A nonconforming item is a unit of product that does not satisfy one or more of the specifications of that product.")
            print("Each specific point at which a specification is not satisfied results in a defect or nonconformity.")
            print("Consequently, a nonconforming item will contain at least one nonconformity.")
            print("However, depending on their nature and severity, it is quite possible for a unit to contain several nonconformities and not be classified as nonconforming.")
                    
            print("Example: suppose we are manufacturing personal computers. Each unit could have one or more very minor flaws in the cabinet finish,")
            print("but since these flaws do not seriously affect the unit's functional operation, it could be classified as conforming.")
            print("However, if there are too many of these flaws, the personal computer should be classified as nonconforming,")
            print("because the flaws would be very noticeable to the customer and might affect the sale of the unit.\n")
                    
            print("There are many practical situations in which we prefer to work directly with the number of defects or nonconformities,")
            print("rather than the fraction nonconforming.")
            print("These include:")
            print("1. Number of defective welds in 100 m of oil pipeline.")
            print("2. Number of broken rivets in an aircraft wing.")
            print("3. Number of functional defects in an electronic logic device.")
            print("4. Number of errors on a document, etc.\n")
                    
            print("It is possible to develop control charts for either the total number of nonconformities in a unit,")
            print("or for the average number of nonconformities per unit.\n")
            print("These control charts usually assume that the occurrence of nonconformities in samples of constant size is well modeled by the Poisson distribution.\n")
                    
            print("Essentially, this requires that the number of opportunities or potential locations for nonconformities be infinitely large;")
            print("and that the probability of occurrence of a nonconformity at any location be small and constant.")
            print("Furthermore, the inspection unit must be the same for each sample.")
            print("That is, each inspection unit must always represent an identical area of opportunity for the occurrence of nonconformities.")
                    
            print("In addition, we can count nonconformities of several different types on one unit, as long as the above conditions are satisfied for each class of nonconformity.\n")
            print("In most practical situations, these conditions will not be perfectly satisfied.")
            print("The number of opportunities for the occurrence of nonconformities may be finite,")
            print("or the probability of occurrence of nonconformities may not be constant.\n")
                    
            print("As long as these departures from the assumptions are not severe,")
            print("the Poisson model will usually work reasonably well.")
            print("There are cases, however, in which the Poisson model is completely inappropriate.")
            print("So, always check carefully the distributions.")
                    
            print("If you are not sure, use the estimatives based on more general assumptions, i.e.,")
            print("The estimative of the natural variation as 3 times the standard deviation;")
            print("or as 3 times the standard error.\n")
                    
            print("Individual samples x Grouped data")
            print("Often, we collect a batch of samples corresponding to the same conditions, and use aggregation measurements such as mean, sum, or standard deviation to represent them.")
            print("In this case, we are grouping our data, and not working with individual measurements.")
            print("In turns, we can collect individual samples: there are no repetitions, only individual measurements corresponding to different conditions.\n")
            print("Usually, time series data is collected individually: each measurement corresponds to an instant, so it is not possible to collect multiple samples corresponding to the same conditions for further grouping.")
            print("Example: instant assessment of pH, temperature, pressure, etc.\n")
            print("Naturally, we can define a time window like a day, and group values on that window.")
            print("The dynamic of the phenomena should not create significant differences between samples collected for a same window, though.")
        
        elif (self.screen_number == 5):
            
            print("CHARTS FOR NUMERICAL VARIABLES\n")
            print("When dealing with a quality characteristic that is a variable, it is usually necessary to monitor both the mean value of the quality characteristic and its variability.")
            print("The control of the process average or mean quality level is usually done with the control chart for means, or the X-bar control chart.")
            print("The process variability can be monitored with either a control chart for the standard deviation, called the s control chart, or with a control chart for the range, called an R control chart.\n")
            
            print("I and MR charts for individual measurements:")
            print("ATTENTION: The I-MR chart can only be used for data that follows the normal distribution.")
            print("That is because the calculus of the control limits are based on the strong hypothesis of normality.")
            print("If you have individual samples that do not follow the normal curve (like skewed data, or data with high kurtosis);")
            print("or data with an unknown distribution, select number 8 for using less restrictive hypotheses for the estimative of the natural variation.\n")
                    
            print("Example: in healthcare, most quality data are count data.")
            print("However, from time to time, there are measurement data present.")
            print("These data are often in the form of physiological parameters or waiting times.")
            print("e.g. a chart of birth weights from 24 babies.")
            print("If the birth weights follow the normal, you can use the individuals chart.\n")
                    
            print("Actually, there are many situations in which the sample size used for process monitoring is n = 1; that is, the sample consists of an individual unit.")
            print("Some other examples of these situations are as follows:")
            print("1. Automated inspection and measurement technology is used, and every unit manufactured is analyzed.")
            print("So, there is no basis for rational subgrouping.")
            print("2. Data comes available relatively slowly, and it is inconvenient to allow sample sizes of n > 1 to accumulate before analysis.") 
            print("The long interval between observations will cause problems with rational subgrouping.")
            print("This occurs frequently in both manufacturing and non-manufacturing situations.")
            print("3. Repeat measurements on the process differ only because of laboratory or analysis error, as in many chemical processes.")
            print("4. Multiple measurements are taken on the same unit of product, such as measuring oxide thickness at several different locations on a wafer in semiconductor manufacturing.")
            print("5. In process plants, such as papermaking, measurements on some parameter (such as coating thickness across the roll) will differ very little and produce a standard deviation that is much too small if the objective is to control coating thickness along the roll.")
                    
            print("In such situations, the control chart for individual units is useful.")
            print("In many applications of the individuals control chart, we use the moving range two successive observations as the basis of estimating the process variability.\n")
            print("I-charts are often accompanied by moving range (MR) charts, which show the absolute difference between neighbouring data points.")
            print("The purpose of the MR chart is to identify sudden changes in the (estimated) within-subgroup variation.")
            print("If any data point in the MR is above the upper control limit, one should interpret the I-chart very cautiously.\n")
        
        elif(self.screen_number == 6):
                    
            print("One important difference: numeric variables are representative of continuous data, usually in the form of real numbers (float values).")
            print("It means that its possible values cannot be counted: there is an infinite number of possible real values.")
            
            print("Categoric variables, in turn, are discrete.")        
            print("It means they can be counted, since there is a finite number of possibilities.")
            print("Such variables are usually present as strings (texts), or as ordinal (integer) numbers.")
                    
            print("If there are only two categories, we have a binary classification.")
            print("Each category can be reduced to a binary system: or the category is present, or it is not.")
            print("This is the idea for the One-Hot Encoding.")
            print("Usually, values in a binary classification are 1 or 0, so that a probability can be easily associated through the sigmoid function.\n")
                    
            print("Some examples of quality characteristics that are based on the analysis of attributes:")
            print("1. Proportion of warped automobile engine connecting rods in a day's production.")
            print("2. Number of nonfunctional semiconductor chips on a wafer.")
            print("3. Number of errors or mistakes made in completing a loan application.")
            print("4. Number of medical errors made in a hospital.\n")
        
        elif((self.screen_number == 8) | (self.screen_number == 14)):
            
            print("If you have a distribution that is not normal, like distributions with high skewness or high kurtosis,")
            print("use less restrictive methodologies to estimate the natural variation.\n")
                    
            print("You may estimate the natural variation as 3 times the standard error; or as 3 times the standard deviation.")
            print("The interval will be symmetric around the mean value.\n")
            print("Recommended: standard error, which normalizes by the total of values.")
                
        elif(self.screen_number == 11):
            
            print("CHARTS FOR NUMERICAL VARIABLES\n")
            print("When dealing with a quality characteristic that is a variable, it is usually necessary to monitor both the mean value of the quality characteristic and its variability.")
            print("The control of the process average or mean quality level is usually done with the control chart for means, or the X-bar control chart.")
            print("The process variability can be monitored with either a control chart for the standard deviation, called the s control chart, or with a control chart for the range, called an R control chart.\n")
                    
            print("X-bar and S charts for average measurements:")
            print("If there is more than one measurement of a numeric variable in each subgroup,")
            print("the Xbar and S charts will display the average and the within-subgroup standard deviation, respectively.")
            print("e.g. a chart of average birth weights per month, for babies born over last year.")
        
        elif(self.screen_number == 12):
            
            print("CHARTS FOR CATEGORICAL VARIABLES\n")
            print("There are 4 widely used attributes control charts: P, nP, U, and C.\n")
                    
            print("To illustrate them, consider a dataset containing the weekly number of hospital acquired pressure ulcers at a hospital")
            print("The hospital has 300 patients, with an average length of stay of four days.") 
            print("Each of the dataframe's 24 rows contains information for one week on: the number of discharges,")
            print("patient days; pressure ulcers; and number of discharged patients with one or more pressure ulcers.")
            print("On average, 8% of discharged patients have 1.5 hospital acquired pressure ulcers.\n")
                    
            print("Some of the charts for categorical variables are based on the definition of the fraction nonconforming.")
            print("The fraction nonconforming is defined as the ratio between:")
            print("the number of nonconforming items in a population; by the total number of items in that population.")
            print("The items may have several quality characteristics that are examined simultaneously by the inspector.")
            print("If the item does not conform to the standards of one or more of these characteristics, it is classified as nonconforming.\n")
                    
            print("ATTENTION: Although it is customary to work with fraction nonconforming,")
            print("we could also analyze the fraction conforming just as easily, resulting in a control chart of process yield.")
            print("Many manufacturing organizations operate a yield-management system at each stage of their manufacturing process,")
            print("with the first-pass yield tracked on a control chart.\n")
            
            print("Traditionally, the term 'defect' has been used to name whatever it is being analyzed through counting with control charts.\n")
            print("There is a subtle, but important, distinction between:")
            print("counting defects, e.g. number of pressure ulcers;")
            print("and counting defectives, e.g. number of patients with one or more pressure ulcers.\n")
                    
            print("Defects are expected to reflect the Poisson distribution,")
            print("while defectives reflect the binomial distribution.\n")
        
        elif(self.screen_number == 17):
                    
            print("P-charts for proportion of defective units:")
            print("The first of these relates to the fraction of nonconforming or defective product produced by a manufacturing process, and is called the control chart for fraction nonconforming, or P-chart.")
            
            print("The P chart is probably the most common control chart in healthcare.")
            print("It is used to plot the proportion (or percent) of defective units.")
            print("e.g. the proportion of patients with one or more pressure ulcers.")
                    
            print("As mentioned, defectives are modelled by the binomial distribution.")
            print("In theory, the P chart is less sensitive to special cause variation than the U chart.")
            print("That is because it discards information by dichotomising inspection units (patients) in defectives and non-defectives ignoring the fact that a unit may have more than one defect (pressure ulcers).")
            print("On the other hand, the P chart often communicates better.")
                    
            print("For most people, not to mention the press, the percent of harmed patients is easier to grasp than the the rate of pressure ulcers expressed in counts per 1000 patient days.\n")
            print("The sample fraction nonconforming is defined as the ratio of the number of nonconforming units in the sample D to the sample size n:")
            print("p = D/n")
            print("From the binomial distribution, the mean should be estimated as p, and the variance s² as p(1-p)/n.")
                    
            print("nP-Charts for number nonconforming:")
            print("It is also possible to base a control chart on the number nonconforming,")
            print("rather than on the fraction nonconforming.")
            print("This is often called as number nonconforming (nP) control chart.\n")
            
        elif(self.screen_number == 18):
            
            print("C-charts for count of defects:")
            print("In some situations, it is more convenient to deal with the number of defects or nonconformities observed,")
            print("rather than the fraction nonconforming.\n")
            
            print("So, another type of control chart, called the control chart for nonconformities, or the C chart,")
            print("is designed to deal with this case.\n")
                    
            print("In the hospital example:")
            print("The correct control chart for the number of pressure ulcers is the C-chart,")
            print("which is based on the Poisson distribution.\n")
                    
            print("As mentioned, DEFECTIVES are modelled by the BINOMIAL distribution, whereas DEFECTS are are modelled by POISSON distribution.\n")
            
            print("U-charts for rate of defects:")
            print("The control chart for nonconformities per unit, or the U-chart, is useful in situations")
            print("where the average number of nonconformities per unit is a more convenient basis for process control.\n")
                
            print("The U-chart is different from the C-chart in that it accounts for variation in the area of opportunity.")
            print("Examples:")
            print("1. Number of patients over time.")
            print("2. Number of patients between units one wishes to compare.")
            print("3. Number of patient days over time.")
            print("4. Number of patient days between units one wishes to compare.\n")
                 
            print("If there are many more patients in the hospital in the winter than in the summer,")
            print("the C-chart may falsely detect special cause variation in the raw number of pressure ulcers.\n")
            
            print("The U-chart plots the rate of defects.")
            print("A rate differs from a proportion in that the numerator and the denominator need not be of the same kind,")
            print("and that the numerator may exceed the denominator.\n")
                
            print("For example: the rate of pressure ulcers may be expressed as the number of pressure ulcers per 1000 patient days.\n")
            print("The larger the numerator, the narrower the control limits.\n")
            print("So, the main difference between U and C-charts is that U is based on the average number of nonconformities per inspection unit.\n")
            
            print("If we find x total nonconformities in a sample of n inspection units,")
            print("then the average number of nonconformities per inspection unit will be:")
            print("u = x/n")
            print("\n")
               
    def open_chart_assistant_screen (self):
                
        import os
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from html2image import Html2Image
        from tensorflow.keras.preprocessing.image import img_to_array, load_img
        # img_to_array: convert the image into its numpy array representation
                
        if (self.assistant_startup): #run if it is True:
            
            self.screen_number = 0 # first screen
        
        if (self.screen_number not in self.numbers_to_end_assistant):
            
            self.print_screen_legend()
            # Use its own method
            
            # Update attributes:
            self.file_to_fetch = "cc_s" + str(self.screen_number) + ".png"
            # Obtain the path of the image (local environment):
            self.img_local_path = os.path.join(self.new_dir, self.file_to_fetch)
                    
            # Load the image and save it on variables:
            assistant_screen = load_img(self.img_local_path)
                    
            # show image with plt.imshow function:
            fig = plt.figure(figsize = (12, 8))
            plt.imshow(assistant_screen)
            # If the image is black and white, you can color it with a cmap as fig.set_cmap('hot')
            
            #set axis off:
            plt.axis('off')
            plt.show()
            print("\n")
            
            # Run again the assistant for next screen (keep assistant on):
            self.keep_assistant_on = True
            # In the next round, the assistant should not be restarted:
            self.assistant_startup = False
            
            screen_number = input("Enter the number you wish here (in the right), according to the shown in the image above: ")
            #convert the screen number to string:
            screen_number = str(screen_number)        
            # Strip spaces and format characters (trim):
            screen_number = screen_number.strip()        
            # We do not call the str attribute for string variables (only for iterables)
            # Convert to integer
            screen_number = int(screen_number)
            # Update the attribute:
            self.screen_number = screen_number
        
        else:
            
            # user selected a value that ends the assistant:
            self.keep_assistant_on = False
            self.assistant_startup = False
        
        # Return the booleans to the main function:
        return self
        
    def chart_selection (self):
                
        # Only if the screen is in the tuple numbers_to_end_assistant:
        if (self.screen_number in self.numbers_to_end_assistant):
                    
            # Variables are created only when requested:
            rare_events_tuple = (3, 4) # g, t
            continuous_dist_not_defined_tuple = (9, 10) # std_error, 3std
            grouped_dist_not_defined_tuple = (15, 16) # std_error, 3std
            grouped_tuple = (13, 19, 20, 21, 22) # x, p, np, c, u
            charts_map_dict = {3:'g', 4:'t', 7:'i_mr', 9:'std_error', 10:'3s_as_natural_variation',
                                13:'xbar_s', 15:'std_error', 16:'3s_as_natural_variation',
                                19:'p', 20:'np', 21:'c', 22:'u'}
                    
            chart_to_use = charts_map_dict[self.screen_number]
                    
            # Variable with subgroups, which will be updated if needed:
            column_with_labels_or_subgroups = None
                    
            # Variable for skewed distribution, which will be updated if needed:
            consider_skewed_dist_when_estimating_with_std = False
                    
            column_with_variable_to_be_analyzed = str(input("Enter here (in the right) the name or number of the column (its header) that will be analyzed with the control chart.\nDo not type it in quotes.\nKeep the exact same format of the dataset, with spaces, characters, upper and lower cases, etc (or an error will be raised): "))
            # Try to convert it to integer, if it is a number:
            try:
                # Clean the string:
                column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed.strip()
                column_with_variable_to_be_analyzed = int(column_with_variable_to_be_analyzed)
                    
            except: # simply pass
                pass
                    
            print("\n")
            
            yes_no = str(input("Do your data have a column containing timestamps or time indication (event order)?\nType yes or no, here (in the right).\nDo not type it in quotes: "))
            yes_no = yes_no.strip()        
            # convert to full lower case, independently of the user:
            yes_no = yes_no.lower()
                    
            if (yes_no == 'yes'):
                    
                print("\n")
                timestamp_tag_column = str(input("Enter here (in the right) the name or number of the column containing timestamps or time indication (event order).\nDo not type it in quotes.\nKeep the exact same format of the dataset, with spaces, characters, upper and lower cases, etc (or an error will be raised): "))
                
                # Try to convert it to integer, if it is a number:
                try:
                    timestamp_tag_column = timestamp_tag_column.strip()
                    timestamp_tag_column = int(timestamp_tag_column)
                        
                except: # simply pass
                    pass
                    
            else:
                timestamp_tag_column = None
                    
            yes_no = str(input("Do your data have a column containing event frame indication; indication for separating time windows for comparison analysis;\nstages; events to be analyzed separately; or any other indication for slicing the time axis for comparison of different means, variations, etc?\nType yes or no, here (in the right).\nDo not type it in quotes: "))
            yes_no = yes_no.strip()
            yes_no = yes_no.lower()
                    
            if (yes_no == 'yes'):
                        
                print("\n")
                column_with_event_frame_indication = str(input("Enter here (in the right) the name or number of the column containing the event frame indication.\nDo not type it in quotes.\nKeep the exact same format of the dataset, with spaces, characters, upper and lower cases, etc (or an error will be raised): "))
                        
                # Try to convert it to integer, if it is a number:
                try:
                    column_with_event_frame_indication = column_with_event_frame_indication.strip()
                    column_with_event_frame_indication = int(column_with_event_frame_indication)
                        
                except: # simply pass
                    pass
            
            else:
                column_with_event_frame_indication = None
                    
            if (self.screen_number in rare_events_tuple):
                        
                print("\n")
                print(f"How are the rare events represented in the column {column_with_variable_to_be_analyzed}?")
                print(f"Before obtaining the chart, you must have modified the {column_with_variable_to_be_analyzed} to labe these data.")
                print("The function cannot work with boolean filters. So, if a value corresponds to a rare event occurrence, modify its value to properly labelling it.")
                print("You can set a special string or a special numeric value for indicating that a particular row corresponds to a rare event.")
                print("That is because rare events occurrences must be compared against all other 'regular' events.")
                print(f"For instance, {column_with_variable_to_be_analyzed} may show a value like 'rare_event', or 'ulcer' (in our example) if it is a rare occurrence.")
                print("Also, you could input a value extremely high, like 1000000000, or extremely low, like -10000000 for marking the rare events in the column.")
                print("The chart will be obtained after finding these rare events marks on the column.\n")
                        
                rare_event_indication = str(input(f"How are the rare events represented in the column {column_with_variable_to_be_analyzed}?\nEnter here (in the right) the text or number representing a rare event.\nDo not type it in quotes.\nKeep the exact same format of the dataset, with spaces, characters, upper and lower cases, etc (or the rare events will not be localized in the dataset): "))
                        
                # Try to convert it to float, if it is a number:
                try:
                    column_with_event_frame_indication = column_with_event_frame_indication.strip()
                    column_with_event_frame_indication = float(column_with_event_frame_indication)
                
                except: # simply pass
                    pass
                        
                rare_event_timedelta_unit = str(input(f"What is the usual order of magnitude for the intervals (timedeltas) between rare events?\nEnter here (in the right).\nYou may type: year, month, day, hour, minute, or second.\nDo not type it in quotes: "))
                rare_event_timedelta_unit = rare_event_timedelta_unit.strip()
                rare_event_timedelta_unit = rare_event_timedelta_unit.lower()
                
                while (rare_event_timedelta_unit not in ['year', 'month', 'day', 'hour', 'minute', 'second']):
                    
                    rare_event_timedelta_unit = str(input("Please, enter a valid timedelta unit: year, month, day, hour, minute, or second.\nDo not type it in quotes: "))
                    rare_event_timedelta_unit = rare_event_timedelta_unit.strip()
                    rare_event_timedelta_unit = rare_event_timedelta_unit.lower()
                    
            else:
                
                rare_event_timedelta_unit = None
                rare_event_indication = None
                        
                if ((self.screen_number in grouped_dist_not_defined_tuple) | (self.screen_number in grouped_tuple)):
                            
                    print("\n")
                    column_with_labels_or_subgroups = str(input("Enter here (in the right) the name or number of the column containing the subgroups or samples for aggregating the measurements in terms of mean, standard deviation, etc.\nIt may be a column with indications like 'A', 'B', or 'C'; 'subgroup1',..., 'sample1',..., or an integer like 1, 2, 3,...\nThis column will allow grouping of rows in terms of the correspondent samples.\nDo not type it in quotes.\nKeep the exact same format of the dataset, with spaces, characters, upper and lower cases, etc (or an error will be raised): "))
                            
                    # Try to convert it to integer, if it is a number:
                    try:
                        column_with_labels_or_subgroups = column_with_labels_or_subgroups.strip()
                        column_with_labels_or_subgroups = int(column_with_labels_or_subgroups)
                    
                    except: # simply pass
                        pass
                
                if ((self.screen_number in grouped_dist_not_defined_tuple) | (self.screen_number in continuous_dist_not_defined_tuple)):
                            
                    print("\n")
                    print("Is data skewed or with high kurtosis? If it is, the median will be used as the central line estimative.")
                    print("median = mean × 0.693\n")
                            
                    yes_no = str(input("Do you want to assume a skewed (or with considerable kurtosis) distribution?\nType yes or no, here (in the right).\nDo not type it in quotes: "))
                    yes_no = yes_no.strip()
                    yes_no = yes_no.lower()
                            
                    if (yes_no == 'yes'):
                        
                        # update the boolean variable
                        consider_skewed_dist_when_estimating_with_std = True
                
                
            print("Finished mapping the variables for obtaining the control chart plots.")
            print("If an error is raised; or if the chart is not complete, check if the columns' names inputs are strictly correct.\n")
            
            return chart_to_use, column_with_labels_or_subgroups, consider_skewed_dist_when_estimating_with_std, column_with_variable_to_be_analyzed, timestamp_tag_column, column_with_event_frame_indication, rare_event_timedelta_unit, rare_event_indication


class spc_plot:
            
    # Initialize instance attributes.
    # define the Class constructor, i.e., how are its objects:
    def __init__ (self, dictionary, column_with_variable_to_be_analyzed, timestamp_tag_column, chart_to_use, column_with_labels_or_subgroups = None, consider_skewed_dist_when_estimating_with_std = False, rare_event_indication = None, rare_event_timedelta_unit = 'day'):
                
        import numpy as np
        import pandas as pd
        
        # If the user passes the argument, use them. Otherwise, use the standard values.
        # Set the class objects' attributes.
        # Suppose the object is named plot. We can access the attribute as:
        # plot.dictionary, for instance.
        # So, we can save the variables as objects' attributes.
        self.dictionary = dictionary
        self.df = self.dictionary['df']
        # Start the attribute number of labels with value 2 (correspondent to moving range)
        self.number_of_labels = 2
        # List the possible numeric data types for a Pandas dataframe column:
        self.numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        # Start a dictionary of constants
        self.dict_of_constants = {}
        self.column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed
        # Indicate which is the timestamp column:
        self.timestamp_tag_column = timestamp_tag_column
        # Set the chart 
        self.chart_to_use = chart_to_use
        
        # Other arguments of the constructor (attributes):
        # These ones have default values to use if omitted when creating the object
        self.column_with_labels_or_subgroups = column_with_labels_or_subgroups
        self.consider_skewed_dist_when_estimating_with_std = consider_skewed_dist_when_estimating_with_std
        self.rare_event_indication = rare_event_indication 
        self.rare_event_timedelta_unit = rare_event_timedelta_unit
        # to check the class attributes, use the __dict__ method. Examples:
        ## object.__dict__ will show all attributes from object
                
    # Define the class methods.
    # All methods must take an object from the class (self) as one of the parameters
   
    # Define a dictionary of constants.
    # Each key in the dictionary corresponds to a number of samples in a subgroup.
    # number_of_labels - This variable represents the total of labels or subgroups n. 
    # If there are multiple labels, this variable will be updated later.
    
    def get_constants (self):
        
        if (self.number_of_labels < 2):
            
            self.number_of_labels = 2
            
        if (self.number_of_labels <= 25):
            
            dict_of_constants = {
                
                2: {'A':2.121, 'A2':1.880, 'A3':2.659, 'c4':0.7979, '1/c4':1.2533, 'B3':0, 'B4':3.267, 'B5':0, 'B6':2.606, 'd2':1.128, '1/d2':0.8865, 'd3':0.853, 'D1':0, 'D2':3.686, 'D3':0, 'D4':3.267},
                3: {'A':1.732, 'A2':1.023, 'A3':1.954, 'c4':0.8862, '1/c4':1.1284, 'B3':0, 'B4':2.568, 'B5':0, 'B6':2.276, 'd2':1.693, '1/d2':0.5907, 'd3':0.888, 'D1':0, 'D2':4.358, 'D3':0, 'D4':2.574},
                4: {'A':1.500, 'A2':0.729, 'A3':1.628, 'c4':0.9213, '1/c4':1.0854, 'B3':0, 'B4':2.266, 'B5':0, 'B6':2.088, 'd2':2.059, '1/d2':0.4857, 'd3':0.880, 'D1':0, 'D2':4.698, 'D3':0, 'D4':2.282},
                5: {'A':1.342, 'A2':0.577, 'A3':1.427, 'c4':0.9400, '1/c4':1.0638, 'B3':0, 'B4':2.089, 'B5':0, 'B6':1.964, 'd2':2.326, '1/d2':0.4299, 'd3':0.864, 'D1':0, 'D2':4.918, 'D3':0, 'D4':2.114},
                6: {'A':1.225, 'A2':0.483, 'A3':1.287, 'c4':0.9515, '1/c4':1.0510, 'B3':0.030, 'B4':1.970, 'B5':0.029, 'B6':1.874, 'd2':2.534, '1/d2':0.3946, 'd3':0.848, 'D1':0, 'D2':5.078, 'D3':0, 'D4':2.004},
                7: {'A':1.134, 'A2':0.419, 'A3':1.182, 'c4':0.9594, '1/c4':1.0423, 'B3':0.118, 'B4':1.882, 'B5':0.113, 'B6':1.806, 'd2':2.704, '1/d2':0.3698, 'd3':0.833, 'D1':0.204, 'D2':5.204, 'D3':0.076, 'D4':1.924},
                8: {'A':1.061, 'A2':0.373, 'A3':1.099, 'c4':0.9650, '1/c4':1.0363, 'B3':0.185, 'B4':1.815, 'B5':0.179, 'B6':1.751, 'd2':2.847, '1/d2':0.3512, 'd3':0.820, 'D1':0.388, 'D2':5.306, 'D3':0.136, 'D4':1.864},
                9: {'A':1.000, 'A2':0.337, 'A3':1.032, 'c4':0.9693, '1/c4':1.0317, 'B3':0.239, 'B4':1.761, 'B5':0.232, 'B6':1.707, 'd2':2.970, '1/d2':0.3367, 'd3':0.808, 'D1':0.547, 'D2':5.393, 'D3':0.184, 'D4':1.816},
                10: {'A':0.949, 'A2':0.308, 'A3':0.975, 'c4':0.9727, '1/c4':1.0281, 'B3':0.284, 'B4':1.716, 'B5':0.276, 'B6':1.669, 'd2':3.078, '1/d2':0.3249, 'd3':0.797, 'D1':0.687, 'D2':5.469, 'D3':0.223, 'D4':1.777},
                11: {'A':0.905, 'A2':0.285, 'A3':0.927, 'c4':0.9754, '1/c4':1.0252, 'B3':0.321, 'B4':1.679, 'B5':0.313, 'B6':1.637, 'd2':3.173, '1/d2':0.3152, 'd3':0.787, 'D1':0.811, 'D2':5.535, 'D3':0.256, 'D4':1.744},
                12: {'A':0.866, 'A2':0.266, 'A3':0.886, 'c4':0.9776, '1/c4':1.0229, 'B3':0.354, 'B4':1.646, 'B5':0.346, 'B6':1.610, 'd2':3.258, '1/d2':0.3069, 'd3':0.778, 'D1':0.922, 'D2':5.594, 'D3':0.283, 'D4':1.717},
                13: {'A':0.832, 'A2':0.249, 'A3':0.850, 'c4':0.9794, '1/c4':1.0210, 'B3':0.382, 'B4':1.618, 'B5':0.374, 'B6':1.585, 'd2':3.336, '1/d2':0.2998, 'd3':0.770, 'D1':1.025, 'D2':5.647, 'D3':0.307, 'D4':1.693},
                14: {'A':0.802, 'A2':0.235, 'A3':0.817, 'c4':0.9810, '1/c4':1.0194, 'B3':0.406, 'B4':1.594, 'B5':0.399, 'B6':1.563, 'd2':3.407, '1/d2':0.2935, 'd3':0.763, 'D1':1.118, 'D2':5.696, 'D3':0.328, 'D4':1.672},
                15: {'A':0.775, 'A2':0.223, 'A3':0.789, 'c4':0.9823, '1/c4':1.0180, 'B3':0.428, 'B4':1.572, 'B5':0.421, 'B6':1.544, 'd2':3.472, '1/d2':0.2880, 'd3':0.756, 'D1':1.203, 'D2':5.741, 'D3':0.347, 'D4':1.653},
                16: {'A':0.750, 'A2':0.212, 'A3':0.763, 'c4':0.9835, '1/c4':1.0168, 'B3':0.448, 'B4':1.552, 'B5':0.440, 'B6':1.526, 'd2':3.532, '1/d2':0.2831, 'd3':0.750, 'D1':1.282, 'D2':5.782, 'D3':0.363, 'D4':1.637},
                17: {'A':0.728, 'A2':0.203, 'A3':0.739, 'c4':0.9845, '1/c4':1.0157, 'B3':0.466, 'B4':1.534, 'B5':0.458, 'B6':1.511, 'd2':3.588, '1/d2':0.2787, 'd3':0.744, 'D1':1.356, 'D2':5.820, 'D3':0.378, 'D4':1.622},
                18: {'A':0.707, 'A2':0.194, 'A3':0.718, 'c4':0.9854, '1/c4':1.0148, 'B3':0.482, 'B4':1.518, 'B5':0.475, 'B6':1.496, 'd2':3.640, '1/d2':0.2747, 'd3':0.739, 'D1':1.424, 'D2':5.856, 'D3':0.391, 'D4':1.608},
                19: {'A':0.688, 'A2':0.187, 'A3':0.698, 'c4':0.9862, '1/c4':1.0140, 'B3':0.497, 'B4':1.503, 'B5':0.490, 'B6':1.483, 'd2':3.689, '1/d2':0.2711, 'd3':0.734, 'D1':1.487, 'D2':5.891, 'D3':0.403, 'D4':1.597},
                20: {'A':0.671, 'A2':0.180, 'A3':0.680, 'c4':0.9869, '1/c4':1.0133, 'B3':0.510, 'B4':1.490, 'B5':0.504, 'B6':1.470, 'd2':3.735, '1/d2':0.2677, 'd3':0.729, 'D1':1.549, 'D2':5.921, 'D3':0.415, 'D4':1.585},
                21: {'A':0.655, 'A2':0.173, 'A3':0.663, 'c4':0.9876, '1/c4':1.0126, 'B3':0.523, 'B4':1.477, 'B5':0.516, 'B6':1.459, 'd2':3.778, '1/d2':0.2647, 'd3':0.724, 'D1':1.605, 'D2':5.951, 'D3':0.425, 'D4':1.575},
                22: {'A':0.640, 'A2':0.167, 'A3':0.647, 'c4':0.9882, '1/c4':1.0119, 'B3':0.534, 'B4':1.466, 'B5':0.528, 'B6':1.448, 'd2':3.819, '1/d2':0.2618, 'd3':0.720, 'D1':1.659, 'D2':5.979, 'D3':0.434, 'D4':1.566},
                23: {'A':0.626, 'A2':0.162, 'A3':0.633, 'c4':0.9887, '1/c4':1.0114, 'B3':0.545, 'B4':1.455, 'B5':0.539, 'B6':1.438, 'd2':3.858, '1/d2':0.2592, 'd3':0.716, 'D1':1.710, 'D2':6.006, 'D3':0.443, 'D4':1.557},
                24: {'A':0.612, 'A2':0.157, 'A3':0.619, 'c4':0.9892, '1/c4':1.0109, 'B3':0.555, 'B4':1.445, 'B5':0.549, 'B6':1.429, 'd2':3.895, '1/d2':0.2567, 'd3':0.712, 'D1':1.759, 'D2':6.031, 'D3':0.451, 'D4':1.548},
                25: {'A':0.600, 'A2':0.153, 'A3':0.606, 'c4':0.9896, '1/c4':1.0105, 'B3':0.565, 'B4':1.435, 'B5':0.559, 'B6':1.420, 'd2':3.931, '1/d2':0.2544, 'd3':0.708, 'D1':1.806, 'D2':6.056, 'D3':0.459, 'D4':1.541},
            }
            
            # Access the key:
            dict_of_constants = dict_of_constants[self.number_of_labels]
            
        else: #>= 26
            
            dict_of_constants = {'A':(3/(self.number_of_labels**(0.5))), 'A2':0.153, 
                                 'A3':3/((4*(self.number_of_labels-1)/(4*self.number_of_labels-3))*(self.number_of_labels**(0.5))), 
                                 'c4':(4*(self.number_of_labels-1)/(4*self.number_of_labels-3)), 
                                 '1/c4':1/((4*(self.number_of_labels-1)/(4*self.number_of_labels-3))), 
                                 'B3':(1-3/(((4*(self.number_of_labels-1)/(4*self.number_of_labels-3)))*((2*(self.number_of_labels-1))**(0.5)))), 
                                 'B4':(1+3/(((4*(self.number_of_labels-1)/(4*self.number_of_labels-3)))*((2*(self.number_of_labels-1))**(0.5)))),
                                 'B5':(((4*(self.number_of_labels-1)/(4*self.number_of_labels-3)))-3/((2*(self.number_of_labels-1))**(0.5))), 
                                 'B6':(((4*(self.number_of_labels-1)/(4*self.number_of_labels-3)))+3/((2*(self.number_of_labels-1))**(0.5))), 
                                 'd2':3.931, '1/d2':0.2544, 'd3':0.708, 'D1':1.806, 'D2':6.056, 'D3':0.459, 'D4':1.541}
        
        # Update the attribute
        self.dict_of_constants = dict_of_constants
        
        return self
    
    def chart_i_mr (self):
        
        import numpy as np
        import pandas as pd
        
        # access the dataframe:
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        
        # CONTROL LIMIT EQUATIONS:
        # X-bar = (sum of measurements)/(number of measurements)
        # R = Absolute value of [(largest in subgroup) - (lowest in subgroup)]
        # Individual chart: subgroup = 1
        # R = Absolute value of [(data) - (next data)]
        # R-bar = (sum of ranges R)/(number of R values calculated)
        # Lower control limit (LCL) = X-bar - (2.66)R-bar
        # Upper control limit (UCL) = X-bar + (2.66)R-bar
        
        # loop through each row from df, starting from the second (row 1):    
        # calculate mR as the difference (Xmax - Xmin) of the difference between
        # df[column_with_variable_to_be_analyzed] on row i and the row
        # i-1. Since we do not know, in principle, which one is the maximum, we can use
        # the max and min functions from Python:
        # https://www.w3schools.com/python/ref_func_max.asp
        # https://www.w3schools.com/python/ref_func_min.asp
        # Also, the moving range here must be calculated as an absolute value
        # https://www.w3schools.com/python/ref_func_abs.asp
        
        moving_range = [abs(max((df[column_with_variable_to_be_analyzed][i]), (df[column_with_variable_to_be_analyzed][(i-1)])) - min((df[column_with_variable_to_be_analyzed][i]), (df[column_with_variable_to_be_analyzed][(i-1)]))) for i in range (1, len(df))]
        x_bar_list = [(df[column_with_variable_to_be_analyzed][i] + df[column_with_variable_to_be_analyzed][(i-1)])/2 for i in range (1, len(df))]
        
        # These lists were created from index 1. We must add a initial element to
        # make their sizes equal to the original dataset length
        # Start the list to store the moving ranges, containing only the number 0
        # for the moving range (by simple concatenation):
        moving_range = [0] + moving_range
        
        # Start the list that stores the mean values of the 2-elements subgroups
        # with the first element itself (index 0):
        x_bar_list = [df[column_with_variable_to_be_analyzed][0]] + x_bar_list
        
        # Save the moving ranges as a new column from df (it may be interesting to check it):
        df['moving_range'] = moving_range
        
        # Save x_bar_list as the column to be analyzed:
        df[column_with_variable_to_be_analyzed] = x_bar_list
        
        # Get the mean values from x_bar:
        x_bar_bar = df[column_with_variable_to_be_analyzed].mean()
        
        # Calculate the mean value of the column moving_range, and save it as r_bar.
        r_bar = df['moving_range'].mean()
        
        # Get the control chart constant A2 from the dictionary, considering n = 2 the
        # number of elements of each subgroup:
        # Apply the get_constants method to update the dict_of_constants attribute:
        self = self.get_constants()
        
        control_chart_constant = self.dict_of_constants['1/d2']
        control_chart_constant = control_chart_constant * 3
        
        # calculate the upper control limit as x_bar + (3/d2)r_bar:
        upper_cl = x_bar_bar + (control_chart_constant) * (r_bar)
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as x_bar - (3/d2)r_bar:
        lower_cl = x_bar_bar - (control_chart_constant) * (r_bar)
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = x_bar_bar
        
        # Update the dataframe in the dictionary and return it:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self

    def chart_3s (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        
        if(self.consider_skewed_dist_when_estimating_with_std):
            
            # Skewed data. Use the median:
            center = df[column_with_variable_to_be_analyzed].median()
            
        else:
            
            center = dictionary['center']
            
        # calculate the upper control limit as the mean + 3s
        upper_cl = center + 3 * (dictionary['std'])
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as the mean - 3s:
        lower_cl = center - 3 * (dictionary['std'])
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = center
        
        # Update the dataframe in the dictionary:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self
    
    def chart_std_error (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        
        n_samples = df[column_with_variable_to_be_analyzed].count()
        
        s = dictionary['std']
        std_error = s/(n_samples**(0.5))
        
        if(self.consider_skewed_dist_when_estimating_with_std):
            
            # Skewed data. Use the median:
            center = df[column_with_variable_to_be_analyzed].median()
            
        else:
            
            center = dictionary['center']
            
        # calculate the upper control limit as the mean + 3 std_error
        upper_cl = center + 3 * (std_error)
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as the mean - 3 std_error:
        lower_cl = center - 3 * (std_error)
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = center
        
        # Update the dataframe in the dictionary:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self
        
    # CONTROL CHARTS FOR SUBGROUPS 
    
    def create_grouped_df (self):
        
        import numpy as np
        import pandas as pd
        from scipy import stats
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        column_with_labels_or_subgroups = self.column_with_labels_or_subgroups
        numeric_dtypes = self.numeric_dtypes
           
        # We need to group each dataframe in terms of the subgroups stored in the variable
        # column_with_labels_or_subgroups.
        # The catehgorical or datetime columns must be aggregated in terms of mode.
        # The numeric variables must be aggregated both in terms of mean and in terms of count
        # (subgroup size)
        
        # 1. Start a list for categorical columns and other for numeric columns:
        categorical_cols = []
        numeric_cols = []
        
        # Variables to map if there are categorical or numeric variables:
        is_categorical = 0
        is_numeric = 0
        
        # 2. Loop through each column from the list of columns of the dataframe:
        for column in list(df.columns):
            
            # check the type of column:
            column_data_type = df[column].dtype
            
            if (column_data_type not in numeric_dtypes):
                
                # If the Pandas series was defined as an object, it means it is categorical
                # (string, date, etc). Also, this if captures the variables converted to datetime64
                # Append the column to the list of categorical columns:
                categorical_cols.append(column)
                
            else:
                # append the column to the list of numeric columns:
                numeric_cols.append(column)
                
        # 3. Check if column_with_labels_or_subgroups is in both lists. 
        # If it is missing, append it. We need that this column in all subsets for grouping.
        if (column_with_labels_or_subgroups not in categorical_cols):
            
            categorical_cols.append(column_with_labels_or_subgroups)
        
        if (column_with_labels_or_subgroups not in numeric_cols):
            
            numeric_cols.append(column_with_labels_or_subgroups)
        
        if (len(categorical_cols) > 1):    
            # There is at least one column plus column_with_labels_or_subgroups:
            is_categorical = 1
            
        if (len(numeric_cols) > 1):
            # There is at least one column plus column_with_labels_or_subgroups:
            is_numeric = 1
            
        # 4. Create copies of df, subsetting by type of column
        # 5. Group the dataframes by column_with_labels_or_subgroups, 
        # according to the aggregate function:
        if (is_categorical == 1):
            
            df_agg_mode = df.copy(deep = True)
            df_agg_mode = df_agg_mode[categorical_cols]
            df_agg_mode = df_agg_mode.groupby(by = column_with_labels_or_subgroups, as_index = False, sort = True).agg(stats.mode)
            
            # 6. df_agg_mode processing:
            # Loop through each column from this dataframe:
            for col_mode in list(df_agg_mode.columns):
                
                # take the mode for all columns, except the column_with_labels_or_subgroups,
                # used for grouping the dataframe. This column already has the correct value
                if (col_mode != column_with_labels_or_subgroups):
            
                    # start a list of modes:
                    list_of_modes = []

                    # Now, loop through each row from the dataset:
                    for i in range(0, len(df_agg_mode)):
                        # i = 0 to i = len(df_agg_mode) - 1

                        mode_array = df_agg_mode[col_mode][i]    

                        try:
                            # try accessing the mode
                            # mode array is like:
                            # ModeResult(mode=array([calculated_mode]), count=array([counting_of_occurrences]))
                            # To retrieve only the mode, we must access the element [0][0] from this array:
                            mode = mode_array[0][0]

                        except:
                            mode = np.nan

                        # Append it to the list of modes:
                        list_of_modes.append(mode)

                    # Finally, make the column the list of modes itself:
                    df_agg_mode[col_mode] = list_of_modes
                
                    # try to convert to datetime64 (case it is not anymore):
                    try:
                        df_agg_mode[col_mode] = df_agg_mode[col_mode].astype(np.datetime64)    

                    except:
                        # simply ignore this step in case it is not possible to parse
                        # because it is a string:
                        pass
                
        if (is_numeric == 1):
            
            df_agg_mean = df.copy(deep = True)
            df_agg_sum = df.copy(deep = True)
            df_agg_std = df.copy(deep = True)
            df_agg_count = df.copy(deep = True)
            
            df_agg_mean = df_agg_mean[numeric_cols]
            df_agg_sum = df_agg_sum[numeric_cols]
            df_agg_std = df_agg_std[numeric_cols]
            df_agg_count = df_agg_count[numeric_cols]
            
            df_agg_mean = df_agg_mean.groupby(by = column_with_labels_or_subgroups, as_index = False, sort = True).mean()
            df_agg_sum = df_agg_sum.groupby(by = column_with_labels_or_subgroups, as_index = False, sort = True).sum()
            df_agg_std = df_agg_std.groupby(by = column_with_labels_or_subgroups, as_index = False, sort = True).std()
            df_agg_count = df_agg_count.groupby(by = column_with_labels_or_subgroups, as_index = False, sort = True).count()
            # argument as_index = False: prevents the grouper variable to be set as index of the new dataframe.
            # (default: as_index = True).
            
            # 7. df_agg_count processing:
            # Here, all original columns contain only the counting of elements in each
            # label. So, let's select only the columns 'key_for_merging' and column_with_variable_to_be_analyzed:
            df_agg_count = df_agg_count[[column_with_variable_to_be_analyzed]]
            
            # Rename the columns:
            df_agg_count.columns = ['count_of_elements_by_label']
            
            # Analogously, let's keep only the colums column_with_variable_to_be_analyzed and
            # 'key_for_merging' from the dataframes df_agg_sum and df_agg_std, and rename them:
            df_agg_sum = df_agg_sum[[column_with_variable_to_be_analyzed]]
            df_agg_std = df_agg_std[[column_with_variable_to_be_analyzed]]
            
            df_agg_sum.columns = ['sum_of_values_by_label']
            df_agg_std.columns = ['std_of_values_by_label']
            
        if ((is_categorical + is_numeric) == 2):
            # Both subsets are present and the column column_with_labels_or_subgroups
            # is duplicated.
            
            # Remove this column from df_agg_mean:
            df_agg_mean = df_agg_mean.drop(columns = column_with_labels_or_subgroups)
            
            # Concatenate all dataframes:
            df = pd.concat([df_agg_mode, df_agg_mean, df_agg_sum, df_agg_std, df_agg_count], axis = 1, join = "inner")
            
        elif (is_numeric == 1):
            
            # Only the numeric dataframes are present. So, concatenate them:
            df = pd.concat([df_agg_mean, df_agg_sum, df_agg_std, df_agg_count], axis = 1, join = "inner")
            
        elif (is_categorical == 1):
            
            # There is only the categorical dataframe:
            df = df_agg_mode
            
        df = df.reset_index(drop = True)
        
        # Notice that now we have a different mean value: we have a mean value
        # of the means calculated for each subgroup. So, we must update the
        # dictionary information:    
        dictionary['center'] = df[column_with_variable_to_be_analyzed].mean()
        dictionary['sum'] = df[column_with_variable_to_be_analyzed].sum()
        dictionary['std'] = df[column_with_variable_to_be_analyzed].std()
        dictionary['var'] = df[column_with_variable_to_be_analyzed].var()
        dictionary['count'] = len(df) # Total entries from the new dataframe
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        # Notice that the number of labels is now the total of entries of the dataframe
        # grouped by labels
        self.number_of_labels = dictionary['count']
        
        return self
            
    def chart_x_bar_s (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        number_of_labels = self.number_of_labels
        
        # CONTROL LIMIT EQUATIONS:
        # X-bar = mean =  (sum of measurements)/(subgroup size)
        # s = standard deviation in each subgroup
        # s-bar = mean (s) = (sum of all s values)/(number of subgroups)
        # x-bar-bar = mean (x-bar) = (sum of all x-bar)/(number of subgroups)
        # Lower control limit (LCL) = X-bar-bar - (A3)(s-bar)
        # Upper control limit (UCL) = X-bar-bar + (A3)(s-bar) 
        
        s = df['std_of_values_by_label']
        
        s_bar = (s.sum())/(number_of_labels)
        x_bar_bar = dictionary['center']
        
        # Retrieve A3
        self = self.get_constants()
        control_chart_constant = self.dict_of_constants['A3']
        
        # calculate the upper control limit as X-bar-bar + (A3)(s-bar):
        upper_cl = x_bar_bar + (control_chart_constant) * (s_bar)
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as X-bar-bar - (A3)(s-bar):
        lower_cl = x_bar_bar - (control_chart_constant) * (s_bar)
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = x_bar_bar
        
        # Update the dataframe in the dictionary:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self
    
    def chart_p (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        number_of_labels = self.number_of_labels
        
        print("\n")
        print("Attention: before obtaining this chart, substitute the values of the analyzed binary variable by 0 or 1 (integers), or an error will be raised.")
        print("This function do not perform the automatic ordinal or One-Hot Encoding of the variables.\n")
        
        # CONTROL LIMIT EQUATIONS:
        # p-chart: control chart for proportion of defectives.
        # p = mean =  (sum of measurements)/(subgroup size)
        # pbar = (sum of subgroup defective counts)/(sum of subgroups sizes)
        # n = subgroup size
        # Lower control limit (LCL) = pbar - 3.sqrt((pbar)*(1-pbar)/n)
        # Upper control limit (UCL) = pbar + 3.sqrt((pbar)*(1-pbar)/n)
        
        count_per_label = df['count_of_elements_by_label']
        p_bar = (df['sum_of_values_by_label'].sum())/(df['count_of_elements_by_label'].sum())
        
        # calculate the upper control limit as pbar + 3.sqrt((pbar)*(1-pbar)/n):
        upper_cl = p_bar + 3 * (((p_bar)*(1 - p_bar)/(count_per_label))**(0.5))
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as pbar - 3.sqrt((pbar)*(1-pbar)/n):
        lower_cl = p_bar - 3 * (((p_bar)*(1 - p_bar)/(count_per_label))**(0.5))
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = p_bar
        
        # Update the dataframe in the dictionary:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self

    def chart_np (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        number_of_labels = self.number_of_labels
        
        print("\n")
        print("Attention: before obtaining this chart, substitute the values of the analyzed binary variable by 0 or 1 (integers), or an error will be raised.")
        print("This function do not perform the automatic ordinal or One-Hot Encoding of the variables.\n")
        
        # CONTROL LIMIT EQUATIONS:
        # np-chart: control chart for count of defectives.
        # p = mean =  (sum of measurements)/(subgroup size)
        # np = sum = subgroup defective count
        # npbar = (sum of subgroup defective counts)/(number of subgroups)
        # n = subgroup size
        # pbar = npbar/n
        # Center line: npbar
        # Lower control limit (LCL) = np - 3.sqrt((np)*(1-p))
        # Upper control limit (UCL) = np + 3.sqrt((np)*(1-p))
        # available function: **(0.5) - 0.5 power
        
        # p = mean
        p = df[column_with_variable_to_be_analyzed]
        
        # Here, the column that we want to evaluate is not the mean, but the sum.
        # Since the graphics will be plotted using the column column_with_variable_to_be_analyzed
        # Let's make this column equals to the column of sums:
        
        df[column_with_variable_to_be_analyzed] = df['sum_of_values_by_label']
        np_series = df[column_with_variable_to_be_analyzed]
        
        npbar = (df['sum_of_values_by_label'].sum())/(number_of_labels) # center
        
        # calculate the upper control limit as np + 3.sqrt((np)*(1-p)):
        upper_cl = np_series + 3 * (((np_series)*(1 - p))**(0.5))
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as np - 3.sqrt((np)*(1-p)):
        lower_cl = np_series - 3 * (((np_series)*(1 - p))**(0.5))
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = npbar
        
        # Update the dataframe in the dictionary:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self
    
    def chart_c (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        number_of_labels = self.number_of_labels
        
        # CONTROL LIMIT EQUATIONS:
        # c-chart: control chart for counts of occurrences per unit.
        # c = sum = sum of subgroup occurrences
        # cbar = (sum of subgroup occurrences)/(number of subgroups)
        # n = subgroup size
        # Lower control limit (LCL) = cbar - 3.sqrt(cbar)
        # Upper control limit (UCL) = cbar + 3.sqrt(cbar)
        
        # Here, the column that we want to evaluate is not the mean, but the sum.
        # Since the graphics will be plotted using the column column_with_variable_to_be_analyzed
        # Let's make this column equals to the column of sums:
        
        df[column_with_variable_to_be_analyzed] = df['sum_of_values_by_label']
        
        c_bar = (df['sum_of_values_by_label'].sum())/(number_of_labels)
        
        # calculate the upper control limit as cbar + 3.sqrt(cbar):
        upper_cl = c_bar + 3 * ((c_bar)**(0.5))
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as cbar - 3.sqrt(cbar):
        lower_cl = c_bar - 3 * ((c_bar)**(0.5))
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = c_bar
        
        # Update the dataframe in the dictionary:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self
    
    def chart_u (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        number_of_labels = self.number_of_labels
        
        # CONTROL LIMIT EQUATIONS:
        # u-chart: control chart for average occurrence per unit.
        # u = mean =  (subgroup count of occurrences)/(subgroup size, in units)
        # ubar = mean value of u
        # n = subgroup size
        # Lower control limit (LCL) = ubar - 3.sqrt(ubar/n)
        # Upper control limit (UCL) = ubar + 3.sqrt(ubar/n)
        
        count_per_label = df['count_of_elements_by_label']
        
        u_bar = dictionary['center']
        
        # calculate the upper control limit as ubar + 3.sqrt(ubar/n):
        upper_cl = u_bar + 3 * ((u_bar/count_per_label)**(0.5))
        
        # add a column 'upper_cl' on the dataframe with this value:
        df['upper_cl'] = upper_cl
        
        # calculate the lower control limit as ubar - 3.sqrt(ubar/n):
        lower_cl = u_bar - 3 * ((u_bar/count_per_label)**(0.5))
        
        # add a column 'lower_cl' on the dataframe with this value:
        df['lower_cl'] = lower_cl
        
        # Add a column with the mean value of the considered interval:
        df['center'] = u_bar
        
        # Update the dataframe in the dictionary:
        dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self
    
    def rare_events_chart (self):
        
        import numpy as np
        import pandas as pd
        
        dictionary = self.dictionary
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        rare_event_indication = self.rare_event_indication
        rare_event_timedelta_unit = self.rare_event_timedelta_unit
        timestamp_tag_column = self.timestamp_tag_column
        numeric_dtypes = self.numeric_dtypes
        chart_to_use = self.chart_to_use
        
        # Filter df to the rare events:
        rare_events_df = df.copy(deep = True)
        rare_events_df = rare_events_df[rare_events_df[column_with_variable_to_be_analyzed] == rare_event_indication]
        
        # rare_events_df stores only the entries for rare events.
        # Let's get a list of the indices of these entries (we did not reset the index):
        rare_events_indices = list(rare_events_df.index)
        
        # Start lists for storing the count of events between the rares and the time between
        # the rare events. Start both lists from np.nan, since we do not have information of
        # any rare event before the first one registered (np.nan is float).
        count_between_rares = [np.nan]
        timedelta_between_rares = [np.nan]
        
        # Check if the times are datetimes or not:
        column_data_type = df[timestamp_tag_column].dtype
        
        if (column_data_type not in numeric_dtypes):            
            # It is a datetime. Let's loop between successive indices:
            if (len(rare_events_indices) > 1):
                
                for i in range(0, (len(rare_events_indices)-1)):
                    # get the timedelta:
                    index_i = rare_events_indices[i]
                    index_i_plus = rare_events_indices[i + 1]  
                    
                    t_i = pd.Timestamp((df[timestamp_tag_column])[index_i], unit = 'ns')
                    t_i_plus = pd.Timestamp((df[timestamp_tag_column])[index_i_plus], unit = 'ns')
                    
                    # to slice a dataframe from row i to row j (including j): df[i:(j+1)]
                    total_events_between_rares = len(df[(index_i + 1):(index_i_plus)])
                    
                    # We sliced the dataframe from index_i + 1 not to include the rare
                    # event, so we started from the next one. Also, the last element is
                    # of index index_i_plus - 1, the element before the next rare.
                    count_between_rares.append(total_events_between_rares)
                    
                    # Calculate the timedelta:
                    # Convert to an integer representing the total of nanoseconds:
                    timedelta = pd.Timedelta(t_i_plus - t_i).delta
                    
                    if (rare_event_timedelta_unit == 'year'):
                        #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
                        #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
                        timedelta = timedelta / (10**9) #in seconds
                        #2. Convert it to minutes (1 min = 60 s):
                        timedelta = timedelta / 60.0 #in minutes
                        #3. Convert it to hours (1 h = 60 min):
                        timedelta = timedelta / 60.0 #in hours
                        #4. Convert it to days (1 day = 24 h):
                        timedelta = timedelta / 24.0 #in days
                        #5. Convert it to years. 1 year = 365 days + 6 h = 365 days + 6/24 h/(h/day)
                        # = (365 + 1/4) days = 365.25 days
                        timedelta = timedelta / (365.25) #in years
                        #The .0 after the numbers guarantees a float division.
                        
                    elif (rare_event_timedelta_unit == 'month'):
                        #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
                        #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
                        timedelta = timedelta / (10**9) #in seconds
                        #2. Convert it to minutes (1 min = 60 s):
                        timedelta = timedelta / 60.0 #in minutes
                        #3. Convert it to hours (1 h = 60 min):
                        timedelta = timedelta / 60.0 #in hours
                        #4. Convert it to days (1 day = 24 h):
                        timedelta = timedelta / 24.0 #in days
                        #5. Convert it to months. Consider 1 month = 30 days
                        timedelta = timedelta / (30.0) #in months
                        #The .0 after the numbers guarantees a float division.
                        
                    elif (rare_event_timedelta_unit == 'day'):
                        #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
                        #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
                        timedelta = timedelta / (10**9) #in seconds
                        #2. Convert it to minutes (1 min = 60 s):
                        timedelta = timedelta / 60.0 #in minutes
                        #3. Convert it to hours (1 h = 60 min):
                        timedelta = timedelta / 60.0 #in hours
                        #4. Convert it to days (1 day = 24 h):
                        timedelta = timedelta / 24.0 #in days
                        
                    elif (rare_event_timedelta_unit == 'hour'):
                        #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
                        #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
                        timedelta = timedelta / (10**9) #in seconds
                        #2. Convert it to minutes (1 min = 60 s):
                        timedelta = timedelta / 60.0 #in minutes
                        #3. Convert it to hours (1 h = 60 min):
                        timedelta = timedelta / 60.0 #in hours
                        
                    elif (rare_event_timedelta_unit == 'minute'):
                        #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
                        #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
                        timedelta = timedelta / (10**9) #in seconds
                        #2. Convert it to minutes (1 min = 60 s):
                        timedelta = timedelta / 60.0 #in minutes
                        
                    elif (rare_event_timedelta_unit == 'second'):
                        #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
                        #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
                        timedelta = timedelta / (10**9) #in seconds
                        
                    else:
                        timedelta = timedelta # nanoseconds.
                    
                    # Append the timedelta to the list:
                    timedelta_between_rares.append(timedelta)
            
            else:
                # There is a single rare event.
                print("There is a single rare event. Impossible to calculate timedeltas and counting between rare events.\n")
                return self
        
        else: 
            # The column is not a timestamp. Simply subtract the values to calculate the
            # timedeltas. Let's loop between successive indices:
            if (len(rare_events_indices) > 1):
                
                for i in range(0, (len(rare_events_indices)-1)):
                    
                    # get the timedelta:
                    index_i = rare_events_indices[i]
                    index_i_plus = rare_events_indices[i + 1]
                    
                    t_i = (df[timestamp_tag_column])[index_i]
                    t_i_plus = (df[timestamp_tag_column])[index_i_plus]
                    
                    timedelta = (t_i_plus - t_i)
                    
                    # to slice a dataframe from row i to row j (including j): df[i:(j+1)] 
                    total_events_between_rares= len(df[(index_i + 1):(index_i_plus)])
                    
                    count_between_rares.append(total_events_between_rares)
                    timedelta_between_rares.append(timedelta)
            else:
                # There is a single rare event.
                print("There is a single rare event. Impossible to calculate timedeltas and counting between rare events.\n")
                return self
            
        # Notice that the lists still have same number of elements of the dataframe of rares.
        
        # Now, lists have the same total elements of the rare_events_df, and can be
        # added as columns:        
        # firstly, reset the index:
        rare_events_df = rare_events_df.reset_index(drop = True)
        
        # Add the columns:
        rare_events_df['count_between_rares'] = count_between_rares
        rare_events_df['timedelta_between_rares'] = timedelta_between_rares
        
        # Now, make the rares dataframe the df itself:
        df = rare_events_df
        
        if (chart_to_use == 'g'):
            
            # Here, the column that we want to evaluate is not the mean, but the 'count_between_rares'.
            # Since the graphics will be plotted using the column column_with_variable_to_be_analyzed
            # Let's make this column equals to the column 'count_between_rares':
            df[column_with_variable_to_be_analyzed] = df['count_between_rares']
            
            g_bar = df['count_between_rares'].median()
            n_samples = len(df['count_between_rares'])
            
            try:
                p = (1/(g_bar + 1))*((n_samples - 1)/n_samples)
            
                # np.log = natural logarithm
                # https://numpy.org/doc/stable/reference/generated/numpy.log.html
                center = ((np.log(0.5))/(np.log(1 - p))) - 1

                # calculate the upper control limit as log(0.00135)/log(1-p)-1:
                upper_cl = ((np.log(0.00135))/(np.log(1 - p))) - 1

                # calculate the lower control limit as Max(0, log(1-0.00135)/log(1-p)-1):
                lower_cl = max(0, ((np.log(1 - 0.00135))/(np.log(1 - p)) - 1))
            
            except:
                # division by zero
                # Here, we are prone to it due to the obtention of the rare dataframe.
                p = np.nan
                center = np.nan
                upper_cl = np.nan
                lower_cl = np.nan
                
            # add a column 'lower_cl' on the dataframe with this value:
            df['lower_cl'] = lower_cl
            
            # add a column 'upper_cl' on the dataframe with this value:
            df['upper_cl'] = upper_cl
            
            # Add a column with the mean value of the considered interval:
            df['center'] = center
            
            # Update the dataframe in the dictionary:
            dictionary['df'] = df
            
        elif (chart_to_use == 't'):
            
            # Here, the column that we want to evaluate is not the mean, but the 'timedelta_between_rares'.
            # Since the graphics will be plotted using the column column_with_variable_to_be_analyzed
            # Let's make this column equals to the column 'timedelta_between_rares':
            df[column_with_variable_to_be_analyzed] = df['timedelta_between_rares']
            
            # Create the transformed series:
            # y = df['timedelta_between_rares']
            # y_transf = y**(1/3.6)
            y_transf = (df['timedelta_between_rares'])**(1/(3.6))
            # Now, let's create an I-MR chart for y_transf        
            moving_range = [abs(max((y_transf[i]), (y_transf[(i-1)])) - min((y_transf[i]), (y_transf[(i-1)]))) for i in range (1, len(y_transf))]
            
            # The first y_transf is np.nan. We cannot use it for calculating the averages.
            # That is because the average with np.nan is np.nan. So, let's skip the first entry,
            # by starting from index i = 2.
            y_bar = [(y_transf[i] + y_transf[(i-1)])/2 for i in range (2, len(y_transf))]
            # These lists were created from index 1. We must add a initial element to
            # make their sizes equal to the original dataset length
            
            # Start the list to store the moving ranges, containing only the number 0
            # for the moving range (by simple concatenation):
            moving_range = [0] + moving_range
            # Since we do not have the first element (which is np.nan), with index 0, let's
            # take the average corresponding to index 1, the first valid entry, as the y_transf[1]
            # itself:
            y_bar = [y_transf[1]] + y_bar
            # Notice that y_bar list did not start from index 0, but from index 1, so it has one
            # element less than moving_range list. With this strategy, we eliminated the null element
            # from the calculation of the mean.
            
            # The presence of other missing values in these lists will turn all results NaN.
            # So, let's convert the list to pandas series, and apply the mean method, which
            # automatically ignores the missing values from calculations (the function
            # np.average for np.arrays would return NaN)
            moving_range = pd.Series(moving_range)
            y_bar = pd.Series(y_bar)
            
            # Now we can get the mean values from y_bar list and moving_range:
            y_bar_bar = y_bar.mean()
            r_bar = moving_range.mean()
            # Get the control chart constant A2 from the dictionary, considering n = 2 the
            # number of elements of each subgroup:
            
            # Update the number of labels attribute for the moving range case
            self.number_of_labels = 2
            self = self.get_constants()
            
            control_chart_constant = self.dict_of_constants['1/d2']
            control_chart_constant = control_chart_constant * 3
            
            # calculate the upper control limit as y_bar_bar + (3/d2)r_bar:
            upper_cl_transf = y_bar_bar + (control_chart_constant) * (r_bar)
        
            # calculate the lower control limit as y_bar_bar - (3/d2)r_bar:
            lower_cl_transf = y_bar_bar - (control_chart_constant) * (r_bar)
          
            # Notice that these values are for the transformed variables:
            # y_transf = (df['timedelta_between_rares'])**(1/(3.6))
            
            # To reconvert to the correct time scale, we reverse this transform as:
            # (y_transf)**(3.6)
            
            # add a column 'upper_cl' on the dataframe with upper_cl_transf
            # converted to the original scale:
            df['upper_cl'] = (upper_cl_transf)**(3.6)
            
            # add a column 'lower_cl' on the dataframe with lower_cl_transf
            # converted to the original scale:
            df['lower_cl'] = (lower_cl_transf)**(3.6)
            
            # Finally, add the central line by reconverting y_bar_bar to the
            # original scale:
            df['center'] = (y_bar_bar)**(3.6)
            
            # Notice that this procedure naturally corrects the deviations caused by
            # the skewness of the distribution. Actually, log and exponential transforms
            # tend to reduce the skewness and to normalize the data.
            # Update the dataframe in the dictionary:
            dictionary['df'] = df
        
        # Update the attributes:
        self.dictionary = dictionary
        self.df = df
        
        return self


class capability_analysis:
            
    # Initialize instance attributes.
    # define the Class constructor, i.e., how are its objects:
    def __init__ (self, df, column_with_variable_to_be_analyzed, specification_limits, total_of_bins = 10, alpha = 0.10):
                
        import numpy as np
        import pandas as pd
        
        # If the user passes the argument, use them. Otherwise, use the standard values.
        # Set the class objects' attributes.
        # Suppose the object is named plot. We can access the attribute as:
        # plot.dictionary, for instance.
        # So, we can save the variables as objects' attributes.
        self.df = df
        self.column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed
        self.specification_limits = specification_limits
        self.sample_size = df[column_with_variable_to_be_analyzed].count()
        self.mu = (df[column_with_variable_to_be_analyzed]).mean() 
        self.median = (df[column_with_variable_to_be_analyzed]).median()
        self.sigma = (df[column_with_variable_to_be_analyzed]).std()
        self.lowest = (df[column_with_variable_to_be_analyzed]).min()
        self.highest = (df[column_with_variable_to_be_analyzed]).max()
        self.total_of_bins = total_of_bins
        self.alpha = alpha
        
        # Start a dictionary of constants
        self.dict_of_constants = {}
        # Get parameters to update later:
        self.histogram_dict = {}
        self.capability_dict = {}
        self.normality_dict = {}
        
        print("WARNING: this capability analysis is based on the strong hypothesis that data follows the normal (Gaussian) distribution.\n")
        
    # Define the class methods.
    # All methods must take an object from the class (self) as one of the parameters
   
    # Define a dictionary of constants.
    # Each key in the dictionary corresponds to a number of samples in a subgroup.
    # sample_size - This variable represents the total of labels or subgroups n. 
    # If there are multiple labels, this variable will be updated later.
    
    def check_data_normality (self):
        
        import numpy as np
        import pandas as pd
        from scipy import stats
        from statsmodels.stats import diagnostic
        
        alpha = self.alpha
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        sample_size = self.sample_size
        mu = self.mu 
        median = self.median
        sigma = self.sigma
        lowest = self.lowest
        highest = self.highest
        normality_dict = self.normality_dict # empty dictionary 
        
        print("WARNING: The statistical tests require at least 20 samples.\n")
        print("Interpretation:")
        print("p-value: probability that data is described by the normal distribution.")
        print("Criterion: the series is not described by normal if p < alpha = %.3f." %(alpha))
        
        if (sample_size < 20):
            
            print(f"Unable to test series normality: at least 20 samples are needed, but found only {sample_size} entries for this series.\n")
            normality_dict['WARNING'] = "Series without the minimum number of elements (20) required to test the normality."
            
        else:
            # Let's test the series.
            y = df[column_with_variable_to_be_analyzed]
            
            # Scipy.stats’ normality test
            # It is based on D’Agostino and Pearson’s test that combines 
            # skew and kurtosis to produce an omnibus test of normality.
            _, scipystats_test_pval = stats.normaltest(y)
            # The underscore indicates an output to be ignored, which is s^2 + k^2, 
            # where s is the z-score returned by skewtest and k is the z-score returned by kurtosistest.
            # https://docs.scipy.org/doc/scipy-1.8.0/html-scipyorg/reference/generated/scipy.stats.normaltest.html
            
            print("\n")
            print("D\'Agostino and Pearson\'s normality test (scipy.stats normality test):")
            print(f"p-value = {scipystats_test_pval:e} = {scipystats_test_pval*100:.2f}% of probability of being normal.")
            # :e indicates the scientific notation; .2f: float with 2 decimal cases
            
            if (scipystats_test_pval < alpha):
                
                print("p = %.3f < %.3f" %(scipystats_test_pval, alpha))
                print(f"According to this test, data is not described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            else:
                
                print("p = %.3f >= %.3f" %(scipystats_test_pval, alpha))
                print(f"According to this test, data is described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            # add this test result to the dictionary:
            normality_dict['dagostino_pearson_p_val'] = scipystats_test_pval
            normality_dict['dagostino_pearson_p_in_pct'] = scipystats_test_pval*100
            
            # Scipy.stats’ Shapiro-Wilk test
            # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.shapiro.html
            shapiro_test = stats.shapiro(y)
            # returns ShapiroResult(statistic=0.9813305735588074, pvalue=0.16855233907699585)
             
            print("\n")
            print("Shapiro-Wilk normality test:")
            print(f"p-value = {shapiro_test[1]:e} = {(shapiro_test[1])*100:.2f}% of probability of being normal.")
            
            if (shapiro_test[1] < alpha):
                
                print("p = %.3f < %.3f" %(shapiro_test[1], alpha))
                print(f"According to this test, data is not described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            else:
                
                print("p = %.3f >= %.3f" %(shapiro_test[1], alpha))
                print(f"According to this test, data is described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            # add this test result to the dictionary:
            normality_dict['shapiro_wilk_p_val'] = shapiro_test[1]
            normality_dict['shapiro_wilk_p_in_pct'] = (shapiro_test[1])*100
            
            # Lilliefors’ normality test
            lilliefors_test = diagnostic.kstest_normal(y, dist = 'norm', pvalmethod = 'table')
            # Returns a tuple: index 0: ksstat: float
            # Kolmogorov-Smirnov test statistic with estimated mean and variance.
            # index 1: p-value:float
            # If the pvalue is lower than some threshold, e.g. 0.10, then we can reject the Null hypothesis that the sample comes from a normal distribution.
            
            print("\n")
            print("Lilliefors\'s normality test:")
            print(f"p-value = {lilliefors_test[1]:e} = {(lilliefors_test[1])*100:.2f}% of probability of being normal.")
            
            if (lilliefors_test[1] < alpha):
                
                print("p = %.3f < %.3f" %(lilliefors_test[1], alpha))
                print(f"According to this test, data is not described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            else:
                
                print("p = %.3f >= %.3f" %(lilliefors_test[1], alpha))
                print(f"According to this test, data is described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            # add this test result to the dictionary:
            normality_dict['lilliefors_p_val'] = lilliefors_test[1]
            normality_dict['lilliefors_p_in_pct'] = (lilliefors_test[1])*100

            # Anderson-Darling normality test
            ad_test = diagnostic.normal_ad(y, axis = 0)
            # Returns a tuple: index 0 - ad2: float
            # Anderson Darling test statistic.
            # index 1 - p-val: float
            # The p-value for hypothesis that the data comes from a normal distribution with unknown mean and variance.
            
            print("\n")
            print("Anderson-Darling (AD) normality test:")
            print(f"p-value = {ad_test[1]:e} = {(ad_test[1])*100:.2f}% of probability of being normal.")
            
            if (ad_test[1] < alpha):
                
                print("p = %.3f < %.3f" %(ad_test[1], alpha))
                print(f"According to this test, data is not described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            else:
                
                print("p = %.3f >= %.3f" %(ad_test[1], alpha))
                print(f"According to this test, data is described by the normal distribution, for the {alpha*100}% confidence level defined.")
            
            # add this test result to the dictionary:
            normality_dict['anderson_darling_p_val'] = ad_test[1]
            normality_dict['anderson_darling_p_in_pct'] = (ad_test[1])*100
            
            # Update the attribute:
            self.normality_dict = normality_dict
            
            return self
    
    def get_constants (self):
        
        if (self.sample_size < 2):
            
            self.sample_size = 2
            
        if (self.sample_size <= 25):
            
            dict_of_constants = {
                
                2: {'A':2.121, 'A2':1.880, 'A3':2.659, 'c4':0.7979, '1/c4':1.2533, 'B3':0, 'B4':3.267, 'B5':0, 'B6':2.606, 'd2':1.128, '1/d2':0.8865, 'd3':0.853, 'D1':0, 'D2':3.686, 'D3':0, 'D4':3.267},
                3: {'A':1.732, 'A2':1.023, 'A3':1.954, 'c4':0.8862, '1/c4':1.1284, 'B3':0, 'B4':2.568, 'B5':0, 'B6':2.276, 'd2':1.693, '1/d2':0.5907, 'd3':0.888, 'D1':0, 'D2':4.358, 'D3':0, 'D4':2.574},
                4: {'A':1.500, 'A2':0.729, 'A3':1.628, 'c4':0.9213, '1/c4':1.0854, 'B3':0, 'B4':2.266, 'B5':0, 'B6':2.088, 'd2':2.059, '1/d2':0.4857, 'd3':0.880, 'D1':0, 'D2':4.698, 'D3':0, 'D4':2.282},
                5: {'A':1.342, 'A2':0.577, 'A3':1.427, 'c4':0.9400, '1/c4':1.0638, 'B3':0, 'B4':2.089, 'B5':0, 'B6':1.964, 'd2':2.326, '1/d2':0.4299, 'd3':0.864, 'D1':0, 'D2':4.918, 'D3':0, 'D4':2.114},
                6: {'A':1.225, 'A2':0.483, 'A3':1.287, 'c4':0.9515, '1/c4':1.0510, 'B3':0.030, 'B4':1.970, 'B5':0.029, 'B6':1.874, 'd2':2.534, '1/d2':0.3946, 'd3':0.848, 'D1':0, 'D2':5.078, 'D3':0, 'D4':2.004},
                7: {'A':1.134, 'A2':0.419, 'A3':1.182, 'c4':0.9594, '1/c4':1.0423, 'B3':0.118, 'B4':1.882, 'B5':0.113, 'B6':1.806, 'd2':2.704, '1/d2':0.3698, 'd3':0.833, 'D1':0.204, 'D2':5.204, 'D3':0.076, 'D4':1.924},
                8: {'A':1.061, 'A2':0.373, 'A3':1.099, 'c4':0.9650, '1/c4':1.0363, 'B3':0.185, 'B4':1.815, 'B5':0.179, 'B6':1.751, 'd2':2.847, '1/d2':0.3512, 'd3':0.820, 'D1':0.388, 'D2':5.306, 'D3':0.136, 'D4':1.864},
                9: {'A':1.000, 'A2':0.337, 'A3':1.032, 'c4':0.9693, '1/c4':1.0317, 'B3':0.239, 'B4':1.761, 'B5':0.232, 'B6':1.707, 'd2':2.970, '1/d2':0.3367, 'd3':0.808, 'D1':0.547, 'D2':5.393, 'D3':0.184, 'D4':1.816},
                10: {'A':0.949, 'A2':0.308, 'A3':0.975, 'c4':0.9727, '1/c4':1.0281, 'B3':0.284, 'B4':1.716, 'B5':0.276, 'B6':1.669, 'd2':3.078, '1/d2':0.3249, 'd3':0.797, 'D1':0.687, 'D2':5.469, 'D3':0.223, 'D4':1.777},
                11: {'A':0.905, 'A2':0.285, 'A3':0.927, 'c4':0.9754, '1/c4':1.0252, 'B3':0.321, 'B4':1.679, 'B5':0.313, 'B6':1.637, 'd2':3.173, '1/d2':0.3152, 'd3':0.787, 'D1':0.811, 'D2':5.535, 'D3':0.256, 'D4':1.744},
                12: {'A':0.866, 'A2':0.266, 'A3':0.886, 'c4':0.9776, '1/c4':1.0229, 'B3':0.354, 'B4':1.646, 'B5':0.346, 'B6':1.610, 'd2':3.258, '1/d2':0.3069, 'd3':0.778, 'D1':0.922, 'D2':5.594, 'D3':0.283, 'D4':1.717},
                13: {'A':0.832, 'A2':0.249, 'A3':0.850, 'c4':0.9794, '1/c4':1.0210, 'B3':0.382, 'B4':1.618, 'B5':0.374, 'B6':1.585, 'd2':3.336, '1/d2':0.2998, 'd3':0.770, 'D1':1.025, 'D2':5.647, 'D3':0.307, 'D4':1.693},
                14: {'A':0.802, 'A2':0.235, 'A3':0.817, 'c4':0.9810, '1/c4':1.0194, 'B3':0.406, 'B4':1.594, 'B5':0.399, 'B6':1.563, 'd2':3.407, '1/d2':0.2935, 'd3':0.763, 'D1':1.118, 'D2':5.696, 'D3':0.328, 'D4':1.672},
                15: {'A':0.775, 'A2':0.223, 'A3':0.789, 'c4':0.9823, '1/c4':1.0180, 'B3':0.428, 'B4':1.572, 'B5':0.421, 'B6':1.544, 'd2':3.472, '1/d2':0.2880, 'd3':0.756, 'D1':1.203, 'D2':5.741, 'D3':0.347, 'D4':1.653},
                16: {'A':0.750, 'A2':0.212, 'A3':0.763, 'c4':0.9835, '1/c4':1.0168, 'B3':0.448, 'B4':1.552, 'B5':0.440, 'B6':1.526, 'd2':3.532, '1/d2':0.2831, 'd3':0.750, 'D1':1.282, 'D2':5.782, 'D3':0.363, 'D4':1.637},
                17: {'A':0.728, 'A2':0.203, 'A3':0.739, 'c4':0.9845, '1/c4':1.0157, 'B3':0.466, 'B4':1.534, 'B5':0.458, 'B6':1.511, 'd2':3.588, '1/d2':0.2787, 'd3':0.744, 'D1':1.356, 'D2':5.820, 'D3':0.378, 'D4':1.622},
                18: {'A':0.707, 'A2':0.194, 'A3':0.718, 'c4':0.9854, '1/c4':1.0148, 'B3':0.482, 'B4':1.518, 'B5':0.475, 'B6':1.496, 'd2':3.640, '1/d2':0.2747, 'd3':0.739, 'D1':1.424, 'D2':5.856, 'D3':0.391, 'D4':1.608},
                19: {'A':0.688, 'A2':0.187, 'A3':0.698, 'c4':0.9862, '1/c4':1.0140, 'B3':0.497, 'B4':1.503, 'B5':0.490, 'B6':1.483, 'd2':3.689, '1/d2':0.2711, 'd3':0.734, 'D1':1.487, 'D2':5.891, 'D3':0.403, 'D4':1.597},
                20: {'A':0.671, 'A2':0.180, 'A3':0.680, 'c4':0.9869, '1/c4':1.0133, 'B3':0.510, 'B4':1.490, 'B5':0.504, 'B6':1.470, 'd2':3.735, '1/d2':0.2677, 'd3':0.729, 'D1':1.549, 'D2':5.921, 'D3':0.415, 'D4':1.585},
                21: {'A':0.655, 'A2':0.173, 'A3':0.663, 'c4':0.9876, '1/c4':1.0126, 'B3':0.523, 'B4':1.477, 'B5':0.516, 'B6':1.459, 'd2':3.778, '1/d2':0.2647, 'd3':0.724, 'D1':1.605, 'D2':5.951, 'D3':0.425, 'D4':1.575},
                22: {'A':0.640, 'A2':0.167, 'A3':0.647, 'c4':0.9882, '1/c4':1.0119, 'B3':0.534, 'B4':1.466, 'B5':0.528, 'B6':1.448, 'd2':3.819, '1/d2':0.2618, 'd3':0.720, 'D1':1.659, 'D2':5.979, 'D3':0.434, 'D4':1.566},
                23: {'A':0.626, 'A2':0.162, 'A3':0.633, 'c4':0.9887, '1/c4':1.0114, 'B3':0.545, 'B4':1.455, 'B5':0.539, 'B6':1.438, 'd2':3.858, '1/d2':0.2592, 'd3':0.716, 'D1':1.710, 'D2':6.006, 'D3':0.443, 'D4':1.557},
                24: {'A':0.612, 'A2':0.157, 'A3':0.619, 'c4':0.9892, '1/c4':1.0109, 'B3':0.555, 'B4':1.445, 'B5':0.549, 'B6':1.429, 'd2':3.895, '1/d2':0.2567, 'd3':0.712, 'D1':1.759, 'D2':6.031, 'D3':0.451, 'D4':1.548},
                25: {'A':0.600, 'A2':0.153, 'A3':0.606, 'c4':0.9896, '1/c4':1.0105, 'B3':0.565, 'B4':1.435, 'B5':0.559, 'B6':1.420, 'd2':3.931, '1/d2':0.2544, 'd3':0.708, 'D1':1.806, 'D2':6.056, 'D3':0.459, 'D4':1.541},
            }
            
            # Access the key:
            dict_of_constants = dict_of_constants[self.sample_size]
            
        else: #>= 26
            
            dict_of_constants = {'A':(3/(self.sample_size**(0.5))), 'A2':0.153, 
                                 'A3':3/((4*(self.sample_size-1)/(4*self.sample_size-3))*(self.sample_size**(0.5))), 
                                 'c4':(4*(self.sample_size-1)/(4*self.sample_size-3)), 
                                 '1/c4':1/((4*(self.sample_size-1)/(4*self.sample_size-3))), 
                                 'B3':(1-3/(((4*(self.sample_size-1)/(4*self.sample_size-3)))*((2*(self.sample_size-1))**(0.5)))), 
                                 'B4':(1+3/(((4*(self.sample_size-1)/(4*self.sample_size-3)))*((2*(self.sample_size-1))**(0.5)))),
                                 'B5':(((4*(self.sample_size-1)/(4*self.sample_size-3)))-3/((2*(self.sample_size-1))**(0.5))), 
                                 'B6':(((4*(self.sample_size-1)/(4*self.sample_size-3)))+3/((2*(self.sample_size-1))**(0.5))), 
                                 'd2':3.931, '1/d2':0.2544, 'd3':0.708, 'D1':1.806, 'D2':6.056, 'D3':0.459, 'D4':1.541}
        
        # Update the attribute
        self.dict_of_constants = dict_of_constants
        
        return self
    
    def get_histogram_array (self):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        y_hist = df[column_with_variable_to_be_analyzed]
        lowest = self.lowest
        highest = self.highest
        sample_size = self.sample_size
        
        # Number of bins set by the user:
        total_of_bins = self.total_of_bins
        
        # Firstly, get the ideal bin-size according to the Montgomery's method:
        # Douglas C. Montgomery (2009). Introduction to Statistical Process Control, 
        # Sixth Edition, John Wiley & Sons.
        # Sort by the column to analyze (ascending order) and reset the index:
        y_hist = y_hist.sort_values(ascending = True)
        y_hist = y_hist.reset_index(drop = True)
        #Calculo do bin size - largura do histograma:
        #1: Encontrar o menor (lowest) e o maior (highest) valor dentro da tabela de dados)
        #2: Calcular rangehist = highest - lowest
        #3: Calcular quantidade de dados (samplesize) de entrada fornecidos
        #4: Calcular a quantidade de celulas da tabela de frequencias (ncells)
        #ncells = numero inteiro mais proximo da (raiz quadrada de samplesize)
        #5: Calcular binsize = (df[column_to_analyze])rangehist/(ncells)
        #ATENCAO: Nao se esquecer de converter range, ncells, samplesize e binsize para valores absolutos (modulos)
        #isso porque a largura do histograma tem que ser um numero positivo 

        # bin-size
        range_hist = abs(highest - lowest)
        n_cells = int(np.rint((sample_size)**(0.5)))
        # We must use the int function to guarantee that the ncells will store an
        # integer number of cells (we cannot have a fraction of a sentence).
        # The int function guarantees that the variable will be stored as an integer.
        # The numpy.rint(a) function rounds elements of the array to the nearest integer.
        # https://numpy.org/doc/stable/reference/generated/numpy.rint.html
        # For values exactly halfway between rounded decimal values, 
        # NumPy rounds to the nearest even value. 
        # Thus 1.5 and 2.5 round to 2.0; -0.5 and 0.5 round to 0.0; etc.
        if (n_cells > 3):
            
            print(f"Ideal number of histogram bins calculated through Montgomery's method = {n_cells} bins.\n")
        
        # Retrieve the histogram array hist_array
        fig, ax = plt.subplots() # (0,0) not to show the plot now:
        
        # Get a 10-bins histogram:
        hist_array = plt.hist(y_hist, bins = total_of_bins)
        plt.delaxes(ax) # this will delete ax, so that it will not be plotted.
        plt.show()
        print("") # use this print not to mix with the final plot

        # hist_array is an array of arrays:
        # hist_array = (array([count_1, count_2, ..., cont_n]), array([bin_center_1,...,
        # bin_center_n])), where n = total_of_bins
        # hist_array[0] is the array of countings for each bin, whereas hist_array[1] is
        # the array of the bin center, i.e., the central value of the analyzed variable for
        # that bin.

        # It is possible that the hist_array[0] contains more elements than hist_array[1].
        # This happens when the last bins created by the division contain zero elements.
        # In this case, we have to pad the sequence of hist_array[0], completing it with zeros.

        MAX_LENGTH = max(len(hist_array[0]), len(hist_array[1])) # Get the length of the longest sequence
        SEQUENCES = [list(hist_array[0]), list(hist_array[1])] # get a list of sequences to pad.
        # Notice that we applied the list attribute to create a list of lists

        # We cannot pad with the function pad_sequences from tensorflow because it converts all values
        # to integers. Then, we have to pad the sequences by looping through the elements from SEQUENCES:

        # Start a support_list
        support_list = []

        # loop through each sequence in SEQUENCES:
        for sequence in SEQUENCES:
            # add a zero at the end of the sequence until its length reaches MAX_LENGTH
            while (len(sequence) < MAX_LENGTH):

                sequence.append(0)

            # append the sequence to support_list:
            support_list.append(sequence)

        # Tuples and arrays are immutable. It means they do not support assignment, i.e., we cannot
        # do tuple[0] = variable. Since arrays support vectorial (element-wise) operations, we can
        # modify the whole array making it equals to support_list at once by using function np.array:
        hist_array = np.array(support_list)

        # Get the bin_size as the average difference between successive elements from support_list[1]:

        diff_lists = []

        for i in range (1, len(support_list[1])):

            diff_lists.append(support_list[1][i] - support_list[1][(i-1)])

        # Now, get the mean value as the bin_size:
        bin_size = np.amax(np.array(diff_lists))

        # Let's get the frequency table, which will be saved on DATASET (to get the code
        # equivalent to the code for the function 'histogram'):

        DATASET = pd.DataFrame(data = {'bin_center': hist_array[1], 'count': hist_array[0]})

        # Get a lists of bin_center and column_to_analyze:
        list_of_bins = list(hist_array[1])
        list_of_counts = list(hist_array[0])

        # get the maximum count:
        max_count = DATASET['count'].max()
        # Get the index of the max count:
        max_count_index = list_of_counts.index(max_count)

        # Get the value bin_center correspondent to the max count (maximum probability):
        bin_of_max_proba = list_of_bins[max_count_index]
        bin_after_the_max_proba = list_of_bins[(max_count_index + 1)] # the next bin
        number_of_bins = len(DATASET) # Total of elements on the frequency table
        
        # Obtain a list of differences between bins
        bins_diffs = [(list_of_bins[i] - list_of_bins[(i-1)]) for i in range (1, len(list_of_bins))]
        # Convert it to Pandas series and use the mean method to retrieve the average bin size:
        bin_size = pd.Series(bins_diffs).mean()
        
        self.histogram_dict = {'df': DATASET, 'list_of_bins': list_of_bins, 'list_of_counts': list_of_counts,
                              'max_count': max_count, 'max_count_index': max_count_index,
                              'bin_of_max_proba': bin_of_max_proba, 'bin_after_the_max_proba': bin_after_the_max_proba,
                              'number_of_bins': number_of_bins, 'bin_size': bin_size}
        
        return self
    
    def get_desired_normal (self):
        
        import numpy as np
        import pandas as pd
        
        # Get a normal completely (6s) in the specifications, and centered
        # within these limits
        
        mu = self.mu
        sigma = self.sigma
        histogram_dict = self.histogram_dict
        max_count = histogram_dict['max_count']
        
        specification_limits = self.specification_limits
        
        lower_spec = specification_limits['lower_spec_lim']
        upper_spec = specification_limits['upper_spec_lim']
        
        if (lower_spec is None):
            
            # There is no lower specification: everything below it is in the specifications.
            # Make it mean - 6sigma (virtually infinite).
            lower_spec = mu - 6*(sigma)
            # Update the dictionary:
            specification_limits['lower_spec_lim'] = lower_spec
        
        if (upper_spec is None):
            
            # There is no upper specification: everything above it is in the specifications.
            # Make it mean + 6sigma (virtually infinite).
            upper_spec = mu + 6*(sigma)
            # Update the dictionary:
            specification_limits['upper_spec_lim'] = upper_spec
        
        # Desired normal mu: center of the specification limits.
        desired_mu = (lower_spec + upper_spec)/2
        
        # Desired sigma: 6 times the variation within the specific limits
        desired_sigma = (upper_spec - lower_spec)/6
        
        if (desired_sigma == 0):
            print("Impossible to obtain a normal curve overlayed, because the standard deviation is zero.\n")
            print("The analyzed variable is constant throughout the whole sample space.\n")
            
            # Get a dictionary of empty lists for this case
            desired_normal = {'x': [], 'y':[]}
            
        else:
            # create lists to store the normal curve. Center the normal curve in the bin
            # of maximum bar (max probability, which will not be the mean if the curve
            # is skewed). For normal distributions, this value will be the mean and the median.

            # set the lowest value x used for obtaining the normal curve as center_of_bin_of_max_proba - 4*sigma
            # the highest x will be center_of_bin_of_max_proba - 4*sigma
            # each value will be created by incrementing (0.10)*sigma

            # The arrays created by the plt.hist method present the value of the extreme left 
            # (the beginning) of the histogram bars, not the bin center. So, let's add half of the bin size
            # to the bin_of_max_proba, so that the adjusted normal will be positioned on the center of the
            # bar of maximum probability. We can do it by taking the average between bin_of_max_proba
            # and the following bin, bin_after_the_max_proba:
            
            # Let's create a normal around the desired mean value. Firstly, create the range X - 4s to
            # X + 4s. The probabilities will be calculated for each value in this range:

            x = (desired_mu - (4 * desired_sigma))
            x_of_normal = [x]

            while (x < (desired_mu + (4 * desired_sigma))):

                x = x + (0.10)*(desired_sigma)
                x_of_normal.append(x)

            # Convert the list to a NumPy array, so that it is possible to perform element-wise
            # (vectorial) operations:
            x_of_normal = np.array(x_of_normal)

            # Create an array of the normal curve y, applying the normal curve equation:
            # normal curve = 1/(sigma* ((2*pi)**(0.5))) * exp(-((x-mu)**2)/(2*(sigma**2)))
            # where pi = 3,14...., and exp is the exponential function (base e)
            # Let's center the normal curve on desired_mu:
            y_normal = (1 / (desired_sigma* (np.sqrt(2 * (np.pi))))) * (np.exp(-0.5 * (((1 / desired_sigma) * (x_of_normal - desired_mu)) ** 2)))
            y_normal = np.array(y_normal)

            # Pick the maximum value obtained for y_normal:
            # https://numpy.org/doc/stable/reference/generated/numpy.amax.html#numpy.amax
            y_normal_max = np.amax(y_normal)

            # Let's get a correction factor, comparing the maximum of the histogram counting, max_count,
            # with y_normal_max:
            correction_factor = max_count/(y_normal_max)

            # Now, multiply each value of the array y_normal by the correction factor, to adjust the height:
            y_normal = y_normal * correction_factor
            # Now the probability density function (values originally from 0 to 1) has the same 
            # height as the histogram.
            
            desired_normal = {'x': x_of_normal, 'y': y_normal}
        
        # Nest the desired_normal dictionary into specification_limits dictionary:
        specification_limits['desired_normal'] = desired_normal
        # Update the attribute:
        self.specification_limits = specification_limits
        
        return self
    
    def get_fitted_normal (self):
        
        import numpy as np
        import pandas as pd
        
        # Get a normal completely (6s) in the specifications, and centered
        # within these limits
        
        mu = self.mu
        sigma = self.sigma
        histogram_dict = self.histogram_dict
        max_count = histogram_dict['max_count']
        bin_of_max_proba = histogram_dict['bin_of_max_proba']
        specification_limits = self.specification_limits
        
        if (sigma == 0):
            print("Impossible to obtain a normal curve overlayed, because the standard deviation is zero.\n")
            print("The analyzed variable is constant throughout the whole sample space.\n")
            
            # Get a dictionary of empty lists for this case
            fitted_normal = {'x': [], 'y':[]}
            
        else:
            # create lists to store the normal curve. Center the normal curve in the bin
            # of maximum bar (max probability, which will not be the mean if the curve
            # is skewed). For normal distributions, this value will be the mean and the median.

            # set the lowest value x used for obtaining the normal curve as bin_of_max_proba - 4*sigma
            # the highest x will be bin_of_max_proba - 4*sigma
            # each value will be created by incrementing (0.10)*sigma

            x = (bin_of_max_proba - (4 * sigma))
            x_of_normal = [x]

            while (x < (bin_of_max_proba + (4 * sigma))):

                x = x + (0.10)*(sigma)
                x_of_normal.append(x)

            # Convert the list to a NumPy array, so that it is possible to perform element-wise
            # (vectorial) operations:
            x_of_normal = np.array(x_of_normal)

            # Create an array of the normal curve y, applying the normal curve equation:
            # normal curve = 1/(sigma* ((2*pi)**(0.5))) * exp(-((x-mu)**2)/(2*(sigma**2)))
            # where pi = 3,14...., and exp is the exponential function (base e)
            # Let's center the normal curve on bin_of_max_proba
            y_normal = (1 / (sigma* (np.sqrt(2 * (np.pi))))) * (np.exp(-0.5 * (((1 / sigma) * (x_of_normal - bin_of_max_proba)) ** 2)))
            y_normal = np.array(y_normal)

            # Pick the maximum value obtained for y_normal:
            # https://numpy.org/doc/stable/reference/generated/numpy.amax.html#numpy.amax
            y_normal_max = np.amax(y_normal)

            # Let's get a correction factor, comparing the maximum of the histogram counting, max_count,
            # with y_normal_max:
            correction_factor = max_count/(y_normal_max)

            # Now, multiply each value of the array y_normal by the correction factor, to adjust the height:
            y_normal = y_normal * correction_factor
            
            fitted_normal = {'x': x_of_normal, 'y': y_normal}
        
        # Nest the fitted_normal dictionary into specification_limits dictionary:
        specification_limits['fitted_normal'] = fitted_normal
        # Update the attribute:
        self.specification_limits = specification_limits
        
        return self
    
    def get_actual_pdf (self):
        
        # PDF: probability density function.
        # KDE: Kernel density estimation: estimation of the actual probability density
        # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.gaussian_kde.html#scipy.stats.gaussian_kde
        
        import numpy as np
        import pandas as pd
        from scipy import stats
        
        df = self.df
        column_with_variable_to_be_analyzed = self.column_with_variable_to_be_analyzed
        array_to_analyze = np.array(df[column_with_variable_to_be_analyzed])
        
        mu = self.mu
        sigma = self.sigma
        lowest = self.lowest
        highest = self.highest
        sample_size = self.sample_size
        
        histogram_dict = self.histogram_dict
        max_count = histogram_dict['max_count']
        specification_limits = self.specification_limits 
        
        # Get the KDE object
        kde = stats.gaussian_kde(array_to_analyze)
        
        # Here, kde may represent a distribution with high skewness and kurtosis. So, let's check
        # if the intervals mu - 6s and mu + 6s are represented by the array:
        inf_kde_lim = mu - 6*sigma
        sup_kde_lim = mu + 6*sigma
        
        if (inf_kde_lim > min(list(array_to_analyze))):
            # make the inferior limit the minimum value from the array:
            inf_kde_lim = min(list(array_to_analyze))
        
        if (sup_kde_lim < max(list(array_to_analyze))):
            # make the superior limit the minimum value from the array:
            sup_kde_lim = max(list(array_to_analyze))
        
        # Let's obtain a X array, consisting with all values from which we will calculate the PDF:
        new_x = inf_kde_lim
        new_x_list = [new_x]
        
        while ((new_x) < sup_kde_lim):
            # There is already the first element, so go to the next one.
            new_x = new_x + (0.10)*sigma
            new_x_list.append(new_x)
        
        # Convert the new_x_list to NumPy array, making it the array_to_analyze:
        array_to_analyze = np.array(new_x_list)
        
        # Apply the pdf method to convert the array_to_analyze into the array of probabilities:
        # i.e., calculate the probability for each one of the values in array_to_analyze:
        # PDF: Probability density function
        # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.gaussian_kde.pdf.html#scipy.stats.gaussian_kde.pdf
        array_of_probs = kde.pdf(array_to_analyze)
        
        # Pick the maximum value obtained for array_of_probs:
        # https://numpy.org/doc/stable/reference/generated/numpy.amax.html#numpy.amax
        array_of_probs_max = np.amax(array_of_probs)

        # Let's get a correction factor, comparing the maximum of the histogram counting, max_count,
        # with array_of_probs_max:
        correction_factor = max_count/(array_of_probs_max)

        # Now, multiply each value of the array y_normal by the correction factor, to adjust the height:
        array_of_probs = array_of_probs * correction_factor
        # Now the probability density function (values originally from 0 to 1) has the same 
        # height as the histogram.
        
        # Define a dictionary
        # X of the probability density plot: values from the series being analyzed.
        # Y of the probability density plot: probabilities calculated for each X.
        actual_pdf = {'x': array_to_analyze, 'y': array_of_probs}
        
        # Nest the desired_normal dictionary into specification_limits dictionary:
        specification_limits['actual_pdf'] = actual_pdf
        # Update the attribute:
        self.specification_limits = specification_limits
        
        return self
    
    def get_capability_indicators (self):
        
        import numpy as np
        import pandas as pd
        
        # Get a normal completely (6s) in the specifications, and centered
        # within these limits
        
        mu = self.mu
        sigma = self.sigma
        histogram_dict = self.histogram_dict
        bin_of_max_proba = histogram_dict['bin_of_max_proba']
        bin_after_the_max_proba = histogram_dict['bin_after_the_max_proba']
        max_count = histogram_dict['max_count']
        
        specification_limits = self.specification_limits
        lower_spec = specification_limits['lower_spec_lim']
        upper_spec = specification_limits['upper_spec_lim']
        desired_mu = (lower_spec + upper_spec)/2 
        # center of the specification limits: we want the mean to be in the center of the
        # specification limits
        
        range_spec = abs(upper_spec - lower_spec)
        
        # Get the constant:
        self = self.get_constants()
        dict_of_constants = self.dict_of_constants
        constant = dict_of_constants['1/c4']
        
        # Calculate corrected sigma:
        sigma_corrected = sigma*constant
        
        # Calculate the capability indicators, adding them to the
        # capability_dict
        cp = (range_spec)/(6*sigma_corrected)
        cr = 100*(6*sigma_corrected)/(range_spec)
        cm = (range_spec)/(8*sigma_corrected)
        zu = (upper_spec - mu)/(sigma_corrected)
        zl = (mu - lower_spec)/(sigma_corrected)
        
        z_min = min(zu, zl)
        cpk = (z_min)/3

        cpm_factor = 1 + ((mu - desired_mu)/sigma_corrected)**2
        cpm_factor = cpm_factor**(0.5) # square root
        cpm = (cp)/(cpm_factor)
        
        capability_dict = {'indicator': ['cp', 'cr', 'cm', 'zu', 'zl', 'z_min', 'cpk', 'cpm'], 
                            'value': [cp, cr, cm, zu, zl, z_min, cpk, cpm]}
        # Already in format for pd.DataFrame constructor
        
        # Update the attribute:
        self.capability_dict = capability_dict
        
        return self
    
    def capability_interpretation (self):
       
        print("Capable process: a process which attends its specifications.")
        print("Naturally, we want processes capable of attending the specifications.\n")
        
        print("Specification range:")
        print("Absolute value of the difference between the upper and the lower limits of specification.\n")
        
        print("6s interval:")
        print("Consider mean value = mu; standard deviation = s")
        print("For a normal distribution, 99.7% of the values range from its (mu - 3s) to (mu + 3s).")
        print("So, if the process follows the normal distribution, we can consider that virtually all of the data is in this range with 6s width.\n")
        
        print ("Cp:")
        print ("Relation between specification range and 6s.\n")
        
        print("Cr:")
        print("Usually, 6s > specification range.")
        print("So, the inverse of Cp is the fraction of 6s correspondent to the specification range.")
        print("Example: if 1/Cp = 0.2, then the specification range corresponds to 0.20 (20%) of the 6s interval.")
        print("Cr = 100 x (1/Cp) - the percent of 6s correspondent to the specification range.")
        print("Again, if 1/Cp = 0.2, then Cr = 20: the specification range corresponds to 20% of the 6s interval.\n")
        
        print("Cm:")
        print("It is a more generalized version of Cp.")
        print("Cm is the relation between specification range and 8s.")
        print("Then, even highly distant values from long-tailed curves are analyzed by this indicator.\n")
        
        print("Zu:")
        print("Represents how far is the mean of the values from the upper specification limit.")
        print("Zu = ([upper specification limit] - mu)/s")
        print("A higher Zu indicates a mean value lower than (and more distant from) the upper specification.")
        print("A negative Zu, in turns, indicates that the mean value is greater than the upper specification (i.e.: in average, specification is not attended).\n")
        
        print("Zl:")
        print("Represents how far is the mean of the values from the lower specification limit.")
        print("Zl = (mu - [lower specification limit])/s\n")
        print("A higher Zl indicates a mean value higher than  (and more distant from) the lower specification.")
        print("A negative Zl, in turns, indicates that the mean value is inferior than the lower specification (i.e.: in average, specification is not attended).\n")
        
        print("Zmin:")
        print("It is the minimum value between Zu and Zl.")
        print("So, Zmin indicates which specification is more difficult for the process to attend: the upper or the lower one.")
        print("Example: if Zmin = Zl, the mean of the process is closer to the lower specification than it is from the upper specification.")
        print("If Zmin, Zu, and Zl are equal, than the process is equally distant from the two specifications.")
        print("Again, if Zmin is negative, at least one of the specifications is not attended.\n")
        
        print("Cpk:")
        print("This is the most fundamental capability indicator.")
        print("Consider again that 99.7% of the normally distributed data are within [(mu - 3s), (mu + 3s)].")
        print("Cpk = Zmin/3")
        print("Cpk = min((([upper specification limit] - mu)/3s), ((mu - [lower specification limit])/3s))")
        print("\n")
        print("Cpk simultaneously assess the process centrality, and if the process is capable of attending its specifications.")
        print("Here, the process centrality is verified as results which are well and simetrically distributed throughout the mean of the specification limits.")
        print("Basically, a perfectly-centralized process has its mean equally distant from both specifications")
        print("i.e., the mean is in the center of the specification interval.")
        print("Cpk = + 1 is usually considered the minimum value acceptable for a process.")
        print("Many quality programs define reaching Cpk = + 1.33 as their goal.")
        print("A 6-sigma process, in turns, is defined as a process with Cpk = + 2.")
        print("\n")
        print("High values of Cpk indicate that the process is not only centralized, but that the differences")
        print("([upper specification limit] - mu) and (mu - [lower specification limit]) are greater than 3s.")
        print("Since mu +- 3s is the range for 99.7% of data, it indicates that most of the values generated fall in a range")
        print("that is only a fraction of the specification range.")
        print("So, it is easier for the process to attend the specifications.")
        print("\n")
        print("Cpk values inferior than 1 indicate that at least one of the intervals ([upper specification limit] - mu) and (mu - [lower specification limit])")
        print("is lower than 3s, i.e., the process naturally generates values beyond at least one of the specifications.")
        print("Low values of Cpk (in particular the negative ones) indicate not-centralized processes and processes not capable of attending their specifications.")
        print("So, lower (and, specially, more negative) Cpk: process' outputs more distant from the specifications.\n")
        
        print("Cpm:")
        print("This indicator is a more generalized version of the Cpk.")
        print("It basically consists on a standard normalization of the Cpk.")
        print("For that, a normalization factor is defined as:")
        print("factor = square root(1 + ((mu - target)/s)**2)")
        print("where target is the center of the specification limits, and **2 represents the second power (square)")
        print("Cpm = Cpk/(factor)")


class regex_help:

    def __init__ (self, start_helper = True, helper_screen = 0):
        
        # from DataCamp course Regular Expressions in Python
        # https://www.datacamp.com/courses/regular-expressions-in-python#!

        self.start_helper = start_helper
        self.helper_screen = helper_screen
        
        self.helper_menu_1 = """

    Regular Expressions (RegEx) Helper
                    
    Input the number in the text box and press enter to visualize help and examples for a topic:

        1. regex basic theory and most common metacharacters
        2. regex quantifiers
        3. regex anchoring and finding
        4. regex greedy and non-greedy search
        5. regex grouping and capturing
        6. regex alternating and non-capturing groups
        7. regex backreferences
        8. regex lookaround
        9. print all topics at once
        10. Finish regex helper
        
        """
        
        # regex basic theory and most common metacharacters
        self.help_text_1 = """
    REGular EXpression or regex:
    String containing a combination of normal characters and special metacharacters that
    describes patterns to find text or positions within a text.

    Example:

    r'st\d\s\w{3,10}'
    - In Python, the r at the beginning indicates a raw string. It is always advisable to use it.
    - We said that a regex contains normal characters, or literal characters we already know. 
        - The normal characters match themselves. 
        - In the case shown above, 'st' exactly matches an 's' followed by a 't'.

    - Most important metacharacters:
        - \d: digit (number);
        - \D: non-digit;
        - \s: whitespace;
        - \s+: one or more consecutive whitespaces.
        - \S: non-whitespace;
        - \w: (word) character;
        - \W: non-word character.
        - {N, M}: indicates that the character on the left appears from N to M consecutive times.
            - \w{3,10}: a word character that appears 3, 4, 5,..., or 10 consecutive times.
        - {N}: indicates that the character on the left appears exactly N consecutive times.
            - \d{4}: a digit appears 4 consecutive times.
        - {N,}: indicates that the character appears at least N times.
            - \d{4,}: a digit appears 4 or more times.
            - phone_number = "John: 1-966-847-3131 Michelle: 54-908-42-42424"
            - re.findall(r"\d{1,2}-\d{3}-\d{2,3}-\d{4,}", phone_number) - returns: ['1-966-847-3131', '54-908-42-42424']

    ATTENTION: Using metacharacters in regular expressions will allow you to match types of characters such as digits. 
    - You can encounter many forms of whitespace such as tabs, space or new line. 
    - To make sure you match all of them always specify whitespaces as \s.

    re module: Python standard library module to search regex within individual strings.

    - .findall method: search all occurrences of the regex within the string, returning a list of strings.
    - Syntax: re.findall(r"regex", string)
        - Example: re.findall(r"#movies", "Love #movies! I had fun yesterday going to the #movies")
            - Returns: ['#movies', '#movies']

    - .split method: splits the string at each occurrence of the regex, returning a list of strings.
    - Syntax: re.split(r"regex", string)
        - Example: re.split(r"!", "Nice Place to eat! I'll come back! Excellent meat!")
            - Returns: ['Nice Place to eat', " I'll come back", ' Excellent meat', '']

    - .sub method: replace one or many matches of the regex with a given string (returns a replaced string).
    - Syntax: re.sub((r"regex", new_substring, original_string))
        - Example: re.sub(r"yellow", "nice", "I have a yellow car and a yellow house in a yellow neighborhood")
        - Returns: 'I have a nice car and a nice house in a nice neighborhood'

    - .search and .match methods: they have the same syntax and are used to find a match. 
        - Both methods return an object with the match found. 
        - The difference is that .match is anchored at the beginning of the string.
    - Syntax: re.search(r"regex", string) and re.match(r"regex", string)
        - Example 1: re.search(r"\d{4}", "4506 people attend the show")
        - Returns: <_sre.SRE_Match object; span=(0, 4), match='4506'>
        - re.match(r"\d{4}", "4506 people attend the show")
        - Returns: <_sre.SRE_Match object; span=(0, 4), match='4506'>
            - In this example, we use both methods to find a digit appearing four times. 
            - Both methods return an object with the match found.
        
        - Example 2: re.search(r"\d+", "Yesterday, I saw 3 shows")
        - Returns: <_sre.SRE_Match object; span=(17, 18), match='3'>
        - re.match(r"\d+","Yesterday, I saw 3 shows")
        - Returns: None
            - In this example,, we used them to find a match for a digit. 
            - In this case, .search finds a match, but .match does not. 
            - This is because the first characters do not match the regex.

    - .group method: detailed in Section 7 (Backreferences).
        - Retrieves the groups captured.
    - Syntax: searched_string = re.search(r"regex", string)
        re.group(N) - returns N-th group captured (group 0 is the regex itself).
        
        Example: text = "Python 3.0 was released on 12-03-2008."
        information = re.search('(\d{1,2})-(\d{2})-(\d{4})', text)
        information.group(3) - returns: '2008'
    - .group can only be used with .search and .match methods.

    Examples of regex:

    1. re.findall(r"User\d", "The winners are: User9, UserN, User8")
        ['User9', 'User8']
    2. re.findall(r"User\D", "The winners are: User9, UserN, User8")
        ['UserN']
    3. re.findall(r"User\w", "The winners are: User9, UserN, User8")
        ['User9', 'UserN', 'User8']
    4. re.findall(r"\W\d", "This skirt is on sale, only $5 today!")
        ['$5']
    5. re.findall(r"Data\sScience", "I enjoy learning Data Science")
        ['Data Science']
    6. re.sub(r"ice\Scream", "ice cream", "I really like ice-cream")
        'I really like ice cream'

    7. regex that matches the user mentions that starts with @ and follows the pattern @robot3!.

    regex = r"@robot\d\W"

    8. regex that matches the number of user mentions given as, for example: User_mentions:9.

    regex = r"User_mentions:\d"

    9. regex that matches the number of likes given as, for example, likes: 5.

    regex = r"likes:\s\d"

    10. regex that matches the number of retweets given as, for example, number of retweets: 4.

    regex = r"number\sof\sretweets:\s\d"

    11. regex that matches the user mentions that starts with @ and follows the pattern @robot3!.

    regex_sentence = r"\W\dbreak\W"

    12. regex that matches the pattern #newH

    regex_words = r"\Wnew\w"

    """

        # regex quantifiers
        self.help_text_2 = """
    Quantifiers: 
    A metacharacter that tells the regex engine how many times to match a character immediately to its left.

        1. +: Once or more times.
            - text = "Date of start: 4-3. Date of registration: 10-04."
            - re.findall(r"\d+-\d+", text) - returns: ['4-3', '10-04']
            - Again, \s+ represents one or more consecutive whitespaces.
        2. *: Zero times or more.
            - my_string = "The concert was amazing! @ameli!a @joh&&n @mary90"
            - re.findall(r"@\w+\W*\w+", my_string) - returns: ['@ameli!a', '@joh&&n', '@mary90']
        3. ?: Zero times or once: ?
            - text = "The color of this image is amazing. However, the colour blue could be brighter."
            - re.findall(r"colou?r", text) - returns: ['color', 'colour']
        
    The quantifier refers to the character immediately on the left:
    - r"apple+" : + applies to 'e' and not to 'apple'.

    Examples of regex:

    1. Most of the times, links start with 'http' and do not contain any whitespace, e.g. https://www.datacamp.com. 
    - regex to find all the matches of http links appearing:
        - regex = r"http\S+"
        - \S is very useful to use when you know a pattern does not contain spaces and you have reached the end when you do find one.

    2. User mentions in Twitter start with @ and can have letters and numbers only, e.g. @johnsmith3.
    - regex to find all the matches of user mentions appearing:
        - regex = r"@\w*\d*"

    3. regex that finds all dates in a format similar to 27 minutes ago or 4 hours ago.
    - regex = r"\d{1,2}\s\w+\sago"

    4. regex that finds all dates in a format similar to 23rd june 2018.
    - regex = r"\d{1,2}\w{2}\s\w+\s\d{4}"

    5. regex that finds all dates in a format similar to 1st september 2019 17:25.
    - regex = r"\d{1,2}\w{2}\s\w+\s\d{4}\s\d{1,2}:\d{2}"

    6. Hashtags start with a # symbol and contain letters and numbers but never whitespace.
    - regex that matches the described hashtag pattern.
        - regex = r"#\w+"
        
    """

        # regex anchoring and finding
        self.help_text_3 = """
    - Anchoring and Finding Metacharacters

        1. . (dot): Match any character (except newline).
            - my_links = "Just check out this link: www.amazingpics.com. It has amazing photos!"
            - re.findall(r"www.+com", my_links) - returns: ['www.amazingpics.com']
                - The dot . metacharacter is very useful when we want to match all repetitions of any character. 
                - However, we need to be very careful how we use it.
        2. ^: Anchoring on start of the string.
            - my_string = "the 80s music was much better that the 90s"
            - If we do re.findall(r"the\s\d+s", my_string) - returns: ['the 80s', 'the 90s']
            - Using ^: re.findall(r"^the\s\d+s", my_string) - returns: ['the 80s']
        3. $: Anchoring at the end of the string.
            - my_string = "the 80s music hits were much better that the 90s"
            - re.findall(r"the\s\d+s$", my_string) - returns: ['the 90s']
        4. \: Escape special characters.
            - my_string = "I love the music of Mr.Go. However, the sound was too loud."
                - re.split(r".\s", my_string) - returns: ['', 'lov', 'th', 'musi', 'o', 'Mr.Go', 'However', 'th', 'soun', 'wa', 'to', 'loud.']
                - re.split(r"\.\s", my_string) - returns: ['I love the music of Mr.Go', 'However, the sound was too loud.']
        5. |: OR Operator
            - my_string = "Elephants are the world's largest land animal! I would love to see an elephant one day"
            - re.findall(r"Elephant|elephant", my_string) - returns: ['Elephant', 'elephant']
        6. []: set of characters representing the OR Operator.
            Example 1 - my_string = "Yesterday I spent my afternoon with my friends: MaryJohn2 Clary3"
            - re.findall(r"[a-zA-Z]+\d", my_string) - returns: ['MaryJohn2', 'Clary3']
            Example 2 - my_string = "My&name&is#John Smith. I%live$in#London."
            - re.sub(r"[#$%&]", " ", my_string) - returns: 'My name is John Smith. I live in London.'
            
            Note 1: within brackets, the characters to be found should not be separated, as in [#$%&].
                - Whitespaces or other separators would be interpreted as characters to be found.
            Note 2: [a-z] represents all word characters from 'a' to 'z', lowercase.
                    - [A-Z] represents all word characters from 'A' to 'Z', uppercase.
                    - Since lower and uppercase are different, we must declare [a-zA-Z] or [A-Za-z] to capture all word characters.
                    - [0-9] represents all digits from 0 to 9.
                    - Something like [a-zA-Z0-9] or [a-z0-9A-Z] will search all word characters and all numbers.
        
        7. [^ ]: OR operator combined to ^ transforms the expression to negative.
            - my_links = "Bad website: www.99.com. Favorite site: www.hola.com"
            - re.findall(r"www[^0-9]+com", my_links) - returns: ['www.hola.com']

    Examples of regex:

    1. You want to find names of files that appear at the start of the string; 
        - always start with a sequence of 2 or 3 upper or lowercase vowels (a e i o u); 
        - and always finish with the txt ending.
            - Write a regex that matches the pattern of the text file names, e.g. aemyfile.txt.
            # . = match any character
            regex = r"^[aeiouAEIOU]{2,3}.+txt"

    2. When a user signs up on the company website, they must provide a valid email address.
        - The company puts some rules in place to verify that the given email address is valid:
        - The first part can contain: Upper A-Z or lowercase letters a-z; 
        - Numbers; Characters: !, #, %, &, *, $, . Must have @. Domain: Can contain any word characters;
        - But only .com ending is allowed. 
            - Write a regular expression to match valid email addresses.
            - Match the regex to the elements contained in emails, and print out the message indicating if it is a valid email or not 
        
        # Write a regex to match a valid email address
        regex = r"^[A-Za-z0-9!#%&*$.]+@\w+\.com"

        for example in emails:
            # Match the regex to the string
            if re.match(regex, example):
                # Complete the format method to print out the result
                print("The email {email_example} is a valid email".format(email_example=example))
            else:
                print("The email {email_example} is invalid".format(email_example=example))
        
        # Notice that we used the .match() method. 
        # The reason is that we want to match the pattern from the beginning of the string.

    3. Rules in order to verify valid passwords: it can contain lowercase a-z and uppercase letters A-Z;
        - It can contain numbers; it can contain the symbols: *, #, $, %, !, &, .
        - It must be at least 8 characters long but not more than 20.
            - Write a regular expression to check if the passwords are valid according to the description.
            - Search the elements in the passwords list to find out if they are valid passwords.
            - Print out the message indicating if it is a valid password or not, complete .format() statement.
        
        # Write a regex to check if the password is valid
        regex = r"[a-z0-9A-Z*#$%!&.]{8,20}"

        for example in passwords:
            # Scan the strings to find a match
            if re.match(regex, example):
                # Complete the format method to print out the result
                print("The password {pass_example} is a valid password".format(pass_example=example))
            else:
                print("The password {pass_example} is invalid".format(pass_example=example))

    """

        # regex greedy and non-greedy search
        self.help_text_4 = """
    There are two types of matching methods: greedy and non-greedy (also called lazy) operators. 

    Greedy operators
    - The standard quantifiers are greedy by default, meaning that they will attempt to match as many characters as possible.
        - Standard quantifiers: * , + , ? , {num, num}
        - Example: re.match(r"\d+", "12345bcada") - returns: <_sre.SRE_Match object; span=(0, 5), match='12345'>
        - We can explain this in the following way: our quantifier will start by matching the first digit found, '1'. 
        - Because it is greedy, it will keep going to find 'more' digits and stop only when no other digit can be matched, returning '12345'.
    - If the greedy quantifier has matched so many characters that can not match the rest of pattern, it will backtrack, giving up characters matched earlier one at a time and try again. 
    - Backtracking is like driving a car without a map. If you drive through a path and hit a dead end street, you need to backtrack along your road to an earlier point to take another street. 
        - Example: re.match(r".*hello", "xhelloxxxxxx") - returns: <_sre.SRE_Match object; span=(0, 6), match='xhello'>
        - We use the greedy quantifier .* to find anything, zero or more times, followed by the letters "h" "e" "l" "l" "o". 
        - We can see here that it returns the pattern 'xhello'. 
        - So our greedy quantifier will start by matching as much as possible, the entire string. 
        - Then, it tries to match the h, but there are no characters left. So it backtracks, giving up one matched character. 
        - Trying again, it still doesn't match the h, so it backtracks one more step repeatedly until it finally matches the h in the regex, and the rest of the characters.

    Non-greedy (lazy) operators
    - Because they have lazy behavior, non-greedy quantifiers will attempt to match as few characters as needed, returning the shortest match. 
    - To obtain non-greedy quantifiers, we can append a question mark at the end of the greedy quantifiers to convert them into lazy. 
        - Example: re.match(r"\d+?", "12345bcada") - returns: <_sre.SRE_Match object; span=(0, 1), match='1'>
        - Now, our non-greedy quantifier will return the pattern '1'. 
        - In this case, our quantifier will start by matching the first digit found, '1'. 
        - Because it is non-greedy, it will stop there, as we stated that we want 'one or more', and 1 is as few as needed.
    - Non-greedy quantifiers also backtrack. 
    - In this case, if they have matched so few characters that the rest of the pattern cannot match, they backtrack, expand the matched character one at a time, and try again. 
    - In the example above: this time we use the lazy quantifier .*?. Interestingly, we obtain the same match 'xhello'. 
    - But, how this match was obtained is different from the first time: the lazy quantifier first matches as little as possible, nothing, leaving the entire string unmatched. 
    - Then it tries to match the 'h', but it doesn't work. 
    - So, it backtracks, matching one more character, the 'x'. Then, it tries again, this time matching the 'h', and afterwards, the rest of the regex.

    - Even though greedy quantifiers lead to longer matches, they are sometimes the best option. 
    - Because lazy quantifiers match as few as possible, they return a shorter match than we expected.
        - Example: if you want to extract a word starting with 'a' and ending with 'e' in the string 'I like apple pie', you may think that applying the greedy regex r"a.+e" will return 'apple'. 
        - However, your match will be 'apple pie'. A way to overcome this is to make it lazy by using '?'' which will return 'apple'.
    - On the other hand, using greedy quantifiers always leads to longer matches that sometimes are not desired. 
        - Making quantifiers lazy by adding '?' to match a shorter pattern is a very important consideration to keep in mind when handling data for text mining.

    Examples of regex:

    1. You want to extract the number contained in the sentence 'I was born on April 24th'. 
        - A lazy quantifier will make the regex return 2 and 4, because they will match as few characters as needed. 
        - However, a greedy quantifier will return the entire 24 due to its need to match as much as possible.

        1.1. Use a lazy quantifier to match all numbers that appear in the variable sentiment_analysis:
        numbers_found_lazy = re.findall(r"[0-9]+?", sentiment_analysis)
        - Output: ['5', '3', '6', '1', '2']
        
        1.2. Now, use a greedy quantifier to match all numbers that appear in the variable sentiment_analysis.
        numbers_found_greedy = re.findall(r"[0-9]+", sentiment_analysis)
        - Output: ['536', '12']

    2.1. Use a greedy quantifier to match text that appears within parentheses in the variable sentiment_analysis.
        
        sentences_found_greedy = re.findall(r"\(.+\)", sentiment_analysis)
        - Output: ["(They were so cute) a few yrs ago. PC crashed, and now I forget the name of the site ('I'm crying)"]

    2.2. Now, use a lazy quantifier to match text that appears within parentheses in the variable sentiment_analysis.

        sentences_found_lazy = re.findall(r"\(.+?\)", sentiment_analysis)
        - Output: ["(They were so cute)", "('I'm crying)"]
        
    """

        # regex grouping and capturing
        self.help_text_5 = """
    Capturing groups in regular expressions
    - Let's say that we have the following text:
        
        text = "Clary has 2 friends who she spends a lot time with. Susan has 3 brothers while John has 4 sisters."
        
    - We want to extract information about a person, how many and which type of relationships they have. 
    - So, we want to extract Clary 2 friends, Susan 3 brothers and John 4 sisters.
    - If we do: re.findall(r'[A-Za-z]+\s\w+\s\d+\s\w+', text), the output will be: ['Clary has 2 friends', 'Susan has 3 brothers', 'John has 4 sisters']
        - The output is quite close, but we do not want the word 'has'.

    - We start simple, by trying to extract only the names. We can place parentheses to group those characters, capture them, and retrieve only that group:
        - re.findall(r'([A-Za-z]+)\s\w+\s\d+\s\w+', text) - returns: ['Clary', 'Susan', 'John']
    - Actually, we can place parentheses around the three groups that we want to capture. 
        - re.findall(r'([A-Za-z]+)\s\w+\s(\d+)\s(\w+)', text)
        
        - Each group will receive a number: 
            - The entire expression will always be group 0. 
            - The first group: 1; the second: 2; and the third: 3.
        
        - The result returned is: [('Clary', '2', 'friends'), ('Susan', '3', 'brothers'), ('John', '4', 'sisters')]
            - We got a list of tuples: 
                - The first element of each tuple is the match captured corresponding to group 1. 
                - The second, to group 2. The last, to group 3.
        
        - We can use capturing groups to match a specific subpattern in a pattern. 
        - We can use this information for retrieving the groups by numbers; or to organize data.
            - Example: pets = re.findall(r'([A-Za-z]+)\s\w+\s(\d+)\s(\w+)', "Clary has 2 dogs but John has 3 cats")
                        pets[0][0] == 'Clary'
                        - In the code, we placed the parentheses to capture the name of the owner, the number and which type of pets each one has. 
                        - We can access the information retrieved by using indexing and slicing as seen in the code. 
    
    - Capturing groups have one important feature. 
        - Remember that quantifiers apply to the character immediately to the left. 
        - So, we can place parentheses to group characters and then apply the quantifier to the entire group. 
        
        Example: re.search(r"(\d[A-Za-z])+", "My user name is 3e4r5fg")
            - returns: <_sre.SRE_Match object; span=(16, 22), match='3e4r5f'>
            - In the code, we have placed parentheses to match the group containing a number and any letter. 
            - We applied the plus quantifier to specify that we want this group repeated once or more times. 
        
    - ATTENTION: It's not the same to capture a repeated group AND to repeat a capturing group. 
        
        my_string = "My lucky numbers are 8755 and 33"
        - re.findall(r"(\d)+", my_string) - returns: ['5', '3']
        - re.findall(r"(\d+)", my_string) - returns: ['8755', '33']
        
        - In the first code, we use findall to match a capturing group containing one number. 
            - We want this capturing group to be repeated once or more times. 
            - We get 5 and 3 as an output, because these numbers are repeated consecutively once or more times. 
        - In the second code, we specify that we should capture a group containing one or more repetitions of a number. 

    - Placing a subpattern inside parenthesis will capture that content and stores it temporarily in memory. This can be later reused.

    Examples of regex:

    1. You want to extract the first part of the email. E.g. if you have the email marysmith90@gmail.com, you are only interested in marysmith90.
    - You need to match the entire expression. So you make sure to extract only names present in emails. Also, you are only interested in names containing upper (e.g. A,B, Z) or lowercase letters (e.g. a, d, z) and numbers.
    - regex to match the email capturing only the name part. The name part appears before the @.
        - regex_email = r"([a-z0-9A-Z]+)@\S+"

    2. Text follows a pattern: "Here you have your boarding pass LA4214 AER-CDB 06NOV."
    - You need to extract the information about the flight: 
        - The two letters indicate the airline (e.g LA); the 4 numbers are the flight number (e.g. 4214);
        - The three letters correspond to the departure (e.g AER); the destination (CDB); the date (06NOV) of the flight.
        - All letters are always uppercase.

    - Regular expression to match and capture all the flight information required.
    - Find all the matches corresponding to each piece of information about the flight. Assign it to flight_matches.
    - Complete the format method with the elements contained in flight_matches: 
        - In the first line print the airline and the flight number. 
        - In the second line, the departure and destination. In the third line, the date.

    # Import re
    import re

    # Write regex to capture information of the flight
    regex = r"([A-Z]{2})(\d{4})\s([A-Z]{3})-([A-Z]{3})\s(\d{2}[A-Z]{3})"

    # Find all matches of the flight information
    flight_matches = re.findall(regex, flight)
        
    #Print the matches
    print("Airline: {} Flight number: {}".format(flight_matches[0][0], flight_matches[0][1]))
    print("Departure: {} Destination: {}".format(flight_matches[0][2], flight_matches[0][3]))
    print("Date: {}".format(flight_matches[0][4]))

        - findall() returns a list of tuples. 
        - The nth element of each tuple is the element corresponding to group n. 
        - This provides us with an easy way to access and organize our data.

    """

        # regex alternating and non-capturing groups
        self.help_text_6 = """
    Alternating and non-capturing groups

    - Vertical bar or pipe operator
        - Suppose we have the following string, and we want to find all matches for pet names. 
        - We can use the pipe operator to specify that we want to match cat or dog or bird:
            - my_string = "I want to have a pet. But I don't know if I want a cat, a dog or a bird."
            - re.findall(r"cat|dog|bird", my_string) - returns: ['cat', 'dog', 'bird']
        
        - Now, we changed the string a little bit, and once more we want to find all the pet names, but only those that come after a number and a whitespace. 
        - So, if we specify this again with the pipe operator, we get the wrong output: 
            - my_string = "I want to have a pet. But I don't know if I want 2 cats, 1 dog or a bird."
            - re.findall(r"\d+\scat|dog|bird", my_string) - returns: ['2 cat', 'dog', 'bird']
        
        - That is because the pipe operator works by comparing everything that is to its left (digit whitespace cat) with everything to the right, dog.
        - In order to solve this, we can use alternation. 
            - In simpler terms, we can use parentheses again to group the optional characters:
            
            - my_string = "I want to have a pet. But I don't know if I want 2 cats, 1 dog or a bird."
            - re.findall(r"\d+\s(cat|dog|bird)", my_string) - returns: ['cat', 'dog']
            
            In the code, now the parentheses are added to group cat or dog or bird.
        
        - In the previous example, we may also want to match the number. 
        - In that case, we need to place parentheses to capture the digit group:
        
            - my_string = "I want to have a pet. But I don't know if I want 2 cats, 1 dog or a bird."
            - re.findall(r"(\d)+\s(cat|dog|bird)", my_string) - returns: [('2', 'cat'), ('1', 'dog')]
        
            - In the code, we now use two pair of parentheses and we use findall in the string, so we get a list with two tuples.
        
    - Non-capturing groups
        - Sometimes, we need to group characters using parentheses, but we are not going to reference back to this group. 
        - For these cases, there are a special type of groups called non-capturing groups. 
        - For using them, we just need to add question mark colon inside the parenthesis but before the regex.
        
        regex = r"(?:regex)"
        
        - Example: we have the following string, and we want to find all matches of numbers. 
        
            my_string = "John Smith: 34-34-34-042-980, Rebeca Smith: 10-10-10-434-425"
        
        - We see that the pattern consists of two numbers and dash repeated three times. After that, three numbers, dash, four numbers. 
        - We want to extract only the last part, without the first repeated elements. 
        - We need to group the first two elements to indicate repetitions, but we do not want to capture them. 
        - So, we use non-capturing groups to group \d repeated two times and dash. Then we indicate this group should be repeated three times. Then, we group \d repeated three times, dash, \d repeated three times:
        
            re.findall(r"(?:\d{2}-){3}(\d{3}-\d{3})", my_string) - returns: ['042-980', '434-425']
        
    - Alternation
        - We can combine non-capturing groups and alternation together. 
        - Remember that alternation implies using parentheses and the pipe operand to group optional characters. 
        - Let's suppose that we have the following string. We want to match all the numbers of the day. 
        
            my_date = "Today is 23rd May 2019. Tomorrow is 24th May 19."
        
        - We know that they are followed by 'th' or 'rd', but we only want to capture the number, and not the letters that follow it. 
        - We write our regex to capture inside parentheses \d repeated once or more times. Then, we can use a non-capturing group. 
        - Inside, we use the pipe operator to choose between 'th' or 'rd':
        
            re.findall(r"(\d+)(?:th|rd)", my_date) - returns: ['23', '24']

    - Non-capturing groups are very often used together with alternation. 
    - Sometimes, you have optional patterns and you need to group them. 
    - However, you are not interested in keeping them. It's a nice feature of regex.

    Examples of regex:

    1. Sentiment analysis project: firstly, you want to identify positive tweets about movies and concerts.
    - You plan to find all the sentences that contain the words 'love', 'like', or 'enjoy', and capture that word. 
    - You will limit the tweets by focusing on those that contain the words 'movie' or 'concert' by keeping the word in another group. 
    - You will also save the movie or concert name.
        - For example, if you have the sentence: 'I love the movie Avengers', you match and capture 'love'. 
        - You need to match and capture 'movie'. Afterwards, you match and capture anything until the dot.
        - The list sentiment_analysis contains the text of tweets.
    - Regular expression to capture the words 'love', 'like', or 'enjoy'; 
        - match and capture the words 'movie' or 'concert'; 
        - match and capture anything appearing until the '.'.

        regex_positive = r"(love|like|enjoy).+?(movie|concert)\s(.+?)\."

        - The pipe operator works by comparing everything that is to its left with everything to the right. 
        - Grouping optional patterns is the way to get the correct result.

    2. After finding positive tweets, you want to do it for negative tweets. 
    - Your plan now is to find sentences that contain the words 'hate', 'dislike' or 'disapprove'. 
    - You will again save the movie or concert name. 
    - You will get the tweet containing the words 'movie' or 'concert', but this time, you do not plan to save the word.
        - For example, if you have the sentence: 'I dislike the movie Avengers a lot.', you match and capture 'dislike'. 
        - You will match, but not capture, the word 'movie'. Afterwards, you match and capture anything until the dot.
    - Regular expression to capture the words 'hate', 'dislike' or 'disapprove'; 
        - Match, but do not capture, the words 'movie' or 'concert'; 
        - Match and capture anything appearing until the '.'.
        
        regex_negative = r"(hate|dislike|disapprove).+?(?:movie|concert)\s(.+?)\."
            
            """

        # regex backreferences
        self.help_text_7 = """
    Backreferences
    - How we can backreference capturing groups.

    Numbered groups
    - Imagine we come across this text, and we want to extract the date: 
        
        text = "Python 3.0 was released on 12-03-2008. It was a major revision of the language. Many of its major features were backported to Python 2.6.x and 2.7.x version series."
        
    - We want to extract only the numbers. So, we can place parentheses in a regex to capture these groups:
        
        regex = r"(\d{1,2})-(\d{1,2})-(\d{4})"

    - We have also seen that each of these groups receive a number. 
    - The whole expression is group 0; the first group, 1; and so on.

    - Let's use .search to match the pattern to the text. 
    - To retrieve the groups captured, we can use the method .group specifying the number of a group we want. 

    Again: .group method retrieves the groups captured.
        - Syntax: searched_string = re.search(r"regex", string)
        re.group(N) - returns N-th group captured (group 0 is the regex itself).

    Example: text = "Python 3.0 was released on 12-03-2008."

        information = re.search('(\d{1,2})-(\d{2})-(\d{4})', text)
        information.group(3) - returns: '2008'
        information.group(0) - returns: '12-03-2008' (regex itself, the entire expression).

    - .group can only be used with .search and .match methods.

    Named groups
    - We can also give names to our capturing groups. 
    - Inside the parentheses, we write '?P', and the name inside angle brackets:

        regex = r"(?P<name>regex)"

    - Let's say we have the following string, and we want to match the name of the city and zipcode in different groups. 
    - We can use capturing groups and assign them the name 'city' and 'zipcode'. 
    - We retrieve the information by using .group, and we indicate the name of the group. 
        
        text = "Austin, 78701"
        cities = re.search(r"(?P<city>[A-Za-z]+).*?(?P<zipcode>\d{5})", text)
        cities.group("city") - returns: 'Austin'
        cities.group("zipcode") - returns: '78701'

    Backreferences
    - There is another way to backreference groups. 
    - In fact, the matched group can be reused inside the same regex or outside for substitution. 
    - We can do this using backslash and the number of the group:

        regex = r'(\d{1,2})-(\d{2})-(\d{4})'
        
        - we can backreference the groups as:
            (\d{1,2}): (\1);
            (\d{2}): (\2)
            (\d{4}): (\3)

    - Example: we have the following string, and we want to find all matches of repeated words. 
    - In the code, we specify that we want to capture a sequence of word characters, then a whitespace.
    - Finally, we write \1. This will indicate that we want to match the first group captured again. 
    - In other words, it says: 'match that sequence of characters that was previously captured once more.' 
        
        sentence = "I wish you a happy happy birthday!"
        re.findall(r"(\w+)\s\1", sentence) - returns: ['happy'] 

    - We get the word 'happy' as an output: this was the repeated word in our string.

    - Now, we want to replace the repeated word with one occurrence of the same word. 
    - In the code, we use the same regex as before, but this time, we use the .sub method. 
    - In the replacement part, we can also reference back to the captured group: 
        - We write r"\1" to say: 'replace the entire expression match with the first captured group.' 
        
        re.sub(r"(\w+)\s\1", r"\1", sentence) - returns: 'I wish you a happy birthday!'
        - In the output string, we have only one occurrence of the word 'happy'.
        
    - We can also use named groups for backreferencing. 
    - To do this, we use ?P= the group name. 

        regex = r"(?P=name)"

    Example:
        sentence = "Your new code number is 23434. Please, enter 23434 to open the door."
        re.findall(r"(?P<code>\d{5}).*?(?P=code)", sentence) - returns: ['23434']

    - In the code, we want to find all matches of the same number. 
    - We use a capturing group and name it 'code'. 
    - Later, we reference back to this group, and we obtain the number as an output.

    - To reference the group back for replacement, we need to use \g and the group name inside angle brackets. 

        regex = r"(\g<name>)"

    Example:
        sentence = "This app is not working! It's repeating the last word word."
        re.sub(r"(?P<word>\w+)\s(?P=word)", r"\g<word>", sentence) - returns: 'This app is not working! It's repeating the last word.'
        
    - In the code, we want to replace repeated words by one occurrence of the same word. 
    - Inside the regex, we use the previous syntax. 
    - In the replacement field, we need to use this new syntax as seen in the code.
    - Backreferences are very helpful when you need to reuse part of the regex match inside the regex.
    - You should remember that the group zero stands for the entire expression matched. 
        - It is always helpful to keep that in mind. Sometimes you will need to use it.

    Examples of regex:

    1. Parsing PDF files: your company gave you some PDF files of signed contracts. The goal of the project is to create a database with the information you parse from them. 
    - Three of these columns should correspond to the day, month, and year when the contract was signed.
    - The dates appear as 'Signed on 05/24/2016' ('05' indicating the month, '24' the day). 
    - You decide to use capturing groups to extract this information. Also, you would like to retrieve that information so you can store it separately in different variables.
    - The variable contract contains the text of one contract.

    - Write a regex that captures the month, day, and year in which the contract was signed. 
    - Scan contract for matches.
    - Assign each captured group to the corresponding keys in the dictionary.
    - Complete the positional method to print out the captured groups. 
    - Use the values corresponding to each key in the dictionary.

        # Write regex and scan contract to capture the dates described
        regex_dates = r"Signed\son\s(\d{2})/(\d{2})/(\d{4})"
        dates = re.search(regex_dates, contract)

        # Assign to each key the corresponding match
        signature = {
            "day": dates.group(2),
            "month": dates.group(1),
            "year": dates.group(3)
        }
        # Complete the format method to print-out
        print("Our first contract is dated back to {data[year]}. Particularly, the day {data[day]} of the month {data[month]}.".format(data=signature))

    - Remember that each capturing group is assigned a number according to its position in the regex. 
    - Only if you use .search() and .match(), you can use .group() to retrieve the groups.

    2. The company is going to develop a new product which will help developers automatically check the code they are writing. 
    - You need to write a short script for checking that every HTML tag that is open has its proper closure.
    - You have an example of a string containing HTML tags: "<title>The Data Science Company</title>"
    - You learn that an opening HTML tag is always at the beginning of the string, and appears inside "<>". 
    - A closing tag also appears inside "<>", but it is preceded by "/".
    - The list html_tags, contains strings with HTML tags.

    - Regex to match closed HTML tags: find if there is a match in each string of the list html_tags. Assign the result to match_tag;
        - If a match is found, print the first group captured and saved in match_tag;
    - If no match is found, regex to match only the text inside the HTML tag. Assign it to notmatch_tag.
        - Print the first group captured by the regex and save it in notmatch_tag.
        - To capture the text inside <>, place parenthesis around the expression: r"<(text)>. To confirm that the same text appears in the closing tag, reference back to the m group captured by using '\m'.
        - To print the 'm' group captured, use .group(m).

        for string in html_tags:
            # Complete the regex and find if it matches a closed HTML tags
            match_tag =  re.match(r"<(\w+)>.*?</\1>", string)

            if match_tag:
                # If it matches print the first group capture
                print("Your tag {} is closed".format(match_tag.group(1))) 
            else:
                # If it doesn't match capture only the tag 
                notmatch_tag = re.match(r"<(\w+)>",string)
                # Print the first group capture
                print("Close your {} tag!".format(notmatch_tag.group(1)))

    3. Your task is to replace elongated words that appear in the tweets. 
    - We define an elongated word as a word that contains a repeating character twice or more times. 
        - e.g. "Awesoooome".
    - Replacing those words is very important since a classifier will treat them as a different term from the source words, lowering their frequency.
    - To find them, you will use capturing groups and reference them back using numbers. E.g \4.
    - If you want to find a match for 'Awesoooome', you firstly need to capture 'Awes'. 
        - Then, match 'o' and reference the same character back, and then, 'me'.
    - The list sentiment_analysis contains the text tweets.
    - Regular expression to match an elongated word as described.
    - Search the elements in sentiment_analysis list to find out if they contain elongated words. Assign the result to match_elongated.
    - Assign the captured group number zero to the variable elongated_word.
        - Print the result contained in the variable elongated_word.

        # Complete the regex to match an elongated word
        regex_elongated = r"\w*(\w)\1*me\w*"

        for tweet in sentiment_analysis:
            # Find if there is a match in each tweet 
            match_elongated = re.search(regex_elongated, tweet)

            if match_elongated:
                # Assign the captured group zero 
                elongated_word = match_elongated.group(0)

                # Complete the format method to print the word
                print("Elongated word found: {word}".format(word=elongated_word))
            else:
                print("No elongated word found") 

            """
        
        # regex lookaround
        self.help_text_8 = """
    Lookaround
    - There are specific types of non-capturing groups that help us look around an expression.
    - Look-around will look for what is behind or ahead of a pattern. 
    - Imagine that we have the following string:
        
        text = "the white cat sat on the chair"

    - We want to see what is surrounding a specific word. 
    - For example, we position ourselves in the word 'cat'. 
    - So look-around will let us answer the following problem: 
        - At my current position, look ahead and search if 'sat' is there. 
        - Or, look behind and search if 'white' is there.
        
    - In other words, looking around allows us to confirm that a sub-pattern is ahead or behind the main pattern.
    - "At my current position in the matching process, look ahead or behind and examine whether some pattern matches or not match before continuing."
    - In the previous example, we are looking for the word 'cat'. 
    - The look ahead expression can be either positive or negative. For positive we use ?=. For negative, ?!.
        - positive: (?=sat)
        - negative: (?!run)

    - Look-ahead
    - This non-capturing group checks whether the first part of the expression is followed or not by the lookahead expression. 
    - As a consequence, it will return the first part of the expression. 
        - Let's imagine that we have a string containing file names and the status of that file. 
        - We want to extract only those files that are followed by the word 'transferred'. 
        - So we start building the regex by indicating any word character followed by .txt.
        - We now indicate we want the first part to be followed by the word transferred. 
        - We do so by writing ?= and then whitespace transferred all inside the parenthesis:
        
        my_text ="tweets.txt transferred, mypass.txt transferred, keywords.txt error"
        re.findall(r"\w+\.txt(?=\stransferred)", my_text) - returns: ['tweets.txt', 'mypass.txt']

    - Negative look-ahead
        - Now, let's use negative lookahead in the same example.
        - In this case, we will say that we want those matches that are NOT followed by the expression 'transferred'. 
        - We use, instead, ?! inside parenthesis:

        my_text = "tweets.txt transferred, mypass.txt transferred, keywords.txt error"
        re.findall(r"\w+\.txt(?!\stransferred)", my_text) - returns: ['keywords.txt']

    - Look-behind
    - The non-capturing group look-behind gets all matches that are preceded or not by a specific pattern.
    - As a consequence, it will return the matches after the look expression.
    - Look behind expression can also be either positive or negative. 
        - For positive, we use ?<=. For negative, ?<!.
        - So, we add an intermediate '<' (angle bracket) sign. In the previous example, we can look before the word 'cat': 
            - positive: (?<=white)
            - negative: (?<!brown)
        
    - Positive look-behind
        - Let's look at the following string, in which we want to find all matches of the names that are preceded by the word 'member'. 
        - We construct our regex with positive look-behind. 
        - At the end of the regex, we indicate that we want a sequence of word characters whitespace another sequence of word characters:
        
        my_text = "Member: Angus Young, Member: Chris Slade, Past: Malcolm Young, Past: Cliff Williams."
        re.findall(r"(?<=Member:\s)\w+\s\w+", my_text) - returns: ['Angus Young', 'Chris Slade']
        
        - Pay attention to the code: the look-behind expression goes before that expression. 
        - We indicate ?<= followed by member, colon, and whitespace. All inside parentheses. 
        - In that way we get the two names that were preceded by the word member, as shown in the output.

    - Negative look-behind
    - Now, we have other string, in which will use negative look-behind. 
    - We will find all matches of the word 'cat' or 'dog' that are not preceded by the word 'brown'. 
    - In this example, we use ?<!, followed by brown, whitespace. All inside the parenthesis. 
    - Then, we indicate our alternation group: 'cat' or 'dog'. 

        my_text = "My white cat sat at the table. However, my brown dog was lying on the couch."
        re.findall(r"(?<!brown\s)(cat|dog)", my_text) - returns: ['cat']

        - Consequently, we get 'cat' as an output, the 'cat' or 'dog' word that is not after the word 'brown'.

    In summary:
    - Positive lookahead (?=) makes sure that first part of the expression is followed by the lookahead expression. 
    - Positive lookbehind (?<=) returns all matches that are preceded by the specified pattern.
    - It is important to know that positive lookahead will return the text matched by the first part of the expression after asserting that it is followed by the lookahead expression,
        - while positive lookbehind will return all matches that follow a specific pattern.
    - Negative lookarounds work in a similar way to positive lookarounds. 
        - They are very helpful when we are looking to exclude certain patterns from our analysis.

    Examples of regex:

    1. You are interested in the words surrounding 'python'. You want to count how many times a specific words appears right before and after it.
    - The variable sentiment_analysis contains the text of one tweet.
    - Get all the words that are followed by the word 'python' in sentiment_analysis. 
    - Print out the word found.
        - In re.findall(). Use \w+ to match the words followed by the word 'python';
        - In re.findall() first argument, include \spython within parentheses to indicate that everything after the word 'python' should be matched.

        # Positive lookahead
        look_ahead = re.findall(r"\w+(?=\spython)", sentiment_analysis)

        # Print out
        print(look_ahead)
    
    1.2. Get all the words that are preceded by the word 'python' or 'Python' in sentiment_analysis. Print out the words found.
    - In re.findall() first argument, include [Pp]ython\s within parentheses to indicate that everything before the word 'python' (or 'Python') should be matched.

        # Positive lookbehind
        look_behind = re.findall(r"(?<=[pP]ython\s)\w+", sentiment_analysis)

        # Print out
        print(look_behind)

    2. You need to write a script for a cell-phone searcher. 
    - It should scan a list of phone numbers and return those that meet certain characteristics.
    - The phone numbers in the list have the structure:
        - Optional area code: 3 numbers
        - Prefix: 4 numbers
        - Line number: 6 numbers
        - Optional extension: 2 numbers
        - E.g. 654-8764-439434-01.
    - You decide to use .findall() and the non-capturing group's negative lookahead (?!) and negative lookbehind (?<!).
    - The list cellphones, contains three phone numbers:
        cellphones = ['4564-646464-01', '345-5785-544245', '6476-579052-01']

    - Get all cell phones numbers that are not preceded by the optional area code.
        - In re.findall() first argument, you use a negative lookbehind ?<! within parentheses () indicating the optional area code.

        for phone in cellphones:
            # Get all phone numbers not preceded by area code
            number = re.findall(r"(?<!\d{3}-)\d{4}-\d{6}-\d{2}", phone)
            print(number)
    
    2.1. Get all the cell phones numbers that are not followed by the optional extension.
        - In re.findall() first argument, you use a negative lookahead ?! within parentheses () indicating the optional extension.

        for phone in cellphones:
            # Get all phone numbers not followed by optional extension
            number = re.findall(r"\d{3}-\d{4}-\d{6}(?!-\d{2})", phone)
            print(number)
        
            """
        
    def show_screen (self):
            
        helper_screen = self.helper_screen
        helper_menu_1 = self.helper_menu_1
            
        if (helper_screen == 0):
                
            # Start screen
            print(self.helper_menu_1)
            print("\n")
            # For the input, strip all whitespaces and, and so convert it to integer:
            helper_screen = int(str(input("Next screen:")).strip())
                
            # the object.__dict__ method returns all attributes from an object as a dictionary.
            # Analogously, the vars function applied to an object vars(object) returns the same
            # dictionary. We can access an attribute from the object by passing the key of this
            # dictionary:
            # vars(object)['key']
                
            while (helper_screen != 10):
                    
                if (helper_screen not in range(0, 11)):
                    # range (0, 11): integers from 0 to 10
                        
                    helper_screen = int(str(input("Input a valid number, from 0 to 10:")).strip())
                    
                else:
                        
                    if (helper_screen == 9):
                        # print all at once:
                        for screen_number in range (1, 9):
                            # integers from 1 to 8
                            key = "help_text_" + str(screen_number)
                            # apply the vars function to get the dictionary of attributes, and call the
                            # attribute by passing its name as key from the dictionary:
                            screen_text = vars(self)[key]
                            # Notice that we cannot directly call the attribute as a string. We would have to
                            # create an if else for each of the 8 attributes.
                            print(screen_text)
                            
                        # Now, make helper_screen = 10 for finishing this step:
                        helper_screen = 10
                        
                    else:
                        key = "help_text_" + str(helper_screen)
                        screen_text = vars(self)[key]
                        print(screen_text)
                        helper_screen = int(str(input("Next screen:")).strip())
            
        print("Finishing regex assistant.\n")
            
        return self


@dataclass
class etl_workflow:

    # Aggregate dataframes and Manipulate Timestamps.
    # Characterize the dataset.
    # Transform the dataset.
    # Analyze the time series.
    # Seggregate the dataset and check for differences.
    # Process diagnosis: statistical process control charts and capability analysis.

    def APPLY_ROW_FILTERS_LIST (df, list_of_row_filters):
        
        import numpy as np
        import pandas as pd
        
        print("Warning: this function filter the rows and results into a smaller dataset, since it removes the non-selected entries.")
        print("If you want to pass a filter to simply label the selected rows, use the function LABEL_DATAFRAME_SUBSETS, which do not eliminate entries from the dataframe.")
        
        # This function applies filters to the dataframe and remove the non-selected entries.
        
        # df: dataframe to be analyzed.
        
        ## define the filters and only them define the filters list
        # EXAMPLES OF BOOLEAN FILTERS TO COMPOSE THE LIST
        # boolean_filter1 = ((None) & (None)) 
        # (condition1 AND (&) condition2)
        # boolean_filter2 = ((None) | (None)) 
        # condition1 OR (|) condition2
        
        # boolean filters result into boolean values True or False.

        ## Examples of filters:
        ## filter1 = (condition 1) & (condition 2)
        ## filter1 = (df['column1'] > = 0) & (df['column2']) < 0)
        ## filter2 = (condition)
        ## filter2 = (df['column3'] <= 2.5)
        ## filter3 = (df['column4'] > 10.7)
        ## filter3 = (condition 1) | (condition 2)
        ## filter3 = (df['column5'] != 'string1') | (df['column5'] == 'string2')

        ## comparative operators: > (higher); >= (higher or equal); < (lower); 
        ## <= (lower or equal); == (equal); != (different)

        ## concatenation operators: & (and): the filter is True only if the 
        ## two conditions concatenated through & are True
        ## | (or): the filter is True if at least one of the two conditions concatenated
        ## through | are True.
        ## ~ (not): inverts the boolean, i.e., True becomes False, and False becomes True. 

        ## separate conditions with parentheses. Use parentheses to define a order
        ## of definition of the conditions:
        ## filter = ((condition1) & (condition2)) | (condition3)
        ## Here, firstly ((condition1) & (condition2)) = subfilter is evaluated. 
        ## Then, the resultant (subfilter) | (condition3) is evaluated.

        ## Pandas .isin method: you can also use this method to filter rows belonging to
        ## a given subset (the row that is in the subset is selected). The syntax is:
        ## is_black_or_brown = dogs["color"].isin(["Black", "Brown"])
        ## or: filter = (dataframe_column_series).isin([value1, value2, ...])
        # The negative of this condition may be acessed with ~ operator:
        ##  filter = ~(dataframe_column_series).isin([value1, value2, ...])
        ## Also, you may use isna() method as filter for missing values:
        ## filter = (dataframe_column_series).isna()
        ## or, for not missing: ~(dataframe_column_series).isna()
        
        # list_of_row_filters: list of boolean filters to be applied to the dataframe
        # e.g. list_of_row_filters = [filter1]
        # applies a single filter saved as filter 1. Notice: even if there is a single
        # boolean filter, it must be declared inside brackets, as a single-element list.
        # That is because the function will loop through the list of filters.
        # list_of_row_filters = [filter1, filter2, filter3, filter4]
        # will apply, in sequence, 4 filters: filter1, filter2, filter3, and filter4.
        # Notice that the filters must be declared in the order you want to apply them.
        
        
        # Set a local copy of the dataframe:
        DATASET = df.copy(deep = True)
        
        # Get the original index and convert it to a list
        original_index = list(DATASET.index)
        
        # Loop through the filters list, applying the filters sequentially:
        # Each element of the list is identified as 'boolean_filter'
        
        if (len(list_of_row_filters) > 0):
            
            # Start a list of indices that were removed. That is because we must
            # remove these elements from the boolean filter series before filtering, avoiding
            # the index mismatch.
            removed_indices = []
            
            # Now, loop through other rows in the list_of_row_filters:
            for boolean_series in list_of_row_filters:
                
                if (len(removed_indices) > 0):
                    # Drop rows in list removed_indices. Set inplace = True to remove by simply applying
                    # the method:
                    # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.drop.html
                    boolean_series.drop(labels = removed_indices, axis = 0, inplace = True)
                
                # Apply the filter:
                DATASET = DATASET[boolean_series]
                
                # Finally, let's update the list of removed indices:
                for index in original_index:
                    if ((index not in list(DATASET.index)) & (index not in removed_indices)):
                        removed_indices.append(index)
        
        
        # Reset index:
        DATASET = DATASET.reset_index(drop = True)
        
        print("Successfully filtered the dataframe. Check the 10 first rows of the filtered and returned dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        return DATASET


    def MERGE_ON_TIMESTAMP (df_left, df_right, left_key, right_key, how_to_join = "inner", merge_method = 'asof', merged_suffixes = ('_left', '_right'), asof_direction = 'nearest', ordered_filling = 'ffill'):
        
        #WARNING: Only two dataframes can be merged on each call of the function.
        
        import numpy as np
        import pandas as pd
        
        # df_left: dataframe to be joined as the left one.
        
        # df_right: dataframe to be joined as the right one
        
        # left_key: (String) name of column of the left dataframe to be used as key for joining.
        
        # right_key: (String) name of column of the right dataframe to be used as key for joining.
        
        # how_to_join: joining method: "inner", "outer", "left", "right". The default is "inner".
        
        # merge_method: which pandas merging method will be applied:
        # merge_method = 'ordered' for using the .merge_ordered method.
        # merge_method = "asof" for using the .merge_asof method.
        # WARNING: .merge_asof uses fuzzy matching, so the how_to_join parameter is not applicable.
        
        # merged_suffixes = ('_left', '_right') - tuple of the suffixes to be added to columns
        # with equal names. Simply modify the strings inside quotes to modify the standard
        # values. If no tuple is provided, the standard denomination will be used.
        
        # asof_direction: this parameter will only be used if the .merge_asof method is
        # selected. The default is 'nearest' to merge the closest timestamps in both 
        # directions. The other options are: 'backward' or 'forward'.
        
        # ordered_filling: this parameter will only be used on the merge_ordered method.
        # The default is None. Input ordered_filling = 'ffill' to fill missings with the
        # previous value.
        
        # Create dataframe local copies to manipulate, avoiding that Pandas operates on
        # the original objects; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DF_LEFT = df_left.copy(deep = True)
        DF_RIGHT = df_right.copy(deep = True)
        
        # Firstly, let's guarantee that the keys were actually read as timestamps of the same type.
        # We will do that by converting all values to np.datetime64. If fails, then
        # try to convert to Pandas timestamps.
        
        # try parsing as np.datetime64:
        try:
            DF_LEFT[left_key] = DF_LEFT[left_key].astype(np.datetime64)
            DF_RIGHT[right_key] = DF_RIGHT[right_key].astype(np.datetime64)
        
        except:
            # Try conversion to pd.Timestamp (less efficient, involves looping)
            # 1. Start lists to store the Pandas timestamps:
            timestamp_list_left = []
            timestamp_list_right = []

            # 2. Loop through each element of the timestamp columns left_key and right_key, 
            # and apply the function to guarantee that all elements are Pandas timestamps

            # left dataframe:
            for timestamp in DF_LEFT[left_key]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column]
                timestamp_list_left.append(pd.Timestamp(timestamp, unit = 'ns'))

            # right dataframe:
            for timestamp in DF_RIGHT[right_key]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column]
                timestamp_list_right.append(pd.Timestamp(timestamp, unit = 'ns'))

            # 3. Set the key columns as the lists of objects converted to Pandas dataframes:
            DF_LEFT[left_key] = timestamp_list_left
            DF_RIGHT[right_key] = timestamp_list_right
        
        
        # Now, even if the dates were read as different types of variables (like string for one
        # and datetime for the other), we converted them to a same type (Pandas timestamp), avoiding
        # compatibility issues.
        
        # For performing merge 'asof', the timestamps must be previously sorted in ascending order.
        # Pandas sort_values method: https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.sort_values.html
        # Let's sort the dataframes in ascending order of timestamps before merging:
        
        DF_LEFT = DF_LEFT.sort_values(by = left_key, ascending = True)
        DF_RIGHT = DF_RIGHT.sort_values(by = right_key, ascending = True)
        
        # Reset indices:
        DF_LEFT = DF_LEFT.reset_index(drop = True)
        DF_RIGHT = DF_RIGHT.reset_index(drop = True)
            
        
        if (merge_method == 'ordered'):
        
            if (ordered_filling == 'ffill'):
                
                merged_df = pd.merge_ordered(DF_LEFT, DF_RIGHT, left_on = left_key, right_on = right_key, how = how_to_join, suffixes = merged_suffixes, fill_method='ffill')
            
            else:
                
                merged_df = pd.merge_ordered(DF_LEFT, DF_RIGHT, left_on = left_key, right_on = right_key, how = how_to_join, suffixes = merged_suffixes)
        
        elif (merge_method == 'asof'):
            
            merged_df = pd.merge_asof(DF_LEFT, DF_RIGHT, left_on = left_key, right_on = right_key, suffixes = merged_suffixes, direction = asof_direction)
        
        else:
            
            print("You did not enter a valid merge method for this function, \'ordered\' or \'asof\'.")
            print("Then, applying the conventional Pandas .merge method, followed by .sort_values method.\n")
            
            #Pandas sort_values method: https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.sort_values.html
            
            merged_df = DF_LEFT.merge(DF_RIGHT, left_on = left_key, right_on = right_key, how = how_to_join, suffixes = merged_suffixes)
            merged_df = merged_df.sort_values(by = merged_df.columns[0], ascending = True)
            #sort by the first column, with index 0.
        
        # Now, reset index positions of the merged dataframe:
        merged_df = merged_df.reset_index(drop = True)
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Dataframe successfully merged. Check its 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(merged_df.head(10))
                
        except: # regular mode
            print(merged_df.head(10))
        
        return merged_df


    def MERGE_AND_SORT_DATAFRAMES (df_left, df_right, left_key, right_key, how_to_join = "inner", merged_suffixes = ('_left', '_right'), sort_merged_df = False, column_to_sort = None, ascending_sorting = True):
        
        #WARNING: Only two dataframes can be merged on each call of the function.
        
        import numpy as np
        import pandas as pd
        
        # df_left: dataframe to be joined as the left one.
        
        # df_right: dataframe to be joined as the right one
        
        # left_key: (String) name of column of the left dataframe to be used as key for joining.
        
        # right_key: (String) name of column of the right dataframe to be used as key for joining.
        
        # how_to_join: joining method: "inner", "outer", "left", "right". The default is "inner".
        
        # merge_method: which pandas merging method will be applied:
        # merge_method = 'ordered' for using the .merge_ordered method.
        # merge_method = "asof" for using the .merge_asof method.
        # WARNING: .merge_asof uses fuzzy matching, so the how_to_join parameter is not applicable.
        
        # merged_suffixes = ('_left', '_right') - tuple of the suffixes to be added to columns
        # with equal names. Simply modify the strings inside quotes to modify the standard
        # values. If no tuple is provided, the standard denomination will be used.
        
        # sort_merged_df = False not to sort the merged dataframe. If you want to sort it,
        # set as True. If sort_merged_df = True and column_to_sort = None, the dataframe will
        # be sorted by its first column.
        
        # column_to_sort = None. Keep it None if the dataframe should not be sorted.
        # Alternatively, pass a string with a column name to sort, such as:
        # column_to_sort = 'col1'; or a list of columns to use for sorting: column_to_sort = 
        # ['col1', 'col2']
        
        # ascending_sorting = True. If you want to sort the column(s) passed on column_to_sort in
        # ascending order, set as True. Set as False if you want to sort in descending order. If
        # you want to sort each column passed as list column_to_sort in a specific order, pass a 
        # list of booleans like ascending_sorting = [False, True] - the first column of the list
        # will be sorted in descending order, whereas the 2nd will be in ascending. Notice that
        # the correspondence is element-wise: the boolean in list ascending_sorting will correspond 
        # to the sorting order of the column with the same position in list column_to_sort.
        # If None, the dataframe will be sorted in ascending order.
        
        # Create dataframe local copies to manipulate, avoiding that Pandas operates on
        # the original objects; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DF_LEFT = df_left.copy(deep = True)
        DF_RIGHT = df_right.copy(deep = True)
        
        # check if the keys are the same:
        boolean_check = (left_key == right_key)
        # if boolean_check is True, we will merge using the on parameter, instead of left_on and right_on:
        
        if (boolean_check): # runs if it is True:
            
            merged_df = DF_LEFT.merge(DF_RIGHT, on = left_key, how = how_to_join, suffixes = merged_suffixes)
        
        else:
            # use left_on and right_on
            merged_df = DF_LEFT.merge(DF_RIGHT, left_on = left_key, right_on = right_key, how = how_to_join, suffixes = merged_suffixes)
        
        # Check if the dataframe should be sorted:
        if (sort_merged_df == True):
            
            # check if column_to_sort = None. If it is, set it as the first column (index 0):
            if (column_to_sort is None):
                
                column_to_sort = merged_df.columns[0]
                print(f"Sorting merged dataframe by its first column = {column_to_sort}\n")
            
            # check if ascending_sorting is None. If it is, set it as True:
            if (ascending_sorting is None):
                
                ascending_sorting = True
                print("Sorting merged dataframe in ascending order.\n")
            
            # Now, sort the dataframe according to the parameters:
            merged_df = merged_df.sort_values(by = column_to_sort, ascending = ascending_sorting)
            #sort by the first column, with index 0.
        
            # Now, reset index positions:
            merged_df = merged_df.reset_index(drop = True)
            print("Merged dataframe successfully sorted.\n")
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Dataframe successfully merged. Check its 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(merged_df.head(10))
                
        except: # regular mode
            print(merged_df.head(10))
        
        return merged_df


    def RECORD_LINKAGE (df_left, df_right, columns_to_block_as_basis_for_comparison = {'left_df_column': None, 'right_df_column': None}, columns_where_exact_matches_are_required = [{'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}], columns_where_similar_strings_should_be_found = [{'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}, {'left_df_column': None, 'right_df_column': None}], threshold_for_percent_of_similarity = 80.0):
        
        #WARNING: Only two dataframes can be merged on each call of the function.
        
        import numpy as np
        import pandas as pd
        import recordlinkage
        
        # Record linkage is the act of linking data from different sources regarding the same entity.
        # Generally, we clean two or more DataFrames, generate pairs of potentially matching records, 
        # score these pairs according to string similarity and other similarity metrics, and link them.
        # Example: we may want to merge data from different clients using the address as key, but there may
        # be differences on the format used for registering the addresses.
        
        # df_left: dataframe to be joined as the left one.
        
        # df_right: dataframe to be joined as the right one
        
        # columns_to_block_as_basis_for_comparison = {'left_df_column': None, 'right_df_column': None}
        # Dictionary of strings, in quotes. Do not change the keys. If a pair of columns should be
        # blocked for being used as basis for merging declare here: in 'left_df_column', input the name
        # of the column of the left dataframe. In right_df_column, input the name of the column on the right
        # dataframe.
        # We first want to generate pairs between both DataFrames. Ideally, we want to generate all
        # possible pairs between our DataFrames.
        # But, if we had big DataFrames, it is possible that we ende up having to generate millions, 
        # if not billions of pairs. It would not prove scalable and could seriously hamper development time.
        
        # This is where we apply what we call blocking, which creates pairs based on a matching column, 
        # reducing the number of possible pairs.
        
        # threshold_for_percent_of_similarity = 80.0 - 0.0% means no similarity and 100% means equal strings.
        # The threshold_for_percent_of_similarity is the minimum similarity calculated from the
        # Levenshtein (minimum edit) distance algorithm. This distance represents the minimum number of
        # insertion, substitution or deletion of characters operations that are needed for making two
        # strings equal.
        
        # columns_where_exact_matches_are_required = [{'left_df_column': None, 'right_df_column': None}]
        # columns_where_similar_strings_should_be_found = [{'left_df_column': None, 'right_df_column': None}]
        
        # Both of these arguments have the same structure. The single difference is that
        # columns_where_exact_matches_are_required is referent to a group of columns (or a single column)
        # where we require perfect correspondence between the dataframes, i.e., where no differences are
        # tolerated. Example: the month and day numbers must be precisely the same.
        # columns_where_similar_strings_should_be_found, in turns, is referent to the columns where there
        # is no rigid standard in the dataset, so similar values should be merged as long as the similarity
        # is equal or higher than threshold_for_percent_of_similarity.
        
        # Let's check the structure for these arguments, using columns_where_similar_strings_should_be_found
        # as example. All instructions are valid for columns_where_exact_matches_are_required.
        
        # columns_where_similar_strings_should_be_found =
        # [{'left_df_column': None, 'right_df_column': None}]
        # This is a list of dictionaries, where each dictionary contains two key-value pairs:
        # the first one contains the column name on the left dataframe; and the second one contains the 
        # correspondent column on the right dataframe.
        # The function will loop through all dictionaries in
        # this list, access the values of the key 'left_df_column' to retrieve a column to analyze in the left 
        # dataframe; and access access the key 'righ_df_column' to obtain the correspondent column in the right
        # dataframe. Then, it will look for potential matches.
        # For columns_where_exact_matches_are_required, only columns with perfect correspondence will be
        # retrieved. For columns_where_similar_strings_should_be_found, when the algorithm finds a correspondence
        # that satisfies the threshold criterium, it will assign it as a match. 
        # For instance, suppose you have a word written in too many ways, in a column named 'continent' that
        # should be used as key: "EU" , "eur" , "Europ" , "Europa" , "Erope" , "Evropa" ...
        # Since they have sufficient similarity, they will be assigned as matching.
        
        # The objects columns_where_similar_strings_should_be_found and
        # columns_where_exact_matches_are_required must be declared as lists, 
        # in brackets, even if there is a single dictionary.
        # Use always the same keys: 'left_df_column' and 'right_df_column'.
        # Notice that this function performs fuzzy matching, so it MAY SEARCH substrings and strings
        # written with different cases (upper or lower) when this portions or modifications make the
        # strings sufficiently similar to each other.
        
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to replace more
        # values.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'left_df_column': df_left_column, 'right_df_column': df_right_column}, 
        # where df_left_column and df_right_column represent the strings for searching and replacement 
        # (If the key contains None, the new dictionary will be ignored).
        
        
        print("Record linkage attempts to join data sources that have similarly fuzzy duplicate values.")
        print("The object is to end up with a final DataFrame with no duplicates by using string similarity.\n")
        
        # Create dataframe local copies to manipulate, avoiding that Pandas operates on
        # the original objects; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DF_LEFT = df_left.copy(deep = True)
        DF_RIGHT = df_right.copy(deep = True)
        
        # If an invalid value was set for threshold_for_percent_of_similarity, correct it to 80% standard:
                    
        if(threshold_for_percent_of_similarity is None):
            threshold_for_percent_of_similarity = 80.0
                    
        if((threshold_for_percent_of_similarity == np.nan) | (threshold_for_percent_of_similarity < 0)):
            threshold_for_percent_of_similarity = 80.0
        
        # Convert the threshold for fraction (as required by recordlinkage) and save it as THRESHOLD:
        THRESHOLD = threshold_for_percent_of_similarity/100
        
        # Before finding the pairs, let's check if the column names on the lists of dictionaries are
        # the same. If they are not, let's rename the right dataframe columns so that they are equal to
        # the left one.
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.rename.html
        
        # Start a list of valid columns for exact matches:
        valid_columns_exact_matches = []
        
        for dictionary in columns_where_exact_matches_are_required:
            
            # Access elements on keys 'left_df_column' and 'right_df_column':
            left_df_column = dictionary['left_df_column']
            right_df_column = dictionary['right_df_column']
            
            # Check if no key is None:
            if ((left_df_column is not None) & (right_df_column is not None)):
                
                # If right_df_column is different from left_df_column, rename them to
                # make them equal:
                if (left_df_column != right_df_column):
                    
                    DF_RIGHT.rename(columns = {right_df_column: left_df_column}, inplace = True)
                
                # Add the column to the validated list:
                valid_columns_exact_matches.append(left_df_column)
        
        # Repeat the procedure for the other list:
        # Start a list of valid columns for similar strings:
        valid_columns_similar_str = []
        for dictionary in columns_where_similar_strings_should_be_found:
            
            # Access elements on keys 'left_df_column' and 'right_df_column':
            left_df_column = dictionary['left_df_column']
            right_df_column = dictionary['right_df_column']
            
            # Check if no key is None:
            if ((left_df_column is not None) & (right_df_column is not None)):
                
                # If right_df_column is different from left_df_column, rename them to
                # make them equal:
                if (left_df_column != right_df_column):
                    
                    DF_RIGHT.rename(columns = {right_df_column: left_df_column}, inplace = True)         
                
                # Add the column to the validated list:
                valid_columns_similar_str.append(left_df_column)
    
        # Now, we can create the objects for linkage:
        
        # Create an indexer object and find possible pairs:
        indexer = recordlinkage.Index()
        
        left_block = columns_to_block_as_basis_for_comparison['left_df_column']
        
        # Check if left_block is in one of the validated columns list. If it is, right_block
        # may have been renamed, and has the same label left_block:
        if ((left_block in valid_columns_exact_matches) | (left_block in valid_columns_similar_str)):
            
            right_block = left_block
        
        else:
            # right_block was not evaluated yet
            right_block = columns_to_block_as_basis_for_comparison['right_df_column']
        
        if ((left_block is not None) & (right_block is not None)):
            # If they are different, make them equal:
            if (left_block != right_block):
                    
                DF_RIGHT.rename(columns = {right_block: left_block}, inplace = True)
            
            # block pairing in this column:
            indexer.block(left_block)
        
        elif (left_block is not None):
            # Try accessing this column on right dataframe:
            try:
                column_block = DF_RIGHT[left_block]
                # If no exception was raised, the column is actually present:
                indexer.block(left_block)
            except:
                pass
        
        elif (right_block is not None):
        # Try accessing this column on left dataframe:
            try:
                column_block = DF_LEFT[right_block]
                # If no exception was raised, the column is actually present:
                indexer.block(right_block)
            except:
                pass
        
        # Now that the columns were renaimed, we can generate the pairs from the object indexer:
        # Generate pairs
        pairs = indexer.index(DF_LEFT, DF_RIGHT)
        # The resulting object, is a pandas multi index object containing pairs of row indices from both 
        # DataFrames, i.e., it is an array containing possible pairs of indices that makes it much easier 
        # to subset DataFrames on.
        
        # Comparing the DataFrames
        # Since we've already generated our pairs, it's time to find potential matches. 
        # We first start by creating a comparison object using the recordlinkage dot compare function. 
        # This is similar to the indexing object we created while generating pairs, but this one is 
        # responsible for assigning different comparison procedures for pairs. 
        # Let's say there are columns for which we want exact matches between the pairs. To do that, 
        # we use the exact method. It takes in the column name in question for each DataFrame, 
        # and a label argument which lets us set the column name in the resulting DataFrame. 
        # Now in order to compute string similarities between pairs of rows for columns that have 
        # fuzzy values, we use the dot string method, which also takes in the column names in question, 
        # the similarity cutoff point in the threshold argument, which takes in a value between 0 and 1.
        # Finally to compute the matches, we use the compute function, which takes in the possible pairs, 
        # and the two DataFrames in question. Note that you need to always have the same order of DataFrames
        # when inserting them as arguments when generating pairs, comparing between columns, 
        # and computing comparisons.
        
        # Create a comparison object
        comp_cl = recordlinkage.Compare()
        
        # Create a counter for assessing the total number of valid columns being analyzed:
        column_counter = len(valid_columns_exact_matches) + len(valid_columns_similar_str)
        
        # Find exact matches for the columns in the list columns_where_exact_matches_are_required.
        # Loop through all elements from the list:
        
        for valid_column in valid_columns_exact_matches:
                
            # set column as the label for merged column:
            LABEL = valid_column
                
            # Find the exact matches:
            comp_cl.exact(valid_column, valid_column, label = LABEL)
    
        # Now, let's repeat the procedure for the columns where we will look for similar strings
        # for fuzzy matching. These columns were indicated in columns_where_similar_strings_should_be_found.
        # So, loop through all elements from this list:
        for valid_column in valid_columns_similar_str:
            
            # set column as the label for merged column:
            LABEL = valid_column
                
            # Find similar matches:
            comp_cl.string(valid_column, valid_column, label = LABEL, threshold = THRESHOLD) 

        # Now, compute the comparison of the pairs by using the .compute() method of comp_cl,
        # i.e, get potential matches:
        potential_matches = comp_cl.compute(pairs, DF_LEFT, DF_RIGHT)
        # potential_matches is a multi index DataFrame, where the first index is the row index from 
        # the first DataFrame (left), and the second index is a list of all row indices in the right dataframe. 
        # The columns are the columns being compared, with values being 1 for a match, and 0 for not a match.
        
        # The columns of our potential matches are the columns we chose to link both DataFrames on, 
        # where the value is 1 for a match, and 0 otherwise.
        # The first step in linking DataFrames, is to isolate the potentially matching pairs to the ones 
        # we're pretty sure of. We can do it by subsetting the rows where the row sum is above a certain 
        # number of columns: column_counter - 1 (i.e., where the match occurs for all columns, or do not
        # happen for a single column).
        # Isolate potential matches with row sum >=column_counter - 1
        matches = potential_matches[potential_matches.sum(axis = 1) >= (column_counter - 1)]
        
        # matches is row indices between DF_LEFT and DF_RIGHT that are most likely duplicates. 
        # Our next step is to extract the one of the index columns, and subsetting its associated 
        # DataFrame to filter for duplicates.
        
        # Get values of second column index of matches (i.e., indices for DF_RIGHT only).
        # We can access a DataFrame's index using the index attribute. Since this is a multi index 
        # DataFrame, it returns a multi index object containing pairs of row indices from DF_LEFT and 
        # DF_RIGHT respectively. We want to extract all DF_RIGHT indices, so we chain it with the 
        # get_level_values method, which takes in which column index we want to extract its values. 
        # We can either input the index column's name, or its order, which is in this case 1.
        matching_indices = matches.index.get_level_values(1)

        # Subset DF_RIGHT on non-duplicate values (i.e., removing the duplicates 
        # selected as matching_indices).
        # To find the duplicates in DF_RIGHTDF_RIGHT, we can simply subset on all indices of DF_RIGHT, 
        # with the ones found through record linkage. 
        # You can choose to examine them further for similarity with their duplicates in DF_LEFT, 
        # but if you're sure of your analysis, you can go ahead and find the non duplicates with 
        # the exact same line of code, except by adding a tilde at the beginning of your subset. 
        non_dup = DF_RIGHT[~DF_RIGHT.index.isin(matching_indices)]
        # ~ is the not (invert) operator: 
        # https://stackoverflow.com/questions/21415661/logical-operators-for-boolean-indexing-in-pandas
        
        # Append non_dup to DF_LEFT.
        # Now that you have your non duplicates, all you need is a simple append 
        # using the DataFrame append method of DF_LEFT, and you have your linked Data.
        merged_df = pd.concat([DF_LEFT, DF_RIGHT], axis = 0)
        
        # Now, reset index positions:
        merged_df = merged_df.reset_index(drop = True)
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Dataframe successfully merged. Check its 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(merged_df.head(10))
                
        except: # regular mode
            print(merged_df.head(10))
        
        return merged_df


    def UNION_DATAFRAMES (list_of_dataframes, what_to_append = 'rows', ignore_index_on_union = True, sort_values_on_union = True, union_join_type = None):
        
        import pandas as pd
        #JOIN can be 'inner' to perform an inner join, eliminating the missing values
        #The default (None) is 'outer': the dataframes will be stacked on the columns with
        #same names but, in case there is no correspondence, the row will present a missing
        #value for the columns which are not present in one of the dataframes.
        #When using the 'inner' method, only the common columns will remain
        
        #list_of_dataframes must be a list containing the dataframe objects
        # example: list_of_dataframes = [df1, df2, df3, df4]
        #Notice that the dataframes are objects, not strings. Therefore, they should not
        # be declared inside quotes.
        # There is no limit of dataframes. In this example, we will concatenate 4 dataframes.
        # If list_of_dataframes = [df1, df2, df3] we would concatenate 3, and if
        # list_of_dataframes = [df1, df2, df3, df4, df5] we would concatenate 5 dataframes.
        
        # what_to_append = 'rows' for appending the rows from one dataframe
        # into the other; what_to_append = 'columns' for appending the columns
        # from one dataframe into the other (horizontal or lateral append).
        
        # When what_to_append = 'rows', Pandas .concat method is defined as
        # axis = 0, i.e., the operation occurs in the row level, so the rows
        # of the second dataframe are added to the bottom of the first one.
        # It is the SQL union, and creates a dataframe with more rows, and
        # total of columns equals to the total of columns of the first dataframe
        # plus the columns of the second one that were not in the first dataframe.
        # When what_to_append = 'columns', Pandas .concat method is defined as
        # axis = 1, i.e., the operation occurs in the column level: the two
        # dataframes are laterally merged using the index as the key, 
        # preserving all columns from both dataframes. Therefore, the number of
        # rows will be the total of rows of the dataframe with more entries,
        # and the total of columns will be the sum of the total of columns of
        # the first dataframe with the total of columns of the second dataframe.
        
        #The other parameters are the same from Pandas .concat method.
        # ignore_index_on_union = ignore_index;
        # sort_values_on_union = sort
        # union_join_type = join
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.concat.html
        
        #Check Datacamp course Joining Data with pandas, Chap.3, 
        # Advanced Merging and Concatenating
        
        # Create dataframe local copies to manipulate, avoiding that Pandas operates on
        # the original objects; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        
        # Start a list of copied dataframes:
        LIST_OF_DATAFRAMES = []
        
        # Loop through each element from list_of_dataframes:
        for dataframe in list_of_dataframes:
            
            # create a copy of the object:
            copied_df = dataframe.copy(deep = True)
            # Append this element to the LIST_OF_DATAFRAMES:
            LIST_OF_DATAFRAMES.append(copied_df)
        
        # Check axis:
        if (what_to_append == 'rows'):
            
            AXIS = 0
        
        elif (what_to_append == 'columns'):
            
            AXIS = 1
            
            # In this case, we must save a list of columns of each one of the dataframes, containing
            # the different column names observed. That is because the concat method eliminates the
            # original column names when AXIS = 1
            # We can start the LIST_OF_COLUMNS as the columns from the first object on the
            # LIST_OF_DATAFRAMES, eliminating one iteration cycle. Since the columns method generates
            # an array, we use the list attribute to convert the array to a regular list:
            
            i = 0
            analyzed_df = LIST_OF_DATAFRAMES[i]
            LIST_OF_COLUMNS = list(analyzed_df.columns)
            
            # Now, loop through each other element on LIST_OF_DATAFRAMES. Since index 0 was already
            # considered, start from index 1:
            for i in range (1, len(LIST_OF_DATAFRAMES)):
                
                analyzed_df = LIST_OF_DATAFRAMES[i]
                
                # Now, loop through each column, named 'col', from the list of columns of analyzed_df:
                for col in list(analyzed_df.columns):
                    
                    # If 'col' is not in LIST_OF_COLUMNS, append it to the list with its current name.
                    # The order of the columns on the concatenated dataframe will be the same (the order
                    # they appear):
                    if not (col in LIST_OF_COLUMNS):
                        LIST_OF_COLUMNS.append(col)
                    
                    else:
                        # There is already a column with this name. So, append col with a suffix:
                        LIST_OF_COLUMNS.append(col + "_df_" + str(i))
                        
            # Now, we have a list of all column names, that we will use for retrieving the headers after
            # concatenation.
        
        else:
            print("No valid string was input to what_to_append, so appending rows (vertical append, equivalent to SQL UNION).\n")
            AXIS = 0
        
        if (union_join_type == 'inner'):
            
            print("Warning: concatenating dataframes using the \'inner\' join method, that removes missing values.\n")
            concat_df = pd.concat(LIST_OF_DATAFRAMES, axis = AXIS, ignore_index = ignore_index_on_union, sort = sort_values_on_union, join = union_join_type)
        
        else:
            
            #In case None or an invalid value is provided, use the default 'outer', by simply
            # not declaring the 'join':
            concat_df = pd.concat(LIST_OF_DATAFRAMES, axis = AXIS, ignore_index = ignore_index_on_union, sort = sort_values_on_union)
        
        if (AXIS == 1):
            # If we concatentated columns, we lost the columns' names (headers). So, use the list
            # LIST_OF_COLUMNS as the new headers for this case:
            concat_df.columns = LIST_OF_COLUMNS
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Dataframes successfully concatenated. Check the 10 first rows of new dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(concat_df.head(10))
                
        except: # regular mode
            print(concat_df.head(10))
        
        #Now return the concatenated dataframe:
        
        return concat_df


    def df_general_characterization (df):
        
        import pandas as pd

        # Set a local copy of the dataframe:
        DATASET = df.copy(deep = True)

        # Show dataframe's header
        print("Dataframe\'s 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))

        # Show dataframe's tail:
        # Line break before next information:
        print("\n")
        print("Dataframe\'s 10 last rows:\n")
        try:
            display(DATASET.tail(10))
        except:
            print(DATASET.tail(10))
        
        # Show dataframe's shape:
        # Line break before next information:
        print("\n")
        df_shape  = DATASET.shape
        print("Dataframe\'s shape = (number of rows, number of columns) =\n")
        try:
            display(df_shape)
        except:
            print(df_shape)
        
        # Show dataframe's columns:
        # Line break before next information:
        print("\n")
        df_columns_array = DATASET.columns
        print("Dataframe\'s columns =\n")
        try:
            display(df_columns_array)
        except:
            print(df_columns_array)
        
        # Show dataframe's columns types:
        # Line break before next information:
        print("\n")
        df_dtypes = DATASET.dtypes
        # Now, the df_dtypes seroes has the original columns set as index, but this index has no name.
        # Let's rename it using the .rename method from Pandas Index object:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Index.rename.html#pandas.Index.rename
        # To access the Index object, we call the index attribute from Pandas dataframe.
        # By setting inplace = True, we modify the object inplace, by simply calling the method:
        df_dtypes.index.rename(name = 'dataframe_column', inplace = True)
        # Let's also modify the series label or name:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.rename.html
        df_dtypes.rename('dtype_series', inplace = True)
        print("Dataframe\'s variables types:\n")
        try:
            display(df_dtypes)
        except:
            print(df_dtypes)
        
        # Show dataframe's general statistics for numerical variables:
        # Line break before next information:
        print("\n")
        df_general_statistics = DATASET.describe()
        print("Dataframe\'s general (summary) statistics for numeric variables:\n")
        try:
            display(df_general_statistics)
        except:
            print(df_general_statistics)
        
        # Show total of missing values for each variable:
        # Line break before next information:
        print("\n")
        total_of_missing_values_series = DATASET.isna().sum()
        # This is a series which uses the original column names as index
        proportion_of_missing_values_series = DATASET.isna().mean()
        percent_of_missing_values_series = proportion_of_missing_values_series * 100
        missingness_dict = {'count_of_missing_values': total_of_missing_values_series,
                        'proportion_of_missing_values': proportion_of_missing_values_series,
                        'percent_of_missing_values': percent_of_missing_values_series}
        
        df_missing_values = pd.DataFrame(data = missingness_dict)
        # Now, the dataframe has the original columns set as index, but this index has no name.
        # Let's rename it using the .rename method from Pandas Index object:
        df_missing_values.index.rename(name = 'dataframe_column', inplace = True)
        
        # Create a one row dataframe with the missingness for the whole dataframe:
        # Pass the scalars as single-element lists or arrays:
        one_row_data = {'dataframe_column': ['missingness_accross_rows'],
                        'count_of_missing_values': [len(DATASET) - len(DATASET.copy(deep = True).dropna(how = 'any'))],
                        'proportion_of_missing_values': [(len(DATASET) - len(DATASET.copy(deep = True).dropna(how = 'any')))/(len(DATASET))],
                        'percent_of_missing_values': [(len(DATASET) - len(DATASET.copy(deep = True).dropna(how = 'any')))/(len(DATASET))*100]
                        }
        one_row_df = pd.DataFrame(data = one_row_data)
        one_row_df.set_index('dataframe_column', inplace = True)
        
        # Append this one_row_df to df_missing_values:
        df_missing_values = pd.concat([df_missing_values, one_row_df])
        
        print("Missing values on each feature; and missingness considering all rows from the dataframe:")
        print("(note: \'missingness_accross_rows\' was calculated by: checking which rows have at least one missing value (NA); and then comparing total rows with NAs with total rows in the dataframe).\n")
        
        try:
            display(df_missing_values)
        except:
            print(df_missing_values)
        
        return df_shape, df_columns_array, df_dtypes, df_general_statistics, df_missing_values


    def drop_columns_or_rows (df, what_to_drop = 'columns', cols_list = None, row_index_list = None, reset_index_after_drop = True):
        
        import pandas as pd
        
        # check https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.drop.html?highlight=drop
        
        # what_to_drop = 'columns' for removing the columns specified by their names (headers)
        # in cols_list (a list of strings).
        # what_to_drop = 'rows' for removing the rows specified by their indices in
        # row_index_list (a list of integers). Remember that the indexing starts from zero, i.e.,
        # the first row is row number zero.
        
        # cols_list = list of strings containing the names (headers) of the columns to be removed
        # For instance: cols_list = ['col1', 'col2', 'col3'] will 
        # remove columns 'col1', 'col2', and 'col3' from the dataframe.
        # If a single column will be dropped, you can declare it as a string (outside a list)
        # e.g. cols_list = 'col1'; or cols_list = ['col1']
        
        # row_index_list = a list of integers containing the indices of the rows that will be dropped.
        # e.g. row_index_list = [0, 1, 2] will drop the rows with indices 0 (1st row), 1 (2nd row), and
        # 2 (third row). Again, if a single row will be dropped, you can declare it as an integer (outside
        # a list).
        # e.g. row_index_list = 20 or row_index_list = [20] to drop the row with index 20 (21st row).
        
        # reset_index_after_drop = True. keep it True to restarting the indexing numeration after dropping.
        # Alternatively, set reset_index_after_drop = False to keep the original numeration (the removed indices
        # will be missing).
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        if (what_to_drop == 'columns'):
            
            if (cols_list is None):
                #check if a list was not input:
                print("Input a list of columns cols_list to be dropped.")
                return "error"
            
            else:
                #Drop the columns in cols_list:
                DATASET = DATASET.drop(columns = cols_list)
                print(f"The columns in {cols_list} headers list were successfully removed.\n")
        
        elif (what_to_drop == 'rows'):
            
            if (row_index_list is None):
                #check if a list was not input:
                print("Input a list of rows indices row_index_list to be dropped.")
                return "error"
            
            else:
                #Drop the rows in row_index_list:
                DATASET = DATASET.drop(row_index_list)
                print(f"The rows in {row_index_list} indices list were successfully removed.\n")
        
        else:
            print("Input a valid string as what_to_drop, rows or columns.")
            return "error"
        
        if (reset_index_after_drop == True):
            
            #restart the indexing
            DATASET = DATASET.reset_index(drop = True)
            print("The indices of the dataset were successfully restarted.\n")
        
        print("Check the 10 first rows from the returned dataset:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        return DATASET


    def remove_duplicate_rows (df, list_of_columns_to_analyze = None, which_row_to_keep = 'first', reset_index_after_drop = True):
        
        import pandas as pd
        # check https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.drop_duplicates.html
        
        # if list_of_columns_to_analyze = None, the whole dataset will be analyzed, i.e., rows
        # will be removed only if they have same values for all columns from the dataset.
        # Alternatively, pass a list of columns names (strings), if you want to remove rows with
        # same values for that combination of columns. Pass it as a list, even if there is a single column
        # being declared.
        # e.g. list_of_columns_to_analyze = ['column1'] will check only 'column1'. Entries with same value
        # on 'column1' will be considered duplicates and will be removed.
        # list_of_columns_to_analyze = ['col1', 'col2',  'col3'] will analyze the combination of 3 columns:
        # 'col1', 'col2', and 'col3'. Only rows with same value for these 3 columns will be considered
        # duplicates and will be removed.
        
        # which_row_to_keep = 'first' will keep the first detected row and remove all other duplicates. If
        # None or an invalid string is input, this method will be selected.
        # which_row_to_keep = 'last' will keep only the last detected duplicate row, and remove all the others.
        
        # reset_index_after_drop = True. keep it True to restarting the indexing numeration after dropping.
        # Alternatively, set reset_index_after_drop = False to keep the original numeration (the removed indices
        # will be missing).
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        if (which_row_to_keep == 'last'):
            
            #keep only the last duplicate.
            if (list_of_columns_to_analyze is None):
                # use the whole dataset
                DATASET = DATASET.drop_duplicates(keep = 'last')
                print(f"The rows with duplicate entries were successfully removed.")
                print("Only the last one of the duplicate entries was kept in the dataset.\n")
            
            else:
                #use the subset of columns
                if (list_of_columns_to_analyze is None):
                    #check if a list was not input:
                    print("Input a list of columns list_of_columns_to_analyze to be analyzed.")
                    return "error"
            
                else:
                    #Drop the columns in cols_list:
                    DATASET = DATASET.drop_duplicates(subset = list_of_columns_to_analyze, keep = 'last')
                    print(f"The rows with duplicate values for the columns in {list_of_columns_to_analyze} headers list were successfully removed.")
                    print("Only the last one of the duplicate entries was kept in the dataset.\n")
        
        else:
            
            #keep only the first duplicate.
            if (list_of_columns_to_analyze is None):
                # use the whole dataset
                DATASET = DATASET.drop_duplicates()
                print(f"The rows with duplicate entries were successfully removed.")
                print("Only the first one of the duplicate entries was kept in the dataset.\n")
            
            else:
                #use the subset of columns
                if (list_of_columns_to_analyze is None):
                    #check if a list was not input:
                    print("Input a list of columns list_of_columns_to_analyze to be analyzed.")
                    return "error"
            
                else:
                    #Drop the columns in cols_list:
                    DATASET = DATASET.drop_duplicates(subset = list_of_columns_to_analyze)
                    print(f"The rows with duplicate values for the columns in {list_of_columns_to_analyze} headers list were successfully removed.")
                    print("Only the first one of the duplicate entries was kept in the dataset.\n")
        
        if (reset_index_after_drop == True):
            
            #restart the indexing
            DATASET = DATASET.reset_index(drop = True)
            print("The indices of the dataset were successfully restarted.\n")
        
        print("Check the 10 first rows from the returned dataset:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        return DATASET


    def remove_completely_blank_rows_and_columns (df, list_of_columns_to_ignore = None):
        
        import numpy as np
        import pandas as pd
        
        # list_of_columns_to_ignore: if you do not want to check a specific column, pass its name
        # (header) as an element from this list. It should be declared as a list even if it contains
        # a single value.
        # e.g. list_of_columns_to_ignore = ['column1'] will not analyze missing values in column named
        # 'column1'; list_of_columns_to_ignore = ['col1', 'col2'] will ignore columns 'col1' and 'col2'
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        # Get dataframe length:
        df_length = len(DATASET)
        
        # Get list of columns from the dataframe:
        df_columns = DATASET.columns
        
        # Get initial totals of rows or columns:
        total_rows = len(DATASET)
        total_cols = len(df_columns)
        
        # Get a list containing only columns to check:
        cols_to_check = []
        
        # Check if there is a list of columns to ignore:
        if not (list_of_columns_to_ignore is None):
            
            # Append all elements from df_columns that are not in the list
            # to ignore:
            for column in df_columns:
                # loop through all elements named 'column' and check if it satisfies both conditions
                if (column not in list_of_columns_to_ignore):
                    cols_to_check.append(column)
            
            # create a ignored dataframe and a checked df:
            checked_df = DATASET[cols_to_check].copy(deep = True)
            # Update total columns:
            total_cols = len(checked_df.columns)
            
            ignored_df = DATASET[list_of_columns_to_ignore].copy(deep = True)
        
        else:
            # There is no column to ignore, so we must check all columns:
            checked_df = DATASET
            # Update the list of columns to check:
            cols_to_check = list(checked_df.columns)
        
        # To remove only rows or columns with only missing values, we set how = 'all' in
        # dropna method:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.dropna.html
        
        # Remove rows that contain only missing values:
        
        checked_df = checked_df.dropna(axis = 0, how = 'all')
        print(f"{total_rows - len(checked_df)} rows were completely blank and were removed.\n")
        
        # Remove columns that contain only missing values:
        checked_df = checked_df.dropna(axis = 1, how = 'all')
        print(f"{total_cols - len(checked_df.columns)} columns were completely blank and were removed.\n")
        
        # If len(cols_to_check) > 0, merge again the subsets:
        if (len(cols_to_check) > 0):

            if not (list_of_columns_to_ignore is None): # There is an ignored dataframe
            
                DATASET = pd.concat([ignored_df, checked_df], axis = 1, join = "inner")
            
            else: # Make the DATASET the checked_df itself:
                DATASET = checked_df

        # Now, reset the index:
        DATASET = DATASET.reset_index(drop = True)
        
        if (((total_rows - len(DATASET)) > 0) | ((total_cols - len(DATASET.columns)) > 0)):
            
            # There were modifications in the dataframe.
            print("Check the first 10 rows of the new returned dataframe:\n")
            
            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(DATASET.head(10))

            except: # regular mode
                print(DATASET.head(10))
        
        else:
            print("No blank columns or rows were found. Returning the original dataframe.\n")
        
        return DATASET


    def characterize_categorical_variables (df, timestamp_tag_column = None):
        
        import numpy as np
        import pandas as pd
        from pandas import json_normalize
        
        # df: dataframe that will be analyzed
        
        # timestamp_tag_colum: name (header) of the column containing the
        # timestamps. Keep timestamp_tag_column = None if the dataframe do not contain
        # timestamps.
        
                # Encoding syntax:
                # dataset.loc[dataset["CatVar"] == 'Value1', "EncodedColumn"] = 1
                # dataset.loc[boolean_filter, EncodedColumn"] = value,
                # boolean_filter = (dataset["CatVar"] == 'Value1') will be True when the 
                # equality is verified. The .loc method filters the dataframe, accesses the
                # column declared after the comma and inputs the value defined (e.g. value = 1)
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        # Get the list of columns:
        cols_list = list(DATASET.columns)
        
        # Start a list of categorical columns:
        categorical_list = []
        is_categorical = 0 # start the variable
        
        # Start a timestamp list that will be empty if there is no timestamp_tag_column
        timestamp_list = []
        if (timestamp_tag_column is not None):
            timestamp_list.append(timestamp_tag_column)
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        # Loop through all valid columns (cols_list)
        for column in cols_list:
            
            # Check if the column is neither in timestamp_list nor in
            # categorical_list yet:
            
            if ((column not in categorical_list) & (column not in timestamp_list)):
                # Notice that, since we already selected the 'timestamp_obj', we remove the original timestamps.
                column_data_type = DATASET[column].dtype
                
                if (column_data_type not in numeric_dtypes):
                    
                    # Append to categorical columns list:
                    categorical_list.append(column)
        
        # Subset the dataframe:
        if (len(categorical_list) >= 1):
            
            DATASET = DATASET[categorical_list]
            is_categorical = 1
        
        # Start a list to store the results:
        summary_list = []
        # It will be a list of dictionaries.
        
        # Loop through all variables on the list:
        for categorical_var in categorical_list:
            
            # Get unique vals and respective counts.

            # Start dictionary that will be appended as a new element from the list:
            # The main dictionary will be an element of the list
            unique_dict = {'categorical_variable': categorical_var}
            
            # Start a list of unique values:
            unique_vals = []

            # Now, check the unique values of the categorical variable:
            unique_vals_array = DATASET[categorical_var].unique()
            # unique_vals_array is a NumPy array containing the different values from the categorical variable.

            # Total rows:
            total_rows = len(DATASET)

            # Check the total of missing values
            # Set a boolean_filter for checking if the row contains a missing value
            boolean_filter = DATASET[categorical_var].isna()

            # Calculate the total of elements when applying the filter:
            total_na = len(DATASET[boolean_filter])

            # Create a dictionary for the missing values:
            na_dict = {
                        'value': np.nan, 
                        'counts_of_occurences': total_na,
                        'percent_of_occurences': ((total_na/total_rows)*100)
                        }
            
            
            # Nest this dictionary as an element from the list unique_vals.
            unique_vals.append(na_dict)
            # notice that the dictionary was nested into a list, which will be itself
            # nested as an element of the dictionary unique_dict
            
            # Now loop through each possible element on unique_vals_array
            for unique_val in unique_vals_array:

                # loop through each possible value of the array. The values are called 'unique_val'
                # Check if the value is not none:
                
                # Depending on the type of variable, the following error may be raised:
                # func 'isnan' not supported for the input types, and the inputs could not be safely coerced 
                # to any supported types according to the casting rule ''safe''
                # To avoid it, we can set the variable as a string using the str attribute and check if
                # the value is not neither 'nan' nor 'NaN'. That is because pandas will automatically convert
                # identified null values to np.nan
                
                # So, since The unique method creates the strings 'nan' or 'NaN' for the missing values,
                # if we read unique_val as string using the str attribute, we can filter out the
                # values 'nan' or 'NaN', which may be present together with the None and the float
                # np.nan:
                if ((str(unique_val) != 'nan') & (str(unique_val) != 'NaN') & (unique_val is not None)):
                    # If one of these conditions is true, the value is None, 'NaN' or 'nan'
                    # so this condition does not run.
                    # It runs if at least one value is not a missing value
                    # (only when the value is neither None nor np.nan)

                    # create a filter to select only the entries where the column == unique_val:
                    boolean_filter = (DATASET[categorical_var] == unique_val)
                    # Calculate the total of elements when applying the filter:
                    total_elements = len(DATASET[boolean_filter])

                    # Create a dictionary for these values:
                    # Use the same keys as before:
                    cat_var_dict = {
                        
                                    'value': unique_val, 
                                    'counts_of_occurences': total_elements,
                                    'percent_of_occurences': ((total_elements/total_rows)*100)
                        
                                    }
                    
                    # Nest this dictionary as an element from the list unique_vals.
                    unique_vals.append(cat_var_dict)
                    # notice that the dictionary was nested into a list, which will be itself
                    # nested as an element of the dictionary unique_dict
            
            # Nest the unique_vals list as an element of the dictionary unique_dict:
            # Set 'unique_values' as the key, and unique_vals as value
            unique_dict['unique_values'] = unique_vals
            # Notice that unique_vals is a list where each element is a dictionary with information
            # from a given unique value of the variable 'categorical_var' being analyzed.
            
            # Finally, append 'unique_dict' as an element of the list summary_list:
            summary_list.append(unique_dict)
            
        
        # We created a highly nested JSON structure with the following format:
        
        # summary_list = [
        #          {
        #            'categorical_variable': categorical_var1,
        #            'unique_values': [
        #                             {
        #                                'value': np.nan, 
        #                               'counts_of_occurences': total_na,
        #                               'percent_of_occurences': ((total_na/total_rows)*100)
        #                      },  {
        #
        #                           'value': unique_val_1, 
        #                           'counts_of_occurences': total_elements_1,
        #                           'percent_of_occurences': ((total_elements_1/total_rows)*100)
        #               
        #                     }, ... , {
        #                           'value': unique_val_N, 
        #                           'counts_of_occurences': total_elements_N,
        #                           'percent_of_occurences': ((total_elements_N/total_rows)*100)
        #               
        #                     }
        #                    ]
        #                 }, ... {
        #                        'categorical_variable': categorical_var_M,
        #                        'unique_values': [...]
        #                       }
        # ]
    
        if (is_categorical == 1):
            # Notice that, if !=1, the list is empty, so the previous loop is not executed.
            # Now, call the same methods used in function json_obj_to_dataframe to 
            # flat the list of dictionaries, if they are present:
        
            JSON = summary_list
            JSON_RECORD_PATH = 'unique_values'
            JSON_FIELD_SEPARATOR = "_"
            JSON_METADATA_PREFIX_LIST = ['categorical_variable']

            cat_vars_summary = json_normalize(JSON, record_path = JSON_RECORD_PATH, sep = JSON_FIELD_SEPARATOR, meta = JSON_METADATA_PREFIX_LIST)
            # JSON_METADATA_PREFIX_LIST: list of strings (in quotes). Manipulates the parameter 
            # 'meta' from json_normalize method. Fields to use as metadata for each record in resulting 
            # table. Declare here the non-nested fields, i.e., the fields in the principal JSON. They
            # will be repeated in the rows of the dataframe to give the metadata (context) of the rows.

            print("\n") # line break
            print("Finished analyzing the categorical variables. Check the summary dataframe:\n")
            
            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(cat_vars_summary)

            except: # regular mode
                print(cat_vars_summary)
            
            return cat_vars_summary
        
        else:
            print("The dataframe has no categorical variables to analyze.\n")
            return "error"


    def GROUP_VARIABLES_BY_TIMESTAMP (df, timestamp_tag_column, subset_of_columns_to_aggregate = None, grouping_frequency_unit = 'day', number_of_periods_to_group = 1, aggregate_function = 'mean', start_time = None, offset_time = None, add_suffix_to_aggregated_col = True, suffix = None):
        
        import numpy as np
        import pandas as pd
        from scipy import stats
        # numpy has no function mode, but scipy's stats module has.
        # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.mode.html?msclkid=ccd9aaf2cb1b11ecb57c6f4b3e03a341
        
        print("WARNING: The categorical variables will be grouped in terms of mode, i.e., as the most common value observed during the aggregated time period. This is the maximum of the statistical distribution of that variable.\n")
        
        # df - dataframe/table containing the data to be grouped
        
        # timestamp_tag_colum: name (header) of the column containing the
        # timestamps for grouping the data.
        
        # subset_of_columns_to_aggregate: list of strings (inside quotes) containing the names 
        # of the columns that will be aggregated. Use this argument if you want to aggregate only a subset,
        # not the whole dataframe. Declare as a list even if there is a single column to group by.
        # e.g. subset_of_columns_to_aggregate = ["response_feature"] will return the column 
        # 'response_feature' grouped. subset_of_columns_to_aggregate = ["col1", 'col2'] will return columns
        # 'col1' and 'col2' grouped.
        # If you want to aggregate the whole subset, keep subset_of_columns_to_aggregate = None.
        
        # grouping_frequency_unit: the frequency of aggregation. The possible values are:
        
        grp_frq_unit_dict = {'year': "Y", 'month': "M", 'week': "W", 
                                'day': "D", 'hour': "H", 'minute': "min", 'second': 'S'}
        
        #Simply provide the key: 'year', 'month', 'week',..., 'second', and this dictionary
        #will convert to the Pandas coding.
        #The default is 'day', so this will be inferred frequency if no value is provided.
        
        #To access the value of a dictionary d = {key1: item1, ...}:
        #d['key1'] = item1. - simply declare the key as a string (under quotes) inside brackets
        #just as if you were accessing a column from the dataframe.
        #Since grouping_frequency_unit is variable storing a string, it should not come under
        #quotes:
        
        #Convert the input to Pandas encoding:
        frq_unit = grp_frq_unit_dict[grouping_frequency_unit]
        
        #https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
        #To group by business day, check the example:
        #https://stackoverflow.com/questions/13019719/get-business-days-between-start-and-end-date-using-pandas
        
        #number_of_periods_to_group: the bin size. The default is 1, so we will group by '1day'
        #if number_of_periods_to_group = 2 we would be grouping by every 2 days.
        #If the unit was minute and number_of_periods_to_group = 30, we would be grouping into
        #30-min bins.
        
        if (number_of_periods_to_group <= 0):
            
            print("Invalid number of periods to group. Changing to 1 period.\n")
            number_of_periods_to_group = 1
        
        if (number_of_periods_to_group == 1):
            
            #Do not put the number 1 prior to the frequency unit
            FREQ =  frq_unit
        
        else:
            #perform the string concatenation. Convert the number into a string:
            number_of_periods_to_group = str(number_of_periods_to_group)
            #Concatenate the strings:
            FREQ = number_of_periods_to_group + frq_unit
            #Expected output be like '2D' for a 2-days grouping
        
        # aggregate_function: Pandas aggregation method: 'mean', 'median', 'std', 'sum', 'min'
        # 'max', 'count', etc. The default is 'mean'. Then, if no aggregate is provided, 
        # the mean will be calculated.
        
        agg_dict = {
            
            'mean': 'mean',
            'sum': 'sum',
            'median': 'median',
            'std': 'std',
            'count': 'count',
            'min': 'min',
            'max': 'max',
            'mode': stats.mode,
            'geometric_mean': stats.gmean,
            'harmonic_mean': stats.hmean,
            'kurtosis': stats.kurtosis,
            'skew': stats.skew,
            'geometric_std': stats.gstd,
            'interquartile_range': stats.iqr,
            'mean_standard_error': stats.sem,
            
        }
        # scipy.stats Summary statistics:
        # https://docs.scipy.org/doc/scipy/reference/stats.html
        
        # Convert the input into the correct aggregation function. Access the value on key
        # aggregate_function in dictionary agg_dict:
        
        if (aggregate_function in agg_dict.keys()):
            
            aggregate_function = agg_dict[aggregate_function]
        
        else:
            print(f"Select a valid aggregate function: {agg_dict.keys()}")
            return "error"
        
        # Now, aggregate_function actually stores the value that must be passed to the agg method.
        
        
        #You can pass a list of multiple aggregations, like: 
        #aggregate_function = [mean, max, sum]
        #You can also pass custom functions, like: pct30 (30-percentile), or np.mean
        #aggregate_function = pct30
        #aggregate_function = np.mean (numpy.mean)
        
        #ADJUST OF GROUPING BASED ON A FIXED TIMESTAMP
        #This parameters are set to None as default.
        #You can specify the origin (start_time) or the offset (offset_time), which are
        #equivalent. The parameter should be declared as a timestamp.
        #For instance: start_time = '2000-10-01 23:30:00'
        
        #WARNING: DECLARE ONLY ONE OF THESE PARAMETERS. DO NOT DECLARE AN OFFSET IF AN 
        #ORIGIN WAS SPECIFIED, AND VICE-VERSA.
        
        #Create a Pandas timestamp object from the timestamp_tag_column. It guarantees that
        #the timestamp manipulation methods can be correctly applied.
        #Let's create using nanoseconds resolution, so that the timestamps present the
        #maximum possible resolution:
        
        # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.
        # This will prevent any compatibility problems.
        
        #The pd.Timestamp function can handle a single timestamp per call. Then, we must
        # loop trough the series, and apply the function to each element.
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        df_copy = df.copy(deep = True)
        
        # try parsing as np.datetime64 (more efficient, without loops):
        try:
            df_copy['timestamp_obj'] = df_copy[timestamp_tag_column].astype(np.datetime64)
        
        except:
            # Obtain pd.Timestamp objects
            # 1. Start a list to store the Pandas timestamps:
            timestamp_list = []

            # 2. Loop through each element of the timestamp column, and apply the function
            # to guarantee that all elements are Pandas timestamps

            for timestamp in df_copy[timestamp_tag_column]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column]
                timestamp_list.append(pd.Timestamp(timestamp, unit = 'ns'))

            # 3. Create a column in the dataframe that will be used as key for the Grouper class
            # The grouper requires a column in the dataframe - it cannot use a list for that.
            # Simply copy the list as the new column:
            df_copy['timestamp_obj'] = timestamp_list
        
        # Now we have a list correspondent to timestamp_tag_column, but only with
        # Pandas timestamp objects
        
        # 4. Sort the dataframe in ascending order of timestamps:
        df_copy = df_copy.sort_values(by = 'timestamp_obj', ascending = True)
        
        # Reset indices before aggregation:
        df_copy = df_copy.reset_index(drop = True)
        
        # In this function, we do not convert the Timestamp to a datetime64 object.
        # That is because the Grouper class specifically requires a Pandas Timestamp
        # object to group the dataframes.
        
        # Get the list of columns:
        cols_list = list(df_copy.columns)
        
        if (subset_of_columns_to_aggregate is not None):
            
            # cols_list will be the subset list:
            cols_list = subset_of_columns_to_aggregate
        
        # Start a list of numerical columns, and a list of categorical columns, containing only the
        # column for aggregation as the first element:
        numeric_list = ['timestamp_obj']
        categorical_list = ['timestamp_obj']
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        # Loop through all valid columns (cols_list)
        for column in cols_list:
            
            # Check if the column is neither in numeric_list nor in
            # categorical_list yet:
            if ((column not in numeric_list) & (column not in categorical_list) & (column != timestamp_tag_column)):
                # Notice that, since we already selected the 'timestamp_obj', we remove the original timestamps.
                column_data_type = df_copy[column].dtype
                
                if (column_data_type not in numeric_dtypes):
                    
                    # Append to categorical columns list:
                    categorical_list.append(column)
                
                else:
                    # Append to numerical columns list:
                    numeric_list.append(column)
        
        # Create variables to map if both are present.
        is_categorical = 0
        is_numeric = 0
        
        # Create two subsets:
        if (len(categorical_list) > 1):
            
            # Has at least one column plus the variable_to_group_by:
            df_categorical = df_copy.copy(deep = True)
            df_categorical = df_categorical[categorical_list]
            is_categorical = 1
        
        if (len(numeric_list) > 1):
            
            df_numeric = df_copy.copy(deep = True)
            df_numeric = df_numeric[numeric_list]
            is_numeric = 1
        
        # Notice that the variables is_numeric and is_categorical have value 1 only when the subsets
        # are present.
        is_cat_num = is_categorical + is_numeric
        # is_cat_num = 0 if no valid dataset was input.
        # is_cat_num = 2 if both subsets are present.
            
        
        # Let's try to group the df_numeric dataframe.
        if (is_numeric == 1):
            
            if (start_time is not None):

                df_numeric = df_numeric.groupby(pd.Grouper(key = 'timestamp_obj' , freq = FREQ, origin = start_time)).agg(aggregate_function)

            elif (offset_time is not None):

                df_numeric = df_numeric.groupby(pd.Grouper(key = 'timestamp_obj' , freq = FREQ, offset = offset_time)).agg(aggregate_function)

            else:

                #Standard situation, when both start_time and offset_time are None
                df_numeric = df_numeric.groupby(pd.Grouper(key = 'timestamp_obj' , freq = FREQ)).agg(aggregate_function)
                
            print (f"Numerical variables of the dataframe grouped in terms of {aggregate_function} by every {number_of_periods_to_group} {frq_unit}.\n")
            
            # Here, it is not possible to set as_index = False not to set the grouper column as index of the new dataframe.
            # So, the timestamp becomes the index. Let's create a column 'timestamp_grouped' to store this index:
            df_numeric['timestamp_grouped'] = df_numeric.index
            # Reset the index:
            df_numeric = df_numeric.reset_index(drop = True)
            
            # Notice that the timestamp became the last element. So, get a list grouped_num_cols
            # That is a slice from the list of columns from df_numeric containing all elements except the last one.
            grouped_num_cols = list(df_numeric.columns)[:-1]
            # The slice [i:] gets all elements from index i, whereas [:j] get all elements until index j
            # (but not including j). [:-1] gets all until the last one, [:-2] gets all until the last 2,...
            
            # Now concatenate with a list containing only the timestamp to make it the first element:
            grouped_num_cols = ['timestamp_grouped'] + grouped_num_cols
            # Select the columns in the new order by passing the list as argument:
            df_numeric = df_numeric[grouped_num_cols]
            
            if (add_suffix_to_aggregated_col == True):
            
                # Let's add a suffix. Check if suffix is None. If it is,
                # set "_" + aggregate_function as suffix:
                if (suffix is None):
                    numeric_suffix = "_" + aggregate_function
                
                else:
                    numeric_suffix = suffix
                
                # New columns names:
                new_num_names = [(str(name) + numeric_suffix) for name in numeric_list]
                # The str attribute guarantees that the name was read as string
                # Pick only the values from the second and concatenate the correct name 
                # for the aggregation column (eliminate the first element from the list):
                new_num_names = ['timestamp_grouped'] + new_num_names[1:]
                # Set new_num_names as the new columns names:
                df_numeric.columns = new_num_names
            
        
        #### LET'S AGGREGATE THE CATEGORICAL VARIABLES
        
        ## Check if there is a list of categorical features. If there is, run the next block of code:
        
        if (is_categorical == 1):
            # There are categorical columns to aggregate too - the list is not empty
            # Consider: a = np.array(['a', 'a', 'b'])
            # The stats.mode function stats.mode(a) returns an array as: 
            # ModeResult(mode=array(['a'], dtype='<U1'), count=array([2]))
            # If we select the first element from this array, stats.mode(a)[0], the function will 
            # return an array as array(['a'], dtype='<U1'). 
            # We want the first element from this array stats.mode(a)[0][0], 
            # which will return a string like 'a'
            
            # We can pass stats.mode as the aggregate function in agg: agg(stats.mode)
            
            # The original timestamps, already converted to Pandas timestamp objects, are stored in:
            # timestamp_list. So, we can again use this list to aggregation. It was saved as the
            # column 'timestamp_obj' from the dataframe df_copy
            
            # This will generate series where each element will be an array like:
            # series = ([mode_for_that_row], [X]), where X is the counting for that row. For example, if we
            # aggregate by week, and there is a 'categorical_value' by day, X will be 7.
            
            # to access a row from the series, for instance, row 0: series[0]. 
            # This element will be an array like:
            # ModeResult(mode=array([mode_for_that_row], dtype='<U19'), count=array([X])).
            # To access the first element of this array, we put another index: series[0][0].
            # This element will be like:
            # array([mode_for_that_row], dtype='<U19')
            # The mode is the first element from this array. To access it, we add another index:
            # series[0][0][0]. The result will be: mode_for_that_row
            
            ## Aggregate the df_categorical dataframe in terms of mode:
            
            if (start_time is not None):

                df_categorical = df_categorical.groupby(pd.Grouper(key = 'timestamp_obj' , freq = FREQ, origin = start_time), as_index = False, sort = True).agg(stats.mode)

            elif (offset_time is not None):

                df_categorical = df_categorical.groupby(pd.Grouper(key = 'timestamp_obj' , freq = FREQ, offset = offset_time), as_index = False, sort = True).agg(stats.mode)

            else:

                #Standard situation, when both start_time and offset_time are None
                df_categorical = df_categorical.groupby(pd.Grouper(key = 'timestamp_obj' , freq = FREQ), as_index = False, sort = True).agg(stats.mode)
            
            print (f"Categorical variables of the dataframe grouped in terms of \'mode\' by every {number_of_periods_to_group} {frq_unit}.\n")
            print(f"The mode is the most common value observed (maximum of the statistical distribution) for the categorical variable when we group data in terms of {number_of_periods_to_group} {frq_unit}.\n")
            
            # delete the first value from categorical_list, which is 'timestamp_obj':
            # This step is required for not to calculate the mode of the timestamps in the next loop.
            del categorical_list[0]
            
            # Loop through each categorical variable:
            for cat_var in categorical_list:
                
                # save as a series:
                cat_var_series = df_categorical[cat_var].copy()
                # Start a list to store only the modes:
                list_of_modes = []

                # Now, loop through each row of cat_var_series. Take the element [0][0]
                # and append it to the list_of_modes:
                for i in range(0, len(cat_var_series)):
                    # Goes from i = 0 to i = len(cat_var_series) - 1, index of the last element
                    # Append the element [0][0] from row [i]
                    try:
                        list_of_modes.append(cat_var_series[i][0][0])

                    except IndexError:
                        # This error is generated when trying to access an array storing no values.
                        # (i.e., with missing values). Since there is no dimension, it is not possible
                        # to access the [0][0] position. In this case, simply append the np.nan (missing value):
                        list_of_modes.append(np.nan)

                # Now we finished the nested for loop, list_of_modes contain only the modes

                # Make the column cat_var the list_of_modes itself:
                df_categorical[cat_var] = list_of_modes

            # Again, it is not possible to set as_index = False so, the timestamp becomes the index. 
            # Let's create a column 'timestamp_grouped' to store this index:
            df_categorical['timestamp_grouped'] = df_categorical.index
            # Reset index:
            df_categorical = df_categorical.reset_index(drop = True)
            
            grouped_cat_cols = list(df_categorical.columns)[:-1]
            grouped_cat_cols = ['timestamp_grouped'] + grouped_cat_cols
            
            # Select the columns in the new order by passing the list as argument:
            df_categorical = df_categorical[grouped_cat_cols]
            
            if (add_suffix_to_aggregated_col == True):
            
                # Let's add a suffix. Check if suffix is None. If it is,
                # set "_" + aggregate_function as suffix:
                if (suffix is None):
                    categorical_suffix = "_mode"
                
                else:
                    categorical_suffix = suffix
                
                # New columns names:
                new_cat_names = [(str(name) + categorical_suffix) for name in categorical_list]
                # Notice that we have already deleted 'timestamp_obj' from categorical_list,
                # avoiding the calculation of the timestamp modes.
                # So, now concatenate the correct name for the aggregation column:
                new_cat_names = ['timestamp_grouped'] + new_cat_names
                # Set new_num_names as the new columns names:
                df_categorical.columns = new_cat_names   
            
            
        if (is_cat_num == 2):
            # Both subsets are present. Remove the column from df_categorical:
            df_categorical.drop(columns = 'timestamp_grouped', inplace = True)
            
            # Concatenate the dataframes in the columns axis (append columns):
            DATASET = pd.concat([df_numeric, df_categorical], axis = 1, join = "inner")
        
        elif (is_categorical == 1):
            # There is only the categorical subset:
            DATASET = df_categorical
        
        elif (is_numeric == 1):
            # There is only the numeric subset:
            DATASET = df_numeric
            
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Dataframe successfully grouped. Check its 10 first rows (without the categorical/object variables):\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
    
        #Now return the grouped dataframe with the timestamp as the first column:
        
        return DATASET


    def GROUP_DATAFRAME_BY_VARIABLE (df, variable_to_group_by, return_summary_dataframe = False, subset_of_columns_to_aggregate = None, aggregate_function = 'mean', add_suffix_to_aggregated_col = True, suffix = None):

        import numpy as np
        import pandas as pd
        from scipy import stats
        
        print("WARNING: Do not use this function to group the dataframe in terms of a timestamp. For this purpose, use function GROUP_VARIABLES_BY_TIMESTAMP.\n")
        
        # df: dataframe being analyzed
        
        # variable_to_group_by: string (inside quotes) containing the name 
        # of the column in terms of which the dataframe will be grouped by. e.g. 
        # variable_to_group_by = "column1" will group the dataframe in terms of 'column1'.
        # WARNING: do not use this function to group a dataframe in terms of a timestamp. To group by
        # a timestamp, use function GROUP_VARIABLES_BY_TIMESTAMP instead.
        
        # return_summary_dataframe = False. Set return_summary_dataframe = True if you want the function
        # to return a dataframe containing summary statistics (obtained with the describe method).
        
        # subset_of_columns_to_aggregate: list of strings (inside quotes) containing the names 
        # of the columns that will be aggregated. Use this argument if you want to aggregate only a subset,
        # not the whole dataframe. Declare as a list even if there is a single column to group by.
        # e.g. subset_of_columns_to_aggregate = ["response_feature"] will return the column 
        # 'response_feature' grouped. subset_of_columns_to_aggregate = ["col1", 'col2'] will return columns
        # 'col1' and 'col2' grouped.
        # If you want to aggregate the whole subset, keep subset_of_columns_to_aggregate = None.
        
        # aggregate_function = 'mean': String defining the aggregation 
        # method that will be applied. Possible values:
        # 'median', 'mean', 'mode', 'sum', 'min', 'max', 'variance', 'count',
        # 'standard_deviation', 'cum_sum', 'cum_prod', 'cum_max', 'cum_min',
        # '10_percent_quantile', '20_percent_quantile',
        # '25_percent_quantile', '30_percent_quantile', '40_percent_quantile',
        # '50_percent_quantile', '60_percent_quantile', '70_percent_quantile',
        # '75_percent_quantile', '80_percent_quantile', '90_percent_quantile',
        # '95_percent_quantile', 'kurtosis', 'skew', 'interquartile_range',
        # 'mean_standard_error', 'entropy'
        # To use another aggregate function, you can use the .agg method, passing 
        # the aggregate as argument, such as in:
        # .agg(scipy.stats.mode), 
        # where the argument is a Scipy aggregate function.
        # If None or an invalid function is input, 'mean' will be used.
        
        # add_suffix_to_aggregated_col = True will add a suffix to the
        # aggregated columns. e.g. 'responseVar_mean'. If add_suffix_to_aggregated_col 
        # = False, the aggregated column will have the original column name.
        
        # suffix = None. Keep it None if no suffix should be added, or if
        # the name of the aggregate function should be used as suffix, after
        # "_". Alternatively, set it as a string. As recommendation, put the
        # "_" sign in the beginning of this string to separate the suffix from
        # the original column name. e.g. if the response variable is 'Y' and
        # suffix = '_agg', the new aggregated column will be named as 'Y_agg'
            

        # Create a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        # Get the list of columns:
        cols_list = list(DATASET.columns)
        
        if (subset_of_columns_to_aggregate is not None):
            
            # cols_list will be the subset list:
            cols_list = subset_of_columns_to_aggregate
        
        # Start a list of numerical columns, and a list of categorical columns, containing only the
        # column for aggregation as the first element:
        numeric_list = [variable_to_group_by]
        categorical_list = [variable_to_group_by]
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        # Loop through all valid columns (cols_list)
        for column in cols_list:
            
            # Check if the column is neither in numeric_list nor in
            # categorical_list yet:
            if ((column not in numeric_list) & (column not in categorical_list)):
                
                column_data_type = DATASET[column].dtype
                
                if (column_data_type not in numeric_dtypes):
                    
                    # Append to categorical columns list:
                    categorical_list.append(column)
                
                else:
                    # Append to numerical columns list:
                    numeric_list.append(column)
        
        # Create variables to map if both are present.
        is_categorical = 0
        is_numeric = 0
        
        # Before grouping, let's remove the missing values, avoiding the raising of TypeError.
        # Pandas deprecated the automatic dropna with aggregation:
        DATASET = DATASET.dropna(axis = 0)
        # It is important to drop before seggregating the dataframes, so that the rows correspondence
        # will not be lost:
        DATASET = DATASET.reset_index(drop = True)
        
        # Convert variable_to_group_by to Pandas 'category' type. If the variable is represented by
        # a number, the dataframe will be grouped in terms of an aggregation of the variable, instead
        # of as a category. It will prevents this to happen:
        DATASET[variable_to_group_by] = DATASET[variable_to_group_by].astype("category")
        
        # Create two subsets:
        if (len(categorical_list) > 1):
            
            # Has at least one column plus the variable_to_group_by:
            df_categorical = DATASET.copy(deep = True)
            df_categorical = df_categorical[categorical_list]
            is_categorical = 1
        
        if (len(numeric_list) > 1):
            
            df_numeric = DATASET.copy(deep = True)
            df_numeric = df_numeric[numeric_list]
            is_numeric = 1
            
        if (return_summary_dataframe):
            summary_agg_df = DATASET.copy(deep = True)
            summary_agg_df = summary_agg_df.groupby(by = variable_to_group_by, as_index = False, sort = True).describe()
                
        # Notice that the variables is_numeric and is_categorical have value 1 only when the subsets
        # are present.
        is_cat_num = is_categorical + is_numeric
        # is_cat_num = 0 if no valid dataset was input.
        # is_cat_num = 2 if both subsets are present.
        
        # Before calling the method, we must guarantee that the variables may be
        # used for that aggregate. Some aggregations are permitted only for numeric variables, so calling
        # the method before selecting the variables may raise warnings or errors.
        list_of_aggregates = ['median', 'mean', 'mode', 'sum', 'min', 'max', 'variance',
                            'standard_deviation', 'count', 'cum_sum', 'cum_prod', 'cum_max', 'cum_min',
                            '10_percent_quantile', '20_percent_quantile', '25_percent_quantile', 
                            '30_percent_quantile', '40_percent_quantile', '50_percent_quantile', 
                            '60_percent_quantile', '70_percent_quantile', '75_percent_quantile', 
                            '80_percent_quantile', '90_percent_quantile', '95_percent_quantile',  
                            'kurtosis', 'skew', 'interquartile_range', 'mean_standard_error', 'entropy']
        
        list_of_numeric_aggregates = ['median', 'mean', 'sum', 'min', 'max', 'variance',
                            'standard_deviation', 'cum_sum', 'cum_prod', 'cum_max', 'cum_min',
                            '10_percent_quantile', '20_percent_quantile', '25_percent_quantile', 
                            '30_percent_quantile', '40_percent_quantile', '50_percent_quantile', 
                            '60_percent_quantile', '70_percent_quantile', '75_percent_quantile', 
                            '80_percent_quantile', '90_percent_quantile', '95_percent_quantile',  
                            'kurtosis', 'skew', 'interquartile_range', 'mean_standard_error']
        
        # Check if an invalid or no aggregation function was selected:
        if ((aggregate_function not in (list_of_aggregates)) | (aggregate_function is None)):
            
            aggregate_function = 'mean'
            print("Invalid or no aggregation function input, so using the default \'mean\'.\n")
        
        # Check if a numeric aggregate was selected:
        if (aggregate_function in list_of_numeric_aggregates):
            
            print("Numeric aggregate selected. Categorical variables will be aggregated in terms of mode, the most common value.\n")
            
            numeric_aggregate = aggregate_function
            categorical_aggregate = 'mode'
        
        else:
            
            print("Categorical aggregate selected. Numeric variables will be aggregated in terms of mean.\n")
            
            categorical_aggregate = aggregate_function
            numeric_aggregate = 'mean'
        
        if (is_numeric == 1):
            # Let's aggregate the numeric subset
        
            if (numeric_aggregate == 'median'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).agg('median')

            elif (numeric_aggregate == 'mean'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).mean()
            
            elif (numeric_aggregate == 'sum'):
            
                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).sum()
        
            elif (numeric_aggregate == 'min'):
            
                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).min()

            elif (numeric_aggregate == 'max'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).max()

            elif (numeric_aggregate == 'variance'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).var()

            elif (numeric_aggregate == 'standard_deviation'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).std()

            elif (numeric_aggregate == 'cum_sum'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).cumsum()

            elif (numeric_aggregate == 'cum_prod'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).cumprod()

            elif (numeric_aggregate == 'cum_max'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).cummax()

            elif (numeric_aggregate == 'cum_min'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).cummin()

            elif (numeric_aggregate == '10_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.10)

            elif (numeric_aggregate == '20_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.20)

            elif (numeric_aggregate == '25_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.25)

            elif (numeric_aggregate == '30_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.30)

            elif (numeric_aggregate == '40_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.40)

            elif (numeric_aggregate == '50_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.50)

            elif (numeric_aggregate == '60_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.60)

            elif (numeric_aggregate == '70_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.30)

            elif (numeric_aggregate == '75_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.75)

            elif (numeric_aggregate == '80_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.80)

            elif (numeric_aggregate == '90_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.90)

            elif (numeric_aggregate == '95_percent_quantile'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).quantile(0.95)

            elif (numeric_aggregate == 'kurtosis'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).agg(stats.kurtosis)

            elif (numeric_aggregate == 'skew'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).agg(stats.skew)

            elif (numeric_aggregate == 'interquartile_range'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).agg(stats.iqr)

            elif (numeric_aggregate == 'mean_standard_error'):

                df_numeric = df_numeric.groupby(by = variable_to_group_by, as_index = False, sort = True).agg(stats.sem)
            
            
            if (add_suffix_to_aggregated_col == True):
            
                # Let's add a suffix. Check if suffix is None. If it is,
                # set "_" + aggregate_function as suffix:
                if (suffix is None):
                    numeric_suffix = "_" + numeric_aggregate
                
                else:
                    numeric_suffix = suffix
                
                # New columns names:
                new_num_names = [(str(name) + numeric_suffix) for name in numeric_list]
                # The str attribute guarantees that the name was read as string
                # Pick only the values from the second and concatenate the correct name 
                # for the aggregation column (eliminate the first element from the list):
                new_num_names = [variable_to_group_by] + new_num_names[1:]
                # Set new_num_names as the new columns names:
                df_numeric.columns = new_num_names
        
        if (is_categorical == 1):
            # Let's aggregate the categorical subset

            if (categorical_aggregate == 'mode'):

                df_categorical = df_categorical.groupby(by = variable_to_group_by, as_index = False, sort = True).agg(stats.mode)
                
                # Loop through each categorical variable:
                for cat_var in categorical_list:

                    # save as a series:
                    cat_var_series = df_categorical[cat_var].copy()
                    # Start a list to store only the modes:
                    list_of_modes = []

                    # Now, loop through each row of cat_var_series. Take the element [0][0]
                    # and append it to the list_of_modes:
                    for i in range(0, len(cat_var_series)):
                        # Goes from i = 0 to i = len(cat_var_series) - 1, index of the last element
                        # Append the element [0][0] from row [i]
                        try:
                            list_of_modes.append(cat_var_series[i][0][0])

                        except IndexError:
                            # This error is generated when trying to access an array storing no values.
                            # (i.e., with missing values). Since there is no dimension, it is not possible
                            # to access the [0][0] position. In this case, simply append the np.nan (missing value):
                            list_of_modes.append(np.nan)

                    # Now we finished the nested for loop, list_of_modes contain only the modes

                    # Make the column cat_var the list_of_modes itself:
                    df_categorical[cat_var] = list_of_modes

                
            elif (categorical_aggregate == 'count'):

                df_categorical = df_categorical.groupby(by = variable_to_group_by, as_index = False, sort = True).count()

            elif (categorical_aggregate == 'entropy'):

                df_categorical = df_categorical.groupby(by = variable_to_group_by, as_index = False, sort = True).agg(stats.entropy)
                
            
            if (add_suffix_to_aggregated_col == True):
            
                # Let's add a suffix. Check if suffix is None. If it is,
                # set "_" + aggregate_function as suffix:
                if (suffix is None):
                    categorical_suffix = "_" + categorical_aggregate
                
                else:
                    categorical_suffix = suffix
                
                # New columns names:
                new_cat_names = [(str(name) + categorical_suffix) for name in categorical_list]
                # The str attribute guarantees that the name was read as string
                # Pick only the values from the second and concatenate the correct name 
                # for the aggregation column (eliminate the first element from the list):
                new_cat_names = [variable_to_group_by] + new_cat_names[1:]
                # Set new_num_names as the new columns names:
                df_categorical.columns = new_cat_names
        
        if (is_cat_num == 2):
            # Both subsets are present. Remove the column from df_categorical:
            df_categorical.drop(columns = variable_to_group_by, inplace = True)
            
            # Concatenate the dataframes in the columns axis (append columns):
            DATASET = pd.concat([df_numeric, df_categorical], axis = 1, join = "inner")
        
        elif (is_categorical == 1):
            # There is only the categorical subset:
            DATASET = df_categorical
        
        elif (is_numeric == 1):
            # There is only the numeric subset:
            DATASET = df_numeric
        
        else:
            print("No valid dataset provided, so returning the input dataset itself.\n")
        
        # Now, reset index positions:
        DATASET = DATASET.reset_index(drop = True)
        
        print("Dataframe successfully grouped. Check its 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        if (return_summary_dataframe == True):
            
            print("\n")
            print("Check the summary statistics dataframe, that is also being returned:\n")
            
            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(summary_agg_df)

            except: # regular mode
                print(summary_agg_df)
            
            return DATASET, summary_agg_df
        
        else:
            # return only the aggregated dataframe:
            return DATASET

        
    def EXTRACT_TIMESTAMP_INFO (df, timestamp_tag_column, list_of_info_to_extract, list_of_new_column_names = None):
        
        import numpy as np
        import pandas as pd
        
        # df: dataframe containing the timestamp.
        
        # timestamp_tag_column: declare as a string under quotes. This is the column from 
        # which we will extract the timestamp.
        
        # list_of_info_to_extract: list of information to extract from the timestamp. Each information
        # will be extracted as a separate column. The allowed values are:
        # 'year', 'month', 'week', 'day', 'hour', 'minute', or 'second'. Declare as a list even if only
        # one information is going to be extracted. For instance:
        # list_of_info_to_extract = ['second'] extracts only the second.
        # list_of_info_to_extract = ['year', 'month', 'week', 'day'] extracts year, month, week and day. 
        
        # list_of_new_column_names: list of names (strings) of the new created columns. 
        # If no value is provided, it will be equals to extracted_info. For instance: if
        # list_of_info_to_extract = ['year', 'month', 'week', 'day'] and list_of_new_column_names = None,
        # the new columns will be named as 'year', 'month', 'week', and 'day'.
        # WARNING: This list must contain the same number of elements of list_of_info_to_extract and both
        # must be in the same order. Considering the same example of list, if list_of_new_column_names =
        # ['col1', 'col2', 'col3', 'col4'], 'col1' will be referrent to 'year', 'col2' to 'month', 'col3'
        # to 'week', and 'col4' to 'day'
        
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        # Check if the list of column names is None. If it is, make it equals to the list of extracted
        # information:
        if (list_of_new_column_names is None):
            
            list_of_new_column_names = list_of_info_to_extract
        
        # try parsing as np.datetime64 (more efficient, without loops):
        try:
            DATASET[timestamp_tag_column] = DATASET[timestamp_tag_column].astype(np.datetime64)
            
            timestamp_list = list(DATASET[timestamp_tag_column])
            
        except:
            # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.
            # This will prevent any compatibility problems.

            # The pd.Timestamp function can handle a single timestamp per call. Then, we must
            # loop trough the series, and apply the function to each element.

            # 1. Start a list to store the Pandas timestamps:
            timestamp_list = []

            # 2. Loop through each element of the timestamp column, and apply the function
            # to guarantee that all elements are Pandas timestamps

            for timestamp in DATASET[timestamp_tag_column]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column]
                timestamp_list.append(pd.Timestamp(timestamp, unit = 'ns'))

            # 3. Save the list as the column timestamp_tag_column itself:
            DATASET[timestamp_tag_column] = timestamp_list
        
        # 4. Sort the dataframe in ascending order of timestamps:
        DATASET = DATASET.sort_values(by = timestamp_tag_column, ascending = True)
        # Reset indices:
        DATASET = DATASET.reset_index(drop = True)

        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timestamp.html

        #Use the extracted_info as key to access the correct command in the dictionary.
        #To access an item from a dictionary d = {'key1': item1, ...}, declare d['key1'],
        #as if you would do to access a column from a dataframe.
        
        #By doing so, you will select the extraction command from the dictionary:
        # Loop through each element of the dataset, access the timestamp, 
        # extract the information and store it in the correspondent position of the 
        # new_column. Again. The methods can only be applied to a single Timestamp object,
        # not to the series. That is why we must loop through each of them:

        # Now, loop through each one of the items from the list 'list_of_info_to_extract'.
        # For each element, we will extract the information indicated by that item.

        for extracted_info in list_of_info_to_extract:

            # name that the new column should have:
            new_column_name = extracted_info
            # The element will be referred as 'new_column_name'

            #start a list to store the values of the new column
            new_column_vals = []
            
            try: # try accessing the dt attribute:
                
                if (extracted_info == 'year'):

                    DATASET[new_column_name] = DATASET[timestamp_tag_column].dt.year

                elif (extracted_info == "month"):

                    DATASET[new_column_name] = DATASET[timestamp_tag_column].dt.month

                elif (extracted_info == "week"):

                    DATASET[new_column_name] = DATASET[timestamp_tag_column].dt.isocalendar().week

                elif (extracted_info == "day"):

                    DATASET[new_column_name] = DATASET[timestamp_tag_column].dt.day

                elif (extracted_info == "hour"):

                    DATASET[new_column_name] = DATASET[timestamp_tag_column].dt.hour

                elif (extracted_info == "minute"):

                    DATASET[new_column_name] = DATASET[timestamp_tag_column].dt.minute

                elif (extracted_info == "second"):

                    DATASET[new_column_name] = DATASET[timestamp_tag_column].dt.second

                else:

                    print("Invalid extracted information. Please select: year, month, week, day, hour, minute, or second.")

            except: # access the attributes from individual objects
            
                for i in range(len(DATASET)):
                    # i goes from zero to the index of the last element of the dataframe DATASET
                    # This element has index len(DATASET) - 1
                    # Append the values to the list according to the selected extracted_info

                    if (extracted_info == 'year'):

                        new_column_vals.append((timestamp_list[i]).year)

                    elif (extracted_info == "month"):

                        new_column_vals.append((timestamp_list[i]).month)

                    elif (extracted_info == "week"):

                        new_column_vals.append((timestamp_list[i]).week)

                    elif (extracted_info == "day"):

                        new_column_vals.append((timestamp_list[i]).day)

                    elif (extracted_info == "hour"):

                        new_column_vals.append((timestamp_list[i]).hour)

                    elif (extracted_info == "minute"):

                        new_column_vals.append((timestamp_list[i]).minute)

                    elif (extracted_info == "second"):

                        new_column_vals.append((timestamp_list[i]).second)

                    else:

                        print("Invalid extracted information. Please select: year, month, week, day, hour, minute, or second.")

                    # Copy the list 'new_column_vals' to a new column of the dataframe, named 'new_column_name':
                    DATASET[new_column_name] = new_column_vals
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Timestamp information successfully extracted. Check dataset\'s 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        #Now that the information were retrieved from all Timestamps, return the new
        #dataframe:
        
        return DATASET


    def CALCULATE_DELAY (df, timestamp_tag_column, new_timedelta_column_name  = None, returned_timedelta_unit = None, return_avg_delay = True):
        
        import numpy as np
        import pandas as pd
        
        #THIS FUNCTION CALCULATES THE DIFFERENCE (timedelta - delay) BETWEEN TWO SUCCESSIVE
        # Timestamps from a same column
        
        #df: dataframe containing the two timestamp columns.
        #timestamp_tag_column: string containing the name of the column with the timestamps
        
        #new_timedelta_column_name: name of the new column. If no value is provided, the default
        #name [timestamp_tag_column1]-[timestamp_tag_column2] will be given:
        
        # return_avg_delay = True will print and return the value of the average delay.
        # return_avg_delay = False will omit this information
        
        if (new_timedelta_column_name is None):
            
            #apply the default name:
            new_timedelta_column_name = "time_delay"
        
        #Pandas Timedelta class: applicable to timedelta objects
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timedelta.html
        #The delta method from the Timedelta class converts returns the timedelta in
        #nanoseconds, guaranteeing the internal compatibility:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timedelta.delta.html#pandas.Timedelta.delta
        
        #returned_timedelta_unit: unit of the new column. If no value is provided, the unit will be
        # considered as nanoseconds. 
        # POSSIBLE VALUES FOR THE TIMEDELTA UNIT:
        #'year', 'month', 'day', 'hour', 'minute', 'second'.
        
        # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.
        # This will prevent any compatibility problems.
        
        #The pd.Timestamp function can handle a single timestamp per call. Then, we must
        # loop trough the series, and apply the function to each element.
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        # try parsing as np.datetime64 (more efficient, without loops):
        try:
            DATASET[timestamp_tag_column] = DATASET[timestamp_tag_column].astype(np.datetime64)
            
            timestamp_list = list(DATASET[timestamp_tag_column])
            
        except:
            # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.
            # 1. Start a list to store the Pandas timestamps:
            timestamp_list = []

            # 2. Loop through each element of the timestamp column, and apply the function
            # to guarantee that all elements are Pandas timestamps

            for timestamp in DATASET[timestamp_tag_column]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column1]
                timestamp_list.append(pd.Timestamp(timestamp, unit = 'ns'))

            # 3. Save the list as the column timestamp_tag_column itself:
            DATASET[timestamp_tag_column] = timestamp_list
        
        # 4. Sort the dataframe in ascending order of timestamps:
        DATASET = DATASET.sort_values(by = timestamp_tag_column, ascending = True)
        # Reset indices:
        DATASET = DATASET.reset_index(drop = True)
        
        # Now, let's create a list of the following timestamps, starting from the second element
        # (index 1) of the timestamp_list:
        following_timestamp = timestamp_list[1:]
        # Append the last element again, since the last timestamp has no following time yet:
        following_timestamp = following_timestamp + timestamp_list[-1:]
        
        # Now, let's store it into a column (series) of the dataframe:
        timestamp_tag_column2 = timestamp_tag_column + "_delayed"
        DATASET[timestamp_tag_column2] = following_timestamp
        
        # Pandas Timestamps can be subtracted to result into a Pandas Timedelta.
        # We will apply the delta method from Pandas Timedeltas.
        
        # 4. Create a timedelta object as the difference between the timestamps:
        
        # NOTICE: Even though a list could not be submitted to direct operations like
        # sum, subtraction and multiplication, the series and NumPy arrays can. When we
        # copied the list as a new column on the dataframes, we converted the lists to series
        # called df[timestamp_tag_column1] and df[timestamp_tag_column2]. These two series now
        # can be submitted to direct operations.
        
        # Delay = next measurement (tag_column2, timestamp higher) - current measurement
        # (tag_column2, timestamp lower). Since we repeated the last timestamp twice,
        # in the last row it will be subtracted from itself, resulting in zero.
        # This is the expected, since we do not have a delay yet
        timedelta_obj = DATASET[timestamp_tag_column2] - DATASET[timestamp_tag_column]
        
        #This timedelta_obj is a series of timedelta64 objects. The Pandas Timedelta function
        # can process only one element of the series in each call. Then, we must loop through
        # the series to obtain the float values in nanoseconds. Even though this loop may 
        # look unecessary, it uses the Delta method to guarantee the internal compatibility.
        # Then, no errors due to manipulation of timestamps with different resolutions, or
        # due to the presence of global variables, etc. will happen. This is the safest way
        # to manipulate timedeltas.
        
        #5. Create an empty list to store the timedeltas in nanoseconds
        TimedeltaList = []
        
        #6. Loop through each timedelta_obj and convert it to nanoseconds using the Delta
        # method. Both pd.Timedelta function and the delta method can be applied to a 
        # a single object.
        #len(timedelta_obj) is the total of timedeltas present.
        
        for i in range(len(timedelta_obj)):
            
            #This loop goes from i = 0 to i = [len(timedelta_obj) - 1], so that
            #all indices are evaluated.
            
            #append the element resultant from the delta method application on the
            # i-th element of the list timedelta_obj, i.e., timedelta_obj[i].
            TimedeltaList.append(pd.Timedelta(timedelta_obj[i]).delta)
        
        #Notice that the loop is needed because Pandas cannot handle a series/list of
        #Timedelta objects simultaneously. It can manipulate a single object
        # in each call or iteration.
        
        #Now the list contains the timedeltas in nanoseconds and guarantees internal
        #compatibility.
        # The delta method converts the Timedelta object to an integer number equals to the
        # value of the timedelta in nanoseconds. Then we are now dealing with numbers, not
        # with timestamps.
        # Even though some steps seem unecessary, they are added to avoid errors and bugs
        # hard to identify, resultant from a timestamp assigned to the wrong type of
        # object.
        
        #The list is not as the series (columns) and arrays: it cannot be directly submitted to 
        # operations like sum, division, and multiplication. For doing so, we can loop through 
        # each element, what would be the case for using the Pandas Timestamp and Timedelta 
        # functions, which can only manipulate one object per call.
        # For simpler operations like division, we can convert the list to a NumPy array and
        # submit the entire array to the operation at the same time, avoiding the use of 
        # memory consuminh iterative methods.
        
        #Convert the timedelta list to a NumPy array:
        # Notice that we could have created a column with the Timedeltalist, so that it would
        # be converted to a series. On the other hand, we still did not defined the name of the
        # new column. So, it is easier to simply convert it to a NumPy array, and then copy
        # the array as a new column.
        TimedeltaList = np.array(TimedeltaList)
        
        #Convert the array to the desired unit by dividing it by the proper factor:
        
        if (returned_timedelta_unit == 'year'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #4. Convert it to days (1 day = 24 h):
            TimedeltaList = TimedeltaList / 24.0 #in days
            
            #5. Convert it to years. 1 year = 365 days + 6 h = 365 days + 6/24 h/(h/day)
            # = (365 + 1/4) days = 365.25 days
            
            TimedeltaList = TimedeltaList / (365.25) #in years
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in years. Considered 1 year = 365 days + 6 h.\n")
        
        
        elif (returned_timedelta_unit == 'month'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #4. Convert it to days (1 day = 24 h):
            TimedeltaList = TimedeltaList / 24.0 #in days
            
            #5. Convert it to months. Consider 1 month = 30 days
            
            TimedeltaList = TimedeltaList / (30.0) #in months
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in months. Considered 1 month = 30 days.\n")
            
        
        elif (returned_timedelta_unit == 'day'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #4. Convert it to days (1 day = 24 h):
            TimedeltaList = TimedeltaList / 24.0 #in days
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in days.\n")
            
        
        elif (returned_timedelta_unit == 'hour'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in hours [h].\n")
        

        elif (returned_timedelta_unit == 'minute'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in minutes [min].\n")
            
            
        elif (returned_timedelta_unit == 'second'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in seconds [s].\n")
            
            
        else:
            
            returned_timedelta_unit = 'ns'
            print("No unit or invalid unit provided for timedelta. Then, returned timedelta in nanoseconds (1s = 10^9 ns).\n")
            
            #In case None unit is provided or a non-valid value or string is provided,
            #The calculus will be in nanoseconds.
        
        #Finally, create a column in the dataframe named as new_timedelta_column_name 
        # with the elements of TimedeltaList converted to the correct unit of time:
        
        #Append the selected unit as a suffix on the new_timedelta_column_name:
        new_timedelta_column_name = new_timedelta_column_name + "_" + returned_timedelta_unit
        
        DATASET[new_timedelta_column_name] = TimedeltaList
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Time delays successfully calculated. Check dataset\'s 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        if (return_avg_delay == True):
            
            # Let's calculate the average delay, print and return it:
            # Firstly, we must remove the last element of the TimedeltaList.
            # Remember that this element is 0 because there is no delay. It was added to allow
            # the element-wise operations between the series.
            # Let's eliminate the last element from TimedeltaList. Since this list was already
            # copied to the dataframe, there is no risk of losing information.
            
            # Index of the last element:
            last_element_index = len(TimedeltaList) - 1
            
            # Slice TimedeltaList until the element of index last_element_index - 1.
            # It will eliminate the last element before we obtain the average:
            TimedeltaList = TimedeltaList[:last_element_index]
            # slice[i:j] slices including index i to index j-1; if the first element is not included,
            # the slices goes from the 1st element; if the last element is not included, slices goes to
            # the last element.
            
            # Now we calculate the average value:
            avg_delay = np.average(TimedeltaList)
            
            print(f"Average delay = {avg_delay} {returned_timedelta_unit}\n")
            
            # Return the dataframe and the average value:
            return DATASET, avg_delay
        
        #Finally, return the dataframe with the new column:
        
        else: 
            # Return only the dataframe
            return DATASET


    def CALCULATE_TIMEDELTA (df, timestamp_tag_column1, timestamp_tag_column2, timedelta_column_name  = None, returned_timedelta_unit = None):
        
        import numpy as np
        import pandas as pd
        
        #THIS FUNCTION PERFORMS THE OPERATION df[timestamp_tag_column1] - df[timestamp_tag_colum2]
        #The declaration order will determine the sign of the output.
        
        #df: dataframe containing the two timestamp columns.
        
        #timestamp_tag_column1: string containing the name of the column with the timestamp
        # on the left (from which the right timestamp will be subtracted).
        
        #timestamp_tag_column2: string containing the name of the column with the timestamp
        # on the right, that will be substracted from the timestamp on the left.
        
        #timedelta_column_name: name of the new column. If no value is provided, the default
        #name [timestamp_tag_column1]-[timestamp_tag_column2] will be given:
        
        if (timedelta_column_name is None):
            
            #apply the default name:
            timedelta_column_name = "[" + timestamp_tag_column1 + "]" + "-" + "[" + timestamp_tag_column2 + "]"
        
        #Pandas Timedelta class: applicable to timedelta objects
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timedelta.html
        #The delta method from the Timedelta class converts the timedelta to
        #nanoseconds, guaranteeing the internal compatibility:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timedelta.delta.html#pandas.Timedelta.delta
        
        #returned_timedelta_unit: unit of the new column. If no value is provided, the unit will be
        # considered as nanoseconds. 
        # POSSIBLE VALUES FOR THE TIMEDELTA UNIT:
        #'year', 'month', 'day', 'hour', 'minute', 'second'.
        
        # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.
        # This will prevent any compatibility problems.
        
        #The pd.Timestamp function can handle a single timestamp per call. Then, we must
        # loop trough the series, and apply the function to each element.
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        # try parsing as np.datetime64 (more efficient, without loops):
        try:
            DATASET[timestamp_tag_column1] = DATASET[timestamp_tag_column1].astype(np.datetime64)
            DATASET[timestamp_tag_column2] = DATASET[timestamp_tag_column2].astype(np.datetime64)
            
        except:
            # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.  
            # 1. Start a list to store the Pandas timestamps:
            timestamp_list = []

            # 2. Loop through each element of the timestamp column, and apply the function
            # to guarantee that all elements are Pandas timestamps

            for timestamp in DATASET[timestamp_tag_column1]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column1]
                timestamp_list.append(pd.Timestamp(timestamp, unit = 'ns'))

            # 3. Create a column in the dataframe that will store the timestamps.
            # Simply copy the list as the column:
            DATASET[timestamp_tag_column1] = timestamp_list

            #Repeate these steps for the other column (timestamp_tag_column2):
            # Restart the list, loop through all the column, and apply the pd.Timestamp function
            # to each element, individually:
            timestamp_list = []

            for timestamp in DATASET[timestamp_tag_column2]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column2]
                timestamp_list.append(pd.Timestamp(timestamp, unit = 'ns'))

            DATASET[timestamp_tag_column2] = timestamp_list
        
        # 4. Sort the dataframe in ascending order of timestamps:
        DATASET = DATASET.sort_values(by = [timestamp_tag_column1, timestamp_tag_column2], ascending = [True, True])
        # Reset indices:
        DATASET = DATASET.reset_index(drop = True)
        
        # Pandas Timestamps can be subtracted to result into a Pandas Timedelta.
        # We will apply the delta method from Pandas Timedeltas.
        
        #5. Create a timedelta object as the difference between the timestamps:
        
        # NOTICE: Even though a list could not be submitted to direct operations like
        # sum, subtraction and multiplication, the series and NumPy arrays can. When we
        # copied the list as a new column on the dataframes, we converted the lists to series
        # called df[timestamp_tag_column1] and df[timestamp_tag_column2]. These two series now
        # can be submitted to direct operations.
        
        timedelta_obj = DATASET[timestamp_tag_column1] - DATASET[timestamp_tag_column2]
        
        #This timedelta_obj is a series of timedelta64 objects. The Pandas Timedelta function
        # can process only one element of the series in each call. Then, we must loop through
        # the series to obtain the float values in nanoseconds. Even though this loop may 
        # look unecessary, it uses the Delta method to guarantee the internal compatibility.
        # Then, no errors due to manipulation of timestamps with different resolutions, or
        # due to the presence of global variables, etc. will happen. This is the safest way
        # to manipulate timedeltas.
        
        #5. Create an empty list to store the timedeltas in nanoseconds
        TimedeltaList = []
        
        #6. Loop through each timedelta_obj and convert it to nanoseconds using the Delta
        # method. Both pd.Timedelta function and the delta method can be applied to a 
        # a single object.
        #len(timedelta_obj) is the total of timedeltas present.
        
        for i in range(len(timedelta_obj)):
            
            #This loop goes from i = 0 to i = [len(timedelta_obj) - 1], so that
            #all indices are evaluated.
            
            #append the element resultant from the delta method application on the
            # i-th element of the list timedelta_obj, i.e., timedelta_obj[i].
            TimedeltaList.append(pd.Timedelta(timedelta_obj[i]).delta)
        
        #Notice that the loop is needed because Pandas cannot handle a series/list of
        #Timedelta objects simultaneously. It can manipulate a single object
        # in each call or iteration.
        
        #Now the list contains the timedeltas in nanoseconds and guarantees internal
        #compatibility.
        # The delta method converts the Timedelta object to an integer number equals to the
        # value of the timedelta in nanoseconds. Then we are now dealing with numbers, not
        # with timestamps.
        # Even though some steps seem unecessary, they are added to avoid errors and bugs
        # hard to identify, resultant from a timestamp assigned to the wrong type of
        # object.
        
        #The list is not as the series (columns) and arrays: it cannot be directly submitted to 
        # operations like sum, division, and multiplication. For doing so, we can loop through 
        # each element, what would be the case for using the Pandas Timestamp and Timedelta 
        # functions, which can only manipulate one object per call.
        # For simpler operations like division, we can convert the list to a NumPy array and
        # submit the entire array to the operation at the same time, avoiding the use of 
        # memory consuminh iterative methods.
        
        #Convert the timedelta list to a NumPy array:
        # Notice that we could have created a column with the Timedeltalist, so that it would
        # be converted to a series. On the other hand, we still did not defined the name of the
        # new column. So, it is easier to simply convert it to a NumPy array, and then copy
        # the array as a new column.
        TimedeltaList = np.array(TimedeltaList)
        
        #Convert the array to the desired unit by dividing it by the proper factor:
        
        if (returned_timedelta_unit == 'year'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #4. Convert it to days (1 day = 24 h):
            TimedeltaList = TimedeltaList / 24.0 #in days
            
            #5. Convert it to years. 1 year = 365 days + 6 h = 365 days + 6/24 h/(h/day)
            # = (365 + 1/4) days = 365.25 days
            
            TimedeltaList = TimedeltaList / (365.25) #in years
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in years. Considered 1 year = 365 days + 6 h.\n")
        
        
        elif (returned_timedelta_unit == 'month'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #4. Convert it to days (1 day = 24 h):
            TimedeltaList = TimedeltaList / 24.0 #in days
            
            #5. Convert it to months. Consider 1 month = 30 days
            
            TimedeltaList = TimedeltaList / (30.0) #in months
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in months. Considered 1 month = 30 days.\n")
            
        
        elif (returned_timedelta_unit == 'day'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #4. Convert it to days (1 day = 24 h):
            TimedeltaList = TimedeltaList / 24.0 #in days
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in days.\n")
            
        
        elif (returned_timedelta_unit == 'hour'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #3. Convert it to hours (1 h = 60 min):
            TimedeltaList = TimedeltaList / 60.0 #in hours
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in hours [h].\n")
        

        elif (returned_timedelta_unit == 'minute'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #2. Convert it to minutes (1 min = 60 s):
            TimedeltaList = TimedeltaList / 60.0 #in minutes
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in minutes [min].\n")
            
            
        elif (returned_timedelta_unit == 'second'):
            
            #1. Convert the list to seconds (1 s = 10**9 ns, where 10**9 represents
            #the potentiation operation in Python, i.e., 10^9. e.g. 10**2 = 100):
            TimedeltaList = TimedeltaList / (10**9) #in seconds
            
            #The .0 after the numbers guarantees a float division.
            
            print("Returned timedelta in seconds [s].\n")
            
            
        else:
            
            returned_timedelta_unit = 'ns'
            print("No unit or invalid unit provided for timedelta. Then, returned timedelta in nanoseconds (1s = 10^9 ns).\n")
            
            #In case None unit is provided or a non-valid value or string is provided,
            #The calculus will be in nanoseconds.
        
        #Finally, create a column in the dataframe named as timedelta_column_name 
        # with the elements of TimedeltaList converted to the correct unit of time:
        
        #Append the selected unit as a suffix on the timedelta_column_name:
        timedelta_column_name = timedelta_column_name + "_" + returned_timedelta_unit
        
        DATASET[timedelta_column_name] = TimedeltaList
        
        # Sort the dataframe in ascending order of timestamps.
        # Importance order: timestamp1, timestamp2, timedelta
        DATASET = DATASET.sort_values(by = [timestamp_tag_column1, timestamp_tag_column2, timedelta_column_name], ascending = [True, True, True])
        # Reset indices:
        DATASET = DATASET.reset_index(drop = True)
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Timedeltas successfully calculated. Check dataset\'s 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        #Finally, return the dataframe with the new column:
        
        return DATASET


    def ADD_TIMEDELTA (df, timestamp_tag_column, timedelta, new_timestamp_col  = None, timedelta_unit = None):
        
        import numpy as np
        import pandas as pd
        
        #THIS FUNCTION PERFORMS THE OPERATION ADDING A FIXED TIMEDELTA (difference of time
        # or offset) to a timestamp.
        
        #df: dataframe containing the timestamp column.
        
        #timestamp_tag_column: string containing the name of the column with the timestamp
        # to which the timedelta will be added to.
        
        #timedelta: numeric value of the timedelta.
        # WARNING: simply input a numeric value, not a string with unit. e.g. timedelta = 2.4
        # If you want to subtract a timedelta, input a negative value. e.g. timedelta = - 2.4
        
        #new_timestamp_col: name of the new column containing the obtained timestamp. 
        # If no value is provided, the default name [timestamp_tag_column]+[timedelta] 
        # will be given (at the end of the code, after we created the timedelta object 
        # with correct units)
        
        #Pandas Timedelta class: applicable to timedelta objects
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timedelta.html
        #The delta method from the Timedelta class converts returns the timedelta in
        #nanoseconds, guaranteeing the internal compatibility:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timedelta.delta.html#pandas.Timedelta.delta
        
        #timedelta_unit: unit of the timedelta interval. If no value is provided, 
        # the unit will be considered 'ns' (default). Possible values are:
        #'day', 'hour', 'minute', 'second', 'ns'.
        
        if (timedelta_unit is None):
            
            timedelta_unit = 'ns'
        
        # Pandas do not support timedeltas in years or months, since these values may
        # be ambiguous (e.g. a month may have 30 or 31 days, so an approximation would
        # be necessary).
        
        # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.
        # This will prevent any compatibility problems.
        
        #The pd.Timestamp function can handle a single timestamp per call. Then, we must
        # loop trough the series, and apply the function to each element.
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        # try parsing as np.datetime64 (more efficient, without loops):
        try:
            DATASET[timestamp_tag_column] = DATASET[timestamp_tag_column].astype(np.datetime64)
            
        except:
            # START: CONVERT ALL TIMESTAMPS/DATETIMES/STRINGS TO pandas.Timestamp OBJECTS.
            #1. Start a list to store the Pandas timestamps:
            timestamp_list = []

            #2. Loop through each element of the timestamp column, and apply the function
            # to guarantee that all elements are Pandas timestamps

            for timestamp in DATASET[timestamp_tag_column]:
                #Access each element 'timestamp' of the series df[timestamp_tag_column1]
                timestamp_list.append(pd.Timestamp(timestamp, unit = 'ns'))

            #3. Create a column in the dataframe that will store the timestamps.
            # Simply copy the list as the column:
            DATASET[timestamp_tag_column] = timestamp_list
        
        # The Pandas Timestamp can be directly added to a Pandas Timedelta.
    
        #Dictionary for converting the timedelta_unit to Pandas encoding for the
        # Timedelta method. to access the element of a dictionary d = {"key": element},
        # simply declare d['key'], as if you were accessing the column of a dataframe. Here,
        # the key is the argument of the function, whereas the element is the correspondent
        # Pandas encoding for this method. With this dictionary we simplify the search for the
        # proper time encoding: actually, depending on the Pandas method the encoding may be
        # 'd', "D" or "day" for day, for instance. So, we avoid having to check the whole
        # documentation by creating a simpler common encoding for the functions in this notebook.
        
        unit_dict = {
            
            'day': 'd',
            'hour': 'h',
            'minute': 'min',
            'second': 's',
            'ns': 'ns'
            
        }
        
        #Create the Pandas timedelta object from the timedelta value and the selected
        # time units:
        timedelta = pd.Timedelta(timedelta, unit_dict[timedelta_unit])
        
        #A pandas Timedelta object has total compatibility with a pandas
        #Timestamp, so we can simply add the Timedelta to the Timestamp to obtain a new 
        #corrected timestamp.
        # Again, notice that the timedelta can be positive (sum of time), or negative
        # (subtraction of time).
        
        #Now, add the timedelta to the timestamp, and store it into a proper list/series:
        new_timestamps = DATASET[timestamp_tag_column].copy()
        new_timestamps = new_timestamps + timedelta
        
        #Finally, create a column in the dataframe named as new_timestamp_col
        #and store the new timestamps into it
        
        if (new_timestamp_col is None):
            
            #apply the default name:
            new_timestamp_col = "[" + timestamp_tag_column + "]" + "+" + "[" + str(timedelta) + "]"
            #The str function converts the timedelta object to a string, so it can be
            #concatenated in this line of code.
            #Notice that we defined the name of the new column at the end of the code so
            #that we already converted the 'timedelta' to a Timedelta object containing
            #the correct units.
        
        DATASET[new_timestamp_col] = new_timestamps
        
        # Sort the dataframe in ascending order of timestamps.
        # Importance order: timestamp, new_timestamp_col
        DATASET = DATASET.sort_values(by = [timestamp_tag_column, new_timestamp_col], ascending = [True, True])
        # Reset indices:
        DATASET = DATASET.reset_index(drop = True)
        
        # Pandas .head(Y) method results in a dataframe containing the first Y rows of the 
        # original dataframe. The default .head() is Y = 5. Print first 10 rows of the 
        # new dataframe:
        print("Timedeltas successfully added. Check dataset\'s 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        #Finally, return the dataframe with the new column:
        
        return DATASET


    def SLICE_DATAFRAME (df, from_row = 'first_only', to_row = 'only', restart_index_of_the_sliced_dataframe = False):
        
        import numpy as np
        import pandas as pd
        
        # restart_index_of_the_sliced_dataframe = False to keep the 
        # same row index of the original dataframe; or 
        # restart_index_of_the_sliced_dataframe = True to reset indices 
        # (start a new index, from 0 for the first row of the 
        # returned dataframe).
        
        # from_row and to_row: integer or strings:
        
        # from_row may be any integer from 0 to the last row of the dataset
        # and the following strings: 'first' and 'first_only'
        
        # to_row may be any integer from 0 to the last row of the dataset
        # and the following strings: 'last', 'last_only', and 'only'
        
        # the combination from_row = 'first', to_row = 'last' will
        # return the original dataframe itself.
        # The same is valid for the combination from_row = 'first_only', 
        # to_row = 'last_only'; or of combinations between from_row = 0
        # (index of the first row) with 'last' or the index
        # of the last row; or combinations between 'first' and the index
        # of the last row.
        
        # These possibilities are the first checked by the code. If none
        # of these special cases are present, then:
        
        # from_row = 'first_only' selects a dataframe containing only the
        # first row, independently of the parameter passed as to_row;
        
        # to_row = 'last_only' selects a dataframe containing only the
        # last row, independently of the parameter passed as from_row;
        
        # if to_row = 'only', the sliced dataframe will be formed by only the
        # row passed as from_row (an integer representing the row index is
        # passed) - explained in the following lines
        
        # These three special cases are dominant over the following ones
        # (they are checked firstly, and force the modifying of slicing
        # limits):
        
        # Other special cases:
        
        # from_row = 'first' starts slicing on the first row (index 0) -
        # the 1st row from the dataframe will be the 1st row of the sliced
        # dataframe too.
        
        # to_row = 'last' finishes slicing in the last row - the last row
        # from the dataframe will be the last row of the sliced dataframe.
        
        # If i and j are integer numbers, they represent the indices of rows:
        
        # from_row = i starts the sliced dataframe from the row of index i
        # of the original dataframe.
        # e.g. from_row = 8 starts the slicing from row with index 8. Since
        # slicing starts from 0, this is the 9th row of the original dataframe.
        
        # to_row = j finishes the sliced dataframe on the row of index j of
        # the original dataframe. Attention: this row with index j is included,
        # and will be the last_row of the sliced dataframe.
        # e.g. if to_row = 21, the last row of the sliced dataframe will be the
        # row with index 21 of the original dataframe. Since slicing starts
        # from 0, this is the 22nd row of the original dataframe.
        
        # In summary, if from_row = 8, to_row = 21, the sliced dataframe
        # will be formed from the row of index 8 to the row of index 21 of
        # the original dataframe, including both the row of index 8 and the row
        # index 21. 
        
        # from_row is effectively the first row of the new dataframe;
        # and to_row is effectively the last row of the new dataframe.
        
        # Notice that the use of to_row < from_row will raise an error.
        
        # Create dataframe local copy to manipulate, avoiding that Pandas operates on
        # the original object; or that Pandas tries to set values on slices or copies,
        # resulting in unpredictable results.
        # Use the copy method to effectively create a second object with the same properties
        # of the input parameters, but completely independent from it.
        DATASET = df.copy(deep = True)
        
        # Store the total number of rows as num_rows:
        num_rows = len(DATASET)
        
        
        # Check if no from_row and to_row arguments were provided:
        if (from_row is None):
            print("No input for the argument \'from_row\'. Then, setting the start of the slice as the first row.\n")
            first_row_index = 0
        
        if (to_row is None):
            print("No input for the argument \'to_row\'. Then, setting the end of the slice as the last row.\n")
            last_row_index = (num_rows - 1)
        
        
        # Check type of the inputs (strings or integers):
        from_row_type = type(from_row)
        to_row_type = type(to_row)
        
        if (from_row_type == str):
            # It is a string
            
            if ((from_row == 'first') | (from_row == 'first_only')):
                # Set the first_row_index as the 0 (1st row index):
                first_row_index = 0
            
            else:
                print("Invalid string input for the argument \'from_row\'. Then, setting the start of the slice as the first row.\n")
                first_row_index = 0
        
        else:
            # Numeric input. Use the int attribute to guarantee that it 
            # was read as an integer. This value is itself the index of
            # the first row of the sliced dataframe:
            first_row_index = int(from_row)
        
        
        if (to_row_type == str):
            # It is a string
            
            if ((to_row == 'last') | (to_row == 'last_only')):
                # Set the last_row_index as the index of the last row of the dataframe:
                last_row_index = (num_rows - 1)
                # In the following code, we do last_row_index = last_row_index + 1 to
                # guarantee that the last row is actually included in the sliced df.
                
                # If to_row == 'last_only', we must correct first_row_index:
                # first_row_index was previously defined as 0 or as the value of the row
                # index provided. It must be the index of the last row, though:
                if (to_row == 'last_only'):
                    first_row_index = last_row_index
                    print("\'last_only\' argument provided, so starting the slicing from the last row of the dataframe.\n")
                
            elif (to_row == 'only'):
                # Use only the row declared as from_row
                last_row_index = first_row_index
                # In the following code, we do last_row_index = last_row_index + 1 to
                # guarantee that the last row is actually included in the sliced df.
            
            else:
                print("Invalid string input for the argument \'to_row\'. Then, setting the end of the slice as the last row.\n")
                last_row_index = (num_rows - 1)
        
        elif (from_row == 'first_only'):
            # In this case, last row index must be zero:
            last_row_index = 0
        
        else:
            # Numeric input. Use the int attribute to guarantee that it 
            # was read as an integer. This value is itself the index of
            # the last row of the sliced dataframe:
            last_row_index = int(to_row)
        
        
        # Check the special combination from = 1st row to last row
        # and return the original dataframe itself, without performing
        # operations:
        
        if ((from_row == 'first_only') & (to_row == 'last_only')):
            
            #return the dataframe without performing any operation
            print("Sliced dataframe is the original dataframe itself.")
            return DATASET
        
        elif ((first_row_index == 0) & (last_row_index == (num_rows - 1))):
            
            #return the dataframe without performing any operation
            print("Sliced dataframe is the original dataframe itself.")
            return DATASET
            
        # The two special combinations were checked, now we can back to
        # the main code
        
        
        # Slice a dataframe: df[i:j]
        # Slice the dataframe, getting only row i to row (j-1)
        # Indexing naturally starts from 0
        # Notice that the slicer defined as df[i:j] takes all columns from
        # the dataframe: it copies the dataframe structure (columns), but
        # selects only the specified rows.
        
        # first_row = df[0:1]
        # This is equivalent to df[:1] - if there is no start for the
        # slicer, the start from 0 is implicit
        # slice: get rows from row 0 to row (1-1) = 0
        # Therefore, we will obtain a copy of the dataframe, but containing
        # only the first row (row 0)
        
        # last_row = df[(num_rows - 1):(num_rows)] 
        # slice the dataframe from row (num_rows - 1), the index of the
        # last row, to row (num_rows) - 1 = (num_rows - 1)
        # Therefore, this slicer is a copy of the dataframe but containing
        # only its last row.
        
        # Slices are (fractions of) pandas dataframes, so elements must be
        # accessed through .iloc or .loc method
        
        
        # Set slicing limits:
        i = first_row_index # i is included
        j = last_row_index + 1
        # df[i:j] will include row i to row j - 1 = 
        # (last_row_index + 1) - 1 = last_row_index
        # Then, by summing 1 we guarantee that the row passed as
        # last_row_index will be actually included.
        # notice that when last_row_index = first_row_index
        # j will be the index of the next line.
        # e.g. the slice of only the first line must be df[0:1]
        # there must be a difference of 1 to include 1 line.
        
        # Now, slice the dataframe from line of index i to
        # line j-1, where line (j-1) is the last one included:
        
        sliced_df = DATASET[i:j]
        
        if (restart_index_of_the_sliced_dataframe == True):
            # Reset the index:
            sliced_df = sliced_df.reset_index(drop = True)
            print("Index of the returned dataframe was restarted.\n")
        
        print(f"Returning sliced dataframe, containing {sliced_df.shape[0]} rows and {sliced_df.shape[1]} columns.\n")
        # dataframe.shape is a tuple (N, M), where dataframe.shape[0] = N is
        # the number of rows; and dataframe.shape[1] = M is the number of columns
        # of the dataframe
        
        print("Check the dataframe below:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(sliced_df)
                
        except: # regular mode
            print(sliced_df)
        
        return sliced_df


    def visualize_and_characterize_missing_values (df, slice_time_window_from = None, slice_time_window_to = None, aggregate_time_in_terms_of = None):

        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import missingno as msno
        # misssingno package is built for visualizing missing values. 
        
        # df: dataframe to be analyzed
        
        # slice_time_window_from and slice_time_window_to (timestamps). When analyzing time series,
        # use these parameters to observe only values in a given time range.
        
        # slice_time_window_from: the inferior limit of the analyzed window. If you declare this value
        # and keep slice_time_window_to = None, then you will analyze all values that comes after
        # slice_time_window_from.
        # slice_time_window_to: the superior limit of the analyzed window. If you declare this value
        # and keep slice_time_window_from = None, then you will analyze all values until
        # slice_time_window_to.
        # If slice_time_window_from = slice_time_window_to = None, only the standard analysis with
        # the whole dataset will be performed. If both values are specified, then the specific time
        # window from 'slice_time_window_from' to 'slice_time_window_to' will be analyzed.
        # e.g. slice_time_window_from = 'May-1976', and slice_time_window_to = 'Jul-1976'
        # Notice that the timestamps must be declares in quotes, just as strings.
        
        # aggregate_time_in_terms_of = None. Keep it None if you do not want to aggregate the time
        # series. Alternatively, set aggregate_time_in_terms_of = 'Y' or aggregate_time_in_terms_of = 
        # 'year' to aggregate the timestamps in years; set aggregate_time_in_terms_of = 'M' or
        # 'month' to aggregate in terms of months; or set aggregate_time_in_terms_of = 'D' or 'day'
        # to aggregate in terms of days.
        
        print("Possible reasons for missing data:\n")
        print("One of the obvious reasons is that data is simply missing at random.")
        print("Other reasons might be that the missingness is dependent on another variable;")
        print("or it is due to missingness of the same variables or other variables.\n")
        
        print("Types of missingness:\n")
        print("Identifying the missingness type helps narrow down the methodologies that you can use for treating missing data.")
        print("We can group the missingness patterns into 3 broad categories:\n")
        
        print("Missing Completely at Random (MCAR)\n")
        print("Missingness has no relationship between any values, observed or missing.")
        print("Example: consider you have a class of students. There are a few students absent on any given day. The students are absent just randomly for their specific reasons. This is missing completely at random.\n")
        
        print("Missing at Random (MAR)\n")
        print("There is a systematic relationship between missingness and other observed data, but not the missing data.")
        print("Example: consider the attendance in a classroom of students during winter, where many students are absent due to the bad weather. Although this might be at random, the hidden cause might be that students sitting closer might have contracted a fever.\n")
        print("Missing at random means that there might exist a relationship with another variable.")
        print("In this example, the attendance is slightly correlated to the season of the year.")
        print("It\'s important to notice that, for MAR, missingness is dependent only on the observed values; and not the other missing values.\n")
        
        print("Missing not at Random (MNAR)\n")
        print("There is a relationship between missingness and its values, missing or non-missing.")
        print("Example: in our class of students, it is Sally\'s birthday. Sally and many of her friends are absent to attend her birthday party. This is not at all random as Sally and only her friends are absent.\n")
        
        # set a local copy of the dataframe:
        DATASET = df.copy(deep = True)
        
        # Start the agg_dict, a dictionary that correlates the input aggregate_time_in_terms_of to
        # the correspondent argument that must be passed to the matrix method:
        agg_dict = {
            
            'year': 'Y',
            'Y': 'Y',
            'month': 'M',
            'M': 'M',
            'day': 'D',
            'D':'D'
        }
        
        
        if not (aggregate_time_in_terms_of is None):
            # access the frequency in the dictionary
            frequency = agg_dict[aggregate_time_in_terms_of] 
        
        df_length = len(DATASET)
        print(f"Count of rows from the dataframe =\n")
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(df_length)
                
        except: # regular mode
            print(df_length)
        
        print("\n")

        # Show total of missing values for each variable:
        total_of_missing_values_series = DATASET.isna().sum()
        # This is a series which uses the original column names as index
        proportion_of_missing_values_series = DATASET.isna().mean()
        percent_of_missing_values_series = proportion_of_missing_values_series * 100
        missingness_dict = {'count_of_missing_values': total_of_missing_values_series,
                        'proportion_of_missing_values': proportion_of_missing_values_series,
                        'percent_of_missing_values': percent_of_missing_values_series}
        
        df_missing_values = pd.DataFrame(data = missingness_dict)
        # Now, the dataframe has the original columns set as index, but this index has no name.
        # Let's rename it using the .rename method from Pandas Index object:
        df_missing_values.index.rename(name = 'dataframe_column', inplace = True)
        
        # Create a one row dataframe with the missingness for the whole dataframe:
        # Pass the scalars as single-element lists or arrays:
        one_row_data = {'dataframe_column': ['missingness_accross_rows'],
                        'count_of_missing_values': [len(DATASET) - len(DATASET.copy(deep = True).dropna(how = 'any'))],
                        'proportion_of_missing_values': [(len(DATASET) - len(DATASET.copy(deep = True).dropna(how = 'any')))/(len(DATASET))],
                        'percent_of_missing_values': [(len(DATASET) - len(DATASET.copy(deep = True).dropna(how = 'any')))/(len(DATASET))*100]
                        }
        one_row_df = pd.DataFrame(data = one_row_data)
        one_row_df.set_index('dataframe_column', inplace = True)
        
        # Append this one_row_df to df_missing_values:
        df_missing_values = pd.concat([df_missing_values, one_row_df])
        
        print("Missing values on each feature; and missingness considering all rows from the dataframe:")
        print("(note: \'missingness_accross_rows\' was calculated by: checking which rows have at least one missing value (NA); and then comparing total rows with NAs with total rows in the dataframe).\n")
        
        try:
            display(df_missing_values)    
        except:
            print(df_missing_values)
        print("\n") # line_break
        
        print("Bar chart of the missing values - Nullity bar:\n")
        msno.bar(DATASET)
        plt.show()
        print("\n")
        print("The nullity bar allows us to visualize the completeness of the dataframe.\n")
        
        print("Nullity Matrix: distribution of missing values through the dataframe:\n")
        msno.matrix(DATASET)
        plt.show()
        print("\n")
        
        if not ((slice_time_window_from is None) | (slice_time_window_to is None)):
            
            # There is at least one of these two values for slicing:
            if not ((slice_time_window_from is None) & (slice_time_window_to is None)):
                    # both are present
                    
                    if not (aggregate_time_in_terms_of is None):
                        print("Nullity matrix for the defined time window and for the selected aggregation frequency:\n")
                        msno.matrix(DATASET.loc[slice_time_window_from:slice_time_window_to], freq = frequency)
                        
                    else:
                        # do not aggregate:
                        print("Nullity matrix for the defined time window:\n")
                        msno.matrix(DATASET.loc[slice_time_window_from:slice_time_window_to])
                    
                    plt.show()
                    print("\n")
            
            elif not (slice_time_window_from is None):
                # slice only from the start. The condition where both limits were present was already
                # checked. To reach this condition, only one is not None
                # slice from 'slice_time_window_from' to the end of dataframe
                
                    if not (aggregate_time_in_terms_of is None):
                        print("Nullity matrix for the defined time window and for the selected aggregation frequency:\n")
                        msno.matrix(DATASET.loc[slice_time_window_from:], freq = frequency)
                    
                    else:
                        # do not aggregate:
                        print("Nullity matrix for the defined time window:\n")
                        msno.matrix(DATASET.loc[slice_time_window_from:])
            
                    plt.show()
                    print("\n")
                
            else:
            # equivalent to elif not (slice_time_window_to is None):
                # slice only from the beginning to the upper limit. 
                # The condition where both limits were present was already checked. 
                # To reach this condition, only one is not None
                # slice from the beginning to 'slice_time_window_to'
                
                    if not (aggregate_time_in_terms_of is None):
                        print("Nullity matrix for the defined time window and for the selected aggregation frequency:\n")
                        msno.matrix(DATASET.loc[:slice_time_window_to], freq = frequency)
                    
                    else:
                        # do not aggregate:
                        print("Nullity matrix for the defined time window:\n")
                        msno.matrix(DATASET.loc[:slice_time_window_to])
                    
                    plt.show()
                    print("\n")
        
        else:
            # Both slice limits are not. Let's check if we have to aggregate the dataframe:
            if not (aggregate_time_in_terms_of is None):
                    print("Nullity matrix for the selected aggregation frequency:\n")
                    msno.matrix(DATASET, freq = frequency)
                    plt.show()
                    print("\n")
        
        print("The nullity matrix allows us to visualize the location of missing values in the dataset.")
        print("The nullity matrix describes the nullity in the dataset and appears blank wherever there are missing values.")
        print("It allows us to quickly analyze the patterns in missing values.")
        print("The sparkline on the right of the matrix summarizes the general shape of data completeness and points out the row with the minimum number of null values in the dataframe.")
        print("In turns, the nullity matrix shows the total counts of columns at its bottom.")
        print("We can previously slice the dataframe for a particular interval of analysis (e.g. slice the time interval) to obtain more clarity on the amount of missingness.")
        print("Slicing will be particularly helpful when analyzing large datasets.\n")
        print("MCAR: plotting the missingness matrix plot (nullity matrix) for a MCAR variable will show values missing at random, with no correlation or clear pattern.")
        print("Correlation here implies the dependency of missing values on another variable present or absent.\n")
        print("MAR: the nullity matrix for MAR can be visualized as the presence of many missing values for a given feature. In this case, there might be a reason for the missingness that cannot be directly observed.\n")
        print("MNAR: the nullity matrix for MNAR shows a strong correlation between the missingness of two variables A and B.")
        print("This correlation is easily observable by sorting the dataframe in terms of A or B before obtaining the matrix.\n")
        
        print("Missingness Heatmap:\n")
        msno.heatmap(DATASET)
        plt.show()
        print("\n")
        
        print("The missingness heatmap describes the correlation of missingness between columns.")
        print("The heatmap is a graph of correlation of missing values between columns.")
        print("It explains the dependencies of missingness between columns.")
        print("In simple terms, if the missingness for two columns are highly correlated, then the heatmap will show high values of coefficient of correlation R2 for them.")
        print("That is because columns where the missing values co-occur the maximum are highly related and vice-versa.\n")
        print("In the graph, the redder the color, the lower the correlation between the missing values of the columns.")
        print("In turns, the bluer the color, the higher the correlation of missingness between the two variables.\n")
        print("ATTENTION: before deciding if the missing values in one variable is correlated with other, so that they would be characterized as MAR or MNAR, check the total of missing values.")
        print("Even if the heatmap shows a certain degree of correlation, the number of missing values may be too small to substantiate that.")
        print("Missingness in very small number may be considered completely random, and missing values can be eliminated.\n")
        
        print("Missingness Dendrogram:\n")
        msno.dendrogram(DATASET)
        plt.show()
        print("\n")
        
        print("A dendrogram is a tree diagram that groups similar objects in close branches.")
        print("So, the missingness dendrogram is a tree diagram of missingness that describes correlation of variables by grouping similarly missing columns together.")
        print("To interpret this graph, read it from a top-down perspective.")
        print("Cluster leaves which are linked together at a distance of zero fully predict one another\'s presence.")
        print("In other words, when two variables are grouped together in the dendogram, one variable might always be empty while another is filled (the presence of one explains the missingness of the other), or they might always both be filled or both empty, and so on (the missingness of one explains the missigness of the other).\n")
        
        return df_missing_values


    def visualizing_and_comparing_missingness_across_numeric_vars (df, column_to_analyze, column_to_compare_with, show_interpreted_example = False, grid = True, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import os
        # Two conditions require the os library, so we import it at the beginning of the function,
        # to avoid importing it twice.
        import shutil # component of the standard library to move or copy files.
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        
        # column_to_analyze, column_to_compare_with: strings (in quotes).
        # column_to_analyze is the column from the dataframe df that will be analyzed in terms of
        # missingness; whereas column_to_compare_with is the column to which column_to_analyze will
        # be compared.
        # e.g. column_to_analyze = 'column1' will analyze 'column1' from df.
        # column_to_compare_with = 'column2' will compare 'column1' against 'column2'
        
        # show_interpreted_example: set as True if you want to see an example of a graphic analyzed and
        # interpreted.
        
        print("Missingness across a variable:\n")
        print("In this analysis, we will graphically analyze the relationship between missing values and non-missing values.")
        print("To do so, we will start by visualizing the missingness of a variable against another variable.")
        print("The scatter plot will show missing values in one color, and non-missing values in other color.")
        print("It will allow us to visualize how missingness of a variable changes against another variable.")
        print("Analyzing the missingness of a variable against another variable helps you determine any relationships between missing and non-missing values.")
        print("This is very similar to how you found correlations of missingness between two columns.")
        print("In summary, we will plot a scatter plot to analyze if there is any correlation of missingness in one column against another column.\n")
        
        # To create the graph, we will use the matplotlib library. 
        # However, matplotlib skips all missing values while plotting. 
        # Therefore, we would need to first create a function that fills in dummy values for all the 
        # missing values in the DataFrame before plotting.
        
        # We will create a function 'fill_dummy_values' that fill in all columns in the DataFrame.
        #The operations involve shifting and scaling the column range with a scaling factor.
            
        # We use a for loop to produce dummy values for all the columns in a given DataFrame. 
        # We can also define the scaling factor so that we can resize the range of dummy values. 
        # In addition to the previous steps of scaling and shifting the dummy values, we'll also have to 
        # create a copy of the DataFrame to fill in dummy values first. Let's now use this function to 
        # create our scatterplot.
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        # define a subfunction for filling the dummy values.
        # In your function definition, set the default value of scaling_factor to be 0.075:
        def fill_dummy_values(df, scaling_factor = 0.075):
            
            # To generate dummy values, we can use the 'rand()' function from 'numpy.random'. 
            # We first store the number of missing values in column_to_analyze to 'num_nulls'
            # and then generate an array of random dummy values of the size 'num_nulls'. 
            # The generated dummy values appear as shown beside on the graph. 
            
            # The rand function always outputs values between 0 and 1. 
            # However, you must observe that the values of both column_to_analyze and column_to_compare_with 
            # have their own ranges, that may be different. 
            # Hence we'll need to scale and shift the generated dummy values so that they nicely fit 
            # into the graph.
            
            from numpy.random import rand
            # https://numpy.org/doc/stable/reference/random/generated/numpy.random.rand.html?msclkid=7414313ace7611eca18491dd4e7e86ae
            
            df_dummy = df.copy(deep = True)
            # Get the list of columns from df_dummy.
            # Use the list attribute to convert the array df_dummy.columns to list:
            df_cols_list = list(df_dummy.columns)
            
            # Calculate the number of missing values in each column of the dummy DataFrame.
            for col_name in df_dummy:
                
                col = df_dummy[col_name]
                # Create a column informing if the element is missing (True)
                # or not (False):
                col_null = col.isnull()
                # Calculate number of missing values in this column:
                num_nulls = col_null.sum()
                
                # Return the index j of column col_name. 
                # Use the index method from lists, setting col_name as argument. It will return
                # the index of col_name from the list of columns
                # https://www.programiz.com/python-programming/methods/list/index#:~:text=The%20list%20index%20%28%29%20method%20can%20take%20a,-%20search%20the%20element%20up%20to%20this%20index?msclkid=a690b8dacfaa11ec8e84e10a50ae45ec
                j = df_cols_list.index(col_name)
                
                # Check if the column is a text or timestamp. In this case, the type
                # of column will be 'object'
                if (col.dtype not in numeric_dtypes):
                    
                    # Try converting it to a datetime64 object:
                    
                    try:
                        
                        col = (col).astype('datetime64[ns]')
                    
                    except:
                        
                        # It is not a timestamp, so conversion was not possible.
                        # Simply ignore it.
                        pass
                    
                # Now, try to perform the scale adjustment:
                try:
                    # Calculate column range
                    col_range = (col.max() - col.min())

                    # Scale the random values to scaling_factor times col_range
                    # Calculate random values with the size of num_nulls.
                    # The rand() function takes in as argument the size of the array to be generated
                    # (i.e. the number num_nulls itself):
                    
                    try:
                        dummy_values = (rand(num_nulls) - 2) * (scaling_factor) * (col_range) + (col.min())
                    
                    except:
                        # It may be a timestamp, so we cannot multiply col_range and sum.
                        dummy_values = (rand(num_nulls) - 2) * (scaling_factor) + (col.min())
                    
                    # We can shift the dummy values from 0 and 1 to -2 and -1 by subtracting 2, as in:
                    # (rand(num_nulls) - 2)
                    # By doing this, we make sure that the dummy values are always below or lesser than 
                    # the actual values, as can be observed from the graph.
                    # So, by subtracting 2, we guarantee that the dummy values will be below the maximum 
                    # possible.

                    # Next, scale your dummy values by scaling_factor and multiply them by col_range:
                    #  * (scaling_factor) * (col_range)
                    # Finally add the bias: the minimum observed for that column col.min():
                    # + (col.min())
                    # When we shift the values to the minimum (col.min()), we make sure that the dummy 
                    # values are just below the actual values.

                    # Therefore, the procedure results in dummy values a distance apart from the 
                    # actual values.

                    # Loop through the array of dummy values generated:
                    # Loop through each row of the dataframe:
                    
                    k = 0 # first element from the array of dummy values
                    for i in range (0, len(df_dummy)):

                            # Check if the position is missing:
                            boolean_filter = col_null[i]
                            if (boolean_filter):

                                # Run if it is True.
                                # Fill the position in col_name with the dummy value
                                # at the position k from the array of dummy values.
                                # This array was created with a single element for each
                                # missing value:
                                df_dummy.iloc[i,j] = dummy_values[k]
                                # go to the next element
                                k = k + 1
                    
                except:
                    # It was not possible, because it is neither numeric nor timestamp.
                    # Simply ignore it.
                    pass
                    
            return df_dummy
    
        # We fill the dummy values to 'df_dummy' with the function `fill_dummy_values`. 
        # The graph can be plotted with 'df_dummy.plot()' of 'x=column_to_analyze', 
        # 'y=column_to_compare_with', 'kind="scatter"' and 'alpha=0.5' for transparency. 
        
        # Call the subfunction for filling the dummy values:
        df_dummy = fill_dummy_values(df)
        
        # The object 'nullity' is the sum of the nullities of column_to_analyze and column_to_compare_with. 
        # It is a series of True and False values. 
        # True implies missing, while False implies not missing.
        
        # The nullity can be used to set the color of the data points with 'cmap="rainbow"'. 
        # Thus, we obtain the graph that we require.
        
        # Set the nullity of column_to_analyze and column_to_compare_with:
        nullity = ((df[column_to_analyze].isnull()) | (df[column_to_compare_with].isnull()))
        # For setting different colors to the missing and non-missing values, you can simply add 
        # the nullity, or the sum of null values of both respective columns that you are plotting, 
        # calculated using the .isnull() method. The nullity returns a Series of True or False 
        # (i.e., a boolean filter) where:
        # True - At least one of col1 or col2 is missing.
        # False - Neither of col1 and col2 values are missing.

        if (plot_title is None):
            plot_title = "missingness_of_" + "[" + column_to_analyze + "]" + "_vs_" + "[" + column_to_compare_with + "]"
        
        #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
        fig = plt.figure(figsize = (12, 8))
        
        # Create a scatter plot of column_to_analyze and column_to_compare_with 
        df_dummy.plot(x = column_to_analyze, y = column_to_compare_with, 
                            kind = 'scatter', alpha = 0.5,
                            # Set color to nullity of column_to_analyze and column_to_compare_with
                            # alpha: transparency. alpha = 0.5 = 50% of transparency.
                            c = nullity,
                            # The c argument controls the color of the points in the plot.
                            cmap = 'rainbow',
                            grid = grid,
                            legend = True,
                            title = plot_title)
        
        if (export_png == True):
            # Image will be exported
            
            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""
            
            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "comparison_of_missing_values"
            
            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330
            
            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)
            
            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")
        
        #fig.tight_layout()
        
        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        print("Plot Legend:") 
        print("1 = Missing value")
        print("0 = Non-missing value")
        
        if (show_interpreted_example):
            # Run if it is True. Requires TensorFlow to load. Load the extra library only
            # if necessary:
            from html2image import Html2Image
            from tensorflow.keras.preprocessing.image import img_to_array, load_img
            # img_to_array: convert the image into its numpy array representation
            
            # Download the images to the notebook's workspace:
            
            # Alternatively, use "wget GNU" (cannot use as .py file):
            # Use the command !wget to download web content:
            #example_na1 = !wget --no-check-certificate https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na1.PNG example_na1.png
            #example_na2 = !wget --no-check-certificate https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na2.PNG example_na2.png
            #example_na3 = !wget --no-check-certificate https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na3.PNG example_na3.png
            #example_na4 = !wget --no-check-certificate https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na4.PNG example_na4.png
            # When saving the !wget calls as variables, we silent the verbosity of the Wget GNU.
            # Then, user do not see that a download has been made.
            # To check the help from !wget GNU, type and run a cell with: 
            # ! wget --help
            
            url1 = "https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na1.PNG"
            url2 = "https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na2.PNG"
            url3 = "https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na3.PNG"
            url4 = "https://github.com/marcosoares-92/img_examples_guides/raw/main/example_na4.PNG"
            
            # Create a new folder to store the images, if the folder do not exists:
            new_dir = "tmp"
            os.makedirs(new_dir, exist_ok = True)
            # exist_ok = True creates the directory only if it does not exist.
            
            # Instantiate the class Html2Image:
            html_img = Html2Image()
            # Download the images:
            # pypi.org/project/html2image/
            img1 = html_img.screenshot(url = url1, save_as = "example_na1.PNG", size = (500, 500))
            img2 = html_img.screenshot(url = url2, save_as = "example_na2.PNG", size = (500, 500))
            img3 = html_img.screenshot(url = url3, save_as = "example_na3.PNG", size = (500, 500))
            img4 = html_img.screenshot(url = url4, save_as = "example_na4.PNG", size = (500, 500))
            # If size is omitted, the image is downloaded in the low-resolution default.
            # save_as must be a file name, a path is not accepted.
            # Make the output from the method equals to a variable eliminates its verbosity
            
            # Create the new paths for the images:
            img1_path = os.path.join(new_dir, "example_na1.PNG")
            img2_path = os.path.join(new_dir, "example_na2.PNG")
            img3_path = os.path.join(new_dir, "example_na3.PNG")
            img4_path = os.path.join(new_dir, "example_na4.PNG")
            
            # Move the image files to their new paths:
            # use shutil.move(source, destination) method to move the files:
            # pynative.com/python-move-files
            # docs.python.org/3/library/shutil.html
            shutil.move("example_na1.PNG", img1_path)
            shutil.move("example_na2.PNG", img2_path)
            shutil.move("example_na3.PNG", img3_path)
            shutil.move("example_na4.PNG", img4_path)
            
            # Load the images and save them on variables:
            sample_image1 = load_img(img1_path)
            sample_image2 = load_img(img2_path)
            sample_image3 = load_img(img3_path)
            sample_image4 = load_img(img4_path)
            
            print("\n")
            print("Example of analysis:\n")
            
            print("Consider the following \'diabetes\' dataset, where scatterplot of \'Serum_Insulin\' and \'BMI\' illustrated below shows the non-missing values in purple and the missing values in red.\n")
            
            # Image example 1:
            # show image with plt.imshow function:
            fig = plt.figure(figsize = (12, 8))
            plt.imshow(sample_image1)
            # If the image is black and white, you can color it with a cmap as 
            # fig.set_cmap('hot')
            #set axis off:
            plt.axis('off')
            plt.show()
            print("\n")
            
            print("The red points along the y-axis are the missing values of \'Serum_Insulin\' plotted against their \'BMI\' values.\n")
            # Image example 2:
            # show image with plt.imshow function:
            fig = plt.figure(figsize = (12, 8))
            plt.imshow(sample_image2)
            plt.axis('off')
            plt.show()
            print("\n")
            
            print("Likewise, the points along the x-axis are the missing values of \'BMI\' against their \'Serum_Insulin\' values.\n")
            # show image with plt.imshow function:
            fig = plt.figure(figsize = (12, 8))
            plt.imshow(sample_image3)
            plt.axis('off')
            plt.show()
            print("\n")
            
            print("The bottom-left corner represents the missing values of both \'BMI\' and \'Serum_Insulin\'.\n")
            # Image example 4:
            # show image with plt.imshow function:
            fig = plt.figure(figsize = (12, 8))
            plt.imshow(sample_image4)
            plt.axis('off')
            plt.show()
            print("\n")
            
            print("To interprete this graph, observe that the missing values of \'Serum_Insulin\' are spread throughout the \'BMI\' column.")
            print("Thus, we do not observe any specific correlation between the missingness of \'Serum_Insulin\' and \'BMI\'.\n")
            
            # Finally, before finishing the function, 
            # delete (remove) the files from the notebook's workspace.
            # The os.remove function deletes a file or directory specified.
            os.remove(img1_path)
            os.remove(img2_path)
            os.remove(img3_path)
            os.remove(img4_path)
            
            # Check if the tmp folder is empty:
            size = os.path.getsize(new_dir)
            # os.path.getsize returns the total size in Bytes from a folder or a file.
            
            # Get the list of sub-folders, files or subdirectories (the content) from the folder:
            list_of_contents = os.listdir(new_dir)
            # doc.python.org/3/library/os.html
            # It returns a list of strings representing the paths of each file or directory 
            # in the analyzed folder.
            
            # If the size is 0 and the length of the list_of_contents is also zero (i.e., there is no
            # previous sub-directory created), then remove the directory:
            if ((size == 0) & (len(list_of_contents) == 0)):
                os.rmdir(new_dir)


    def handle_missing_values (df, subset_columns_list = None, drop_missing_val = True, fill_missing_val = False, eliminate_only_completely_empty_rows = False, min_number_of_non_missing_val_for_a_row_to_be_kept = None, value_to_fill = None, fill_method = "fill_with_zeros", interpolation_order = 'linear'):
        
        import numpy as np
        import pandas as pd
        from scipy import stats
        # numpy has no function mode, but scipy's stats module has.
        # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.mode.html?msclkid=ccd9aaf2cb1b11ecb57c6f4b3e03a341
        # Pandas dropna method: remove rows containing missing values.
        # Pandas fillna method: fill missing values.
        # Pandas interpolate method: fill missing values with interpolation:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.dropna.html
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.fillna.html
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.interpolate.html#pandas.DataFrame.interpolate
        
        
        # subset_columns_list = list of columns to look for missing values. Only missing values
        # in these columns will be considered for deciding which columns to remove.
        # Declare it as a list of strings inside quotes containing the columns' names to look at,
        # even if this list contains a single element. e.g. subset_columns_list = ['column1']
        # will check only 'column1'; whereas subset_columns_list = ['col1', 'col2', 'col3'] will
        # chek the columns named as 'col1', 'col2', and 'col3'.
        # ATTENTION: Subsets are considered only for dropping missing values, not for filling.
        
        # drop_missing_val = True to eliminate the rows containing missing values.
        
        # fill_missing_val = False. Set this to True to activate the mode for filling the missing
        # values.
        
        # eliminate_only_completely_empty_rows = False - This parameter shows effect only when
        # drop_missing_val = True. If you set eliminate_only_completely_empty_rows = True, then
        # only the rows where all the columns are missing will be eliminated.
        # If you define a subset, then only the rows where all the subset columns are missing
        # will be eliminated.
        
        # min_number_of_non_missing_val_for_a_row_to_be_kept = None - 
        # This parameter shows effect only when drop_missing_val = True. 
        # If you set min_number_of_non_missing_val_for_a_row_to_be_kept equals to an integer value,
        # then only the rows where at least this integer number of non-missing values will be kept
        # after dropping the NAs.
        # e.g. if min_number_of_non_missing_val_for_a_row_to_be_kept = 2, only rows containing at
        # least two columns without missing values will be kept.
        # If you define a subset, then the criterium is applied only to the subset.
        
        # value_to_fill = None - This parameter shows effect only when
        # fill_missing_val = True. Set this parameter as a float value to fill all missing
        # values with this value. e.g. value_to_fill = 0 will fill all missing values with
        # the number 0. You can also pass a function call like 
        # value_to_fill = np.sum(dataset['col1']). In this case, the missing values will be
        # filled with the sum of the series dataset['col1']
        # Alternatively, you can also input a string to fill the missing values. e.g.
        # value_to_fill = 'text' will fill all the missing values with the string "text".
        
        # You can also input a dictionary containing the column(s) to be filled as key(s);
        # and the values to fill as the correspondent values. For instance:
        # value_to_fill = {'col1': 10} will fill only 'col1' with value 10.
        # value_to_fill = {'col1': 0, 'col2': 'text'} will fill 'col1' with zeros; and will
        # fill 'col2' with the value 'text'
        
        # fill_method = "fill_with_zeros". - This parameter shows effect only 
        # when fill_missing_val = True.
        # Alternatively: fill_method = "fill_with_zeros" - fill all the missing values with 0
        
        # fill_method = "fill_with_value_to_fill" - fill the missing values with the value
        # defined as the parameter value_to_fill
        
        # fill_method = "fill_with_avg_or_mode" - fill the missing values with the average value for 
        # each column, if the column is numeric; or fill with the mode, if the column is categorical.
        # The mode is the most commonly observed value.
        
        # fill_method = "ffill" - Forward (pad) fill: propagate last valid observation forward 
        # to next valid.
        # fill_method = 'bfill' - backfill: use next valid observation to fill gap.
        # fill_method = 'nearest' - 'ffill' or 'bfill', depending if the point is closest to the
        # next or to the previous non-missing value.
        
        # fill_method = "fill_by_interpolating" - fill by interpolating the previous and the 
        # following value. For categorical columns, it fills the
        # missing with the previous value, just as like fill_method = 'ffill'
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.interpolate.html#pandas.DataFrame.interpolate
        
        # interpolation_order: order of the polynomial used for interpolating if fill_method =
        # "fill_by_interpolating". If interpolation_order = None, interpolation_order = 'linear',
        # or interpolation_order = 1, a linear (1st-order polynomial) will be used.
        # If interpolation_order is an integer > 1, then it will represent the polynomial order.
        # e.g. interpolation_order = 2, for a 2nd-order polynomial; interpolation_order = 3 for a
        # 3rd-order, and so on.
        
        # WARNING: if the fillna method is selected (fill_missing_val == True), but no filling
        # methodology is selected, the missing values of the dataset will be filled with 0.
        # The same applies when a non-valid fill methodology is selected.
        # Pandas fillna method does not allow us to fill only a selected subset.
        
        # WARNING: if fill_method == "fill_with_value_to_fill" but value_to_fill is None, the 
        # missing values will be filled with the value 0.
        
        
        # Set a local copy of df to manipulate.
        # The methods used in this function can modify the original object itself. So,
        # here we apply the copy method setting deep = True
        cleaned_df = df.copy(deep = True)
    
        if (subset_columns_list is None):
            # all the columns are considered:
            total_columns = cleaned_df.shape[1]
        
        else:
            # Only the columns in the subset are considered.
            # Total columns is the length of the list of columns to subset:
            total_columns = len(subset_columns_list)
            
        # thresh argument of dropna method: int, optional - Require that many non-NA values.
        # This is the minimum of non-missing values that a row must have in order to be kept:
        THRESHOLD = min_number_of_non_missing_val_for_a_row_to_be_kept
        
        if ((drop_missing_val is None) & (fill_missing_val is None)):
            print("No valid input set for neither \'drop_missing_val\' nor \'fill_missing_val\'. Then, setting \'drop_missing_val\' = True and \'fill_missing_val\' = False.\n")
            drop_missing_val = True
            fill_missing_val = False
        
        elif (drop_missing_val is None):
            # The condition where both were missing was already tested. This one is tested only when the
            # the first if was not run.
            drop_missing_val = False
            fill_missing_val = True
        
        elif (fill_missing_val is None):
            drop_missing_val = True
            fill_missing_val = False
        
        elif ((drop_missing_val == True) & (fill_missing_val == True)):
            print("Both options \'drop_missing_val\' and \'fill_missing_val\' set as True. Then, selecting \'drop_missing_val\', which has preference.\n")
            fill_missing_val = False
        
        elif ((drop_missing_val == False) & (fill_missing_val == False)):
            print("Both options \'drop_missing_val\' and \'fill_missing_val\' set as False. Then, setting \'drop_missing_val\' = True.\n")
            drop_missing_val = True
        
        boolean_filter1 = (drop_missing_val == True)

        boolean_filter2 = (boolean_filter1) & (subset_columns_list is None)
        # These filters are True only if both conditions inside parentheses are True.
        # The operator & is equivalent to 'And' (intersection).
        # The operator | is equivalent to 'Or' (union).
        
        boolean_filter3 = (fill_missing_val == True) & (fill_method is None)
        # boolean_filter3 represents the situation where the fillna method was selected, but
        # no filling method was set.
        
        boolean_filter4 = (value_to_fill is None) & (fill_method == "fill_with_value_to_fill")
        # boolean_filter4 represents the situation where the fillna method will be used and the
        # user selected to fill the missing values with 'value_to_fill', but did not set a value
        # for 'value_to_fill'.
        
        if (boolean_filter1 == True):
            # drop missing values
            
            print("Dropping rows containing missing values, accordingly to the provided parameters.\n")
            
            if (boolean_filter2 == True):
                # no subset to filter
                
                if (eliminate_only_completely_empty_rows == True):
                    #Eliminate only completely empty rows
                    cleaned_df = cleaned_df.dropna(axis = 0, how = "all")
                    # if axis = 1, dropna will eliminate each column containing missing values.
                
                elif (min_number_of_non_missing_val_for_a_row_to_be_kept is not None):
                    # keep only rows containing at least the specified number of non-missing values:
                    cleaned_df = cleaned_df.dropna(axis = 0, thresh = THRESHOLD)
                
                else:
                    #Eliminate all rows containing missing values.
                    #The only parameter is drop_missing_val
                    cleaned_df = cleaned_df.dropna(axis = 0)
            
            else:
                #In this case, there is a subset for applying the Pandas dropna method.
                #Only the coluns in the subset 'subset_columns_list' will be analyzed.
                    
                if (eliminate_only_completely_empty_rows == True):
                    #Eliminate only completely empty rows
                    cleaned_df = cleaned_df.dropna(subset = subset_columns_list, how = "all")
                
                elif (min_number_of_non_missing_val_for_a_row_to_be_kept is not None):
                    # keep only rows containing at least the specified number of non-missing values:
                    cleaned_df = cleaned_df.dropna(subset = subset_columns_list, thresh = THRESHOLD)
                
                else:
                    #Eliminate all rows containing missing values.
                    #The only parameter is drop_missing_val
                    cleaned_df = cleaned_df.dropna(subset = subset_columns_list)
            
            print("Finished dropping of missing values.\n")
        
        else:
            
            print("Filling missing values.\n")
            
            # In this case, the user set a value for the parameter fill_missing_val to fill 
            # the missing data.
            
            # Check if a filling dictionary was passed as value_to_fill:
            if (type(value_to_fill) == dict):
                
                print(f"Applying the filling dictionary. Filling columns {value_to_fill.keys()} with the values {value_to_fill.values()}, respectively.\n")
                cleaned_df = cleaned_df.fillna(value = value_to_fill)
            
            elif (boolean_filter3 == True):
                # If this condition was reached, no filling dictionary was input.
                # fillna method was selected, but no filling method was set.
                # Then, filling with zero.
                print("No filling method defined, so filling missing values with 0.\n")
                cleaned_df = cleaned_df.fillna(0)
            
            elif (boolean_filter4 == True):
                # fill_method == "fill_with_value_to_fill" but value_to_fill is None.
                # Then, filling with zero.
                print("No value input for filling, so filling missing values with 0.\n")
                cleaned_df = cleaned_df.fillna(0)
            
            else:
                # A filling methodology was selected.
                if (fill_method == "fill_with_zeros"):
                    print("Filling missing values with 0.\n")
                    cleaned_df = cleaned_df.fillna(0)
                
                elif (fill_method == "fill_with_value_to_fill"):
                    print(f"Filling missing values with {value_to_fill}.\n")
                    cleaned_df = cleaned_df.fillna(value_to_fill)
                
                elif ((fill_method == "fill_with_avg_or_mode") | (fill_method == "fill_by_interpolating")):
                    
                    # We must separate the dataset into numerical columns and categorical columns
                    # 1. Get dataframe's columns list:
                    df_cols = cleaned_df.columns
                    
                    # 2. start a list for the numeric and a list for the text (categorical) columns:
                    numeric_list = []
                    categorical_list = []
                    # List the possible numeric data types for a Pandas dataframe column:
                    numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
                    
                    # 3. Loop through each column on df_cols, to put it in the correspondent type of column:
                    for column in df_cols:
                        
                        # Check if the column is neither in numeric_list nor in
                        # categorical_list yet:
                        if ((column not in numeric_list) & (column not in categorical_list)):
                            
                            column_data_type = cleaned_df[column].dtype

                            if (column_data_type not in numeric_dtypes):

                                # Append to categorical columns list:
                                categorical_list.append(column)

                            else:
                                # Append to numerical columns list:
                                numeric_list.append(column)
                    
                    
                    # Create variables to map if both are present.
                    is_categorical = 0
                    is_numeric = 0

                    # Create two subsets:
                    if (len(categorical_list) > 0):

                        # Has at least one column:
                        df_categorical = df_copy.copy(deep = True)
                        df_categorical = df_categorical[categorical_list]
                        is_categorical = 1

                    if (len(numeric_list) > 0):

                        df_numeric = df_copy.copy(deep = True)
                        df_numeric = df_numeric[numeric_list]
                        is_numeric = 1
                    
                    # Notice that the variables is_numeric and is_categorical have value 1 only when the subsets
                    # are present.
                    is_cat_num = is_categorical + is_numeric
                    # is_cat_num = 0 if no valid dataset was input.
                    # is_cat_num = 2 if both subsets are present.
            
                    # Now, we have two subsets , one for the categoricals, other
                    # for the numeric. It will avoid trying to fill categorical columns with the
                    # mean values.
                    
                    if (fill_method == "fill_with_avg_or_mode"):
                        
                        # Start a filling dictionary:
                        fill_dict = {}
                        
                        print("Filling missing values with the average values (numeric variables); or with the modes (categorical variables). The mode is the most commonly observed value of the categorical variable.\n")
                        
                        if (is_numeric == 1):
                            
                            for column in numeric_list:
                                
                                # add column as the key, and the mean as the value:
                                fill_dict[column] = df_numeric[column].mean()
                        
                        if (is_categorical == 1):
                            
                            for column in categorical_list:
                                
                                mode_array = stats.mode(df_categorical[column])
                                
                                # The function stats.mode(X) returns an array as: 
                                # ModeResult(mode=array(['a'], dtype='<U1'), count=array([2]))
                                # If we select the first element from this array, stats.mode(X)[0], 
                                # the function will return an array as array(['a'], dtype='<U1'). 
                                # We want the first element from this array stats.mode(X)[0][0], 
                                # which will return a string like 'a':
                                try:
                                    fill_dict[column] = mode_array[0][0]

                                except IndexError:
                                    # This error is generated when trying to access an array storing no values.
                                    # (i.e., with missing values). Since there is no dimension, it is not possible
                                    # to access the [0][0] position. In this case, simply append the np.nan 
                                    # the (missing value):
                                    fill_dict[column] = np.nan
                        
                        # Now, fill_dict contains the mapping of columns (keys) and 
                        # correspondent values for imputation with the method fillna.
                        # It is equivalent to use:
                        # from sklearn.impute import SimpleImputer
                        # mean_imputer = SimpleImputer(strategy='mean')
                        # mode_imputer = SimpleImputer(strategy='most_frequent')
                        # as in the advanced imputation function.
                            
                        # In fillna documentation, we see that the argument 'value' must have a dictionary
                        # with this particular format as input:
                        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.fillna.html#pandas.DataFrame.fillna

                        # This dictionary correlates each column to its average value.

                        #6. Finally, use this dictionary to fill the missing values of each column
                        # with the average value of that column
                        cleaned_df = cleaned_df.fillna(value = fill_dict)
                        # The method will search the column name in fill_dict (key of the dictionary),
                        # and will use the correspondent value (average) to fill the missing values.
                    

                    elif (fill_method == "fill_by_interpolating"):
                        # Pandas interpolate method
                        
                        # Separate the dataframes into a dataframe for filling with interpolation (numeric
                        # variables); and a dataframe for forward filling (categorical variables).
                        
                        # Before subsetting, check if the list is not empty.
                        
                        if (is_numeric == 1):
                        
                            if (type(interpolation_order) == int):
                                # an integer number was input

                                if (interpolation_order > 1):

                                    print(f"Performing interpolation of numeric variables with {interpolation_order}-degree polynomial to fill missing values.\n")
                                    df_numeric = df_numeric.interpolate(method = 'polynomial', order = interpolation_order)

                                else:
                                    # 1st order or invalid order (0 or negative) was used
                                    print("Performing linear interpolation of numeric variables to fill missing values.\n")
                                    df_numeric = df_numeric.interpolate(method = 'linear')

                            else:
                                # 'linear', None or invalid text was input:
                                print("Performing linear interpolation of numeric variables to fill missing values.\n")
                                df_numeric = df_numeric.interpolate(method = 'linear')
                        
                        # Now, we finished the interpolation of the numeric variables. Let's check if
                        # there are categorical variables to forward fill.
                        if (is_categorical == 1):
                            
                            # Now, fill missing values by forward filling:
                            print("Using forward filling to fill missing values of the categorical variables.\n")
                            df_categorical = df_categorical.fillna(method = "ffill")
                        
                        # Now, let's check if there are both a numeric_subset and a text_subset to merge
                        
                        if (is_cat_num == 2):
                            # Both subsets are present.
                            # Concatenate the dataframes in the columns axis (append columns):
                            cleaned_df = pd.concat([df_numeric, df_categorical], axis = 1, join = "inner")

                        elif (is_categorical == 1):
                            # There is only the categorical subset:
                            cleaned_df = df_categorical

                        elif (is_numeric == 1):
                            # There is only the numeric subset:
                            cleaned_df = df_numeric
                        
                        else:
                            print("No valid dataset provided, so returning the input dataset itself.\n")
                
                elif ((fill_method == "ffill") | (fill_method == "bfill")):
                    # use forward or backfill
                    cleaned_df = cleaned_df.fillna(method = fill_method)
                
                elif (fill_method == "nearest"):
                    # nearest: applies the 'bfill' or 'ffill', depending if the point
                    # is closes to the previous or to the next non-missing value.
                    # It is a Pandas dataframe interpolation method, not a fillna one.
                    cleaned_df = cleaned_df.interpolate(method = 'nearest')
                
                else:
                    print("No valid filling methodology was selected. Then, filling missing values with 0.\n")
                    cleaned_df = cleaned_df.fillna(0)
            
            
        #Reset index before returning the cleaned dataframe:
        cleaned_df = cleaned_df.reset_index(drop = True)
        
        
        print(f"Number of rows of the dataframe before cleaning = {df.shape[0]} rows.")
        print(f"Number of rows of the dataframe after cleaning = {cleaned_df.shape[0]} rows.")
        print(f"Percentual variation of the number of rows = {(df.shape[0] - cleaned_df.shape[0])/(df.shape[0]) * 100} %\n")
        print("Check the 10 first rows of the cleaned dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(cleaned_df.head(10))
                
        except: # regular mode
            print(cleaned_df.head(10))
        
        return cleaned_df


    def adv_imputation_missing_values (df, column_to_fill, timestamp_tag_column = None, test_value_to_fill = None, show_imputation_comparison_plots = True):
        
        # Check DataCamp course Dealing with Missing Data in Python
        # https://app.datacamp.com/learn/courses/dealing-with-missing-data-in-python
        
        # This function handles only one column by call, whereas handle_missing_values can process the whole
        # dataframe at once.
        # The strategies used for handling missing values is different here. You can use the function to
        # process data that does not come from time series, but only plot the graphs for time series data.
        
        # This function is more indicated for dealing with missing values on time series data than handle_missing_values.
        # This function will search for the best imputer for a given column.
        # It can process both numerical and categorical columns.
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from scipy.stats import linregress
        from sklearn.impute import SimpleImputer
        from sklearn.preprocessing import OrdinalEncoder
        from fancyimpute import KNN, IterativeImputer
        
        # column_to_fill: string (in quotes) indicating the column with missing values to fill.
        # e.g. if column_to_fill = 'col1', imputations will be performed on column 'col1'.
        
        # timestamp_tag_column = None. string containing the name of the column with the timestamp. 
        # If timestamp_tag_column is None, the index will be used for testing different imputations.
        # be the time series reference. declare as a string under quotes. This is the column from 
        # which we will extract the timestamps or values with temporal information. e.g.
        # timestamp_tag_column = 'timestamp' will consider the column 'timestamp' a time column.
        
        # test_value_to_fill: the function will test the imputation of a constant. Specify this constant here
        # or the tested constant will be zero. e.g. test_value_to_fill = None will test the imputation of 0.
        # test_value_to_fill = 10 will test the imputation of value zero.
        
        # show_imputation_comparison_plots = True. Keep it True to plot the scatter plot comparison
        # between imputed and original values, as well as the Kernel density estimate (KDE) plot.
        # Alternatively, set show_imputation_comparison_plots = False to omit the plots.
        
        # The following imputation techniques will be tested, and the best one will be automatically
        # selected: mean_imputer, median_imputer, mode_imputer, constant_imputer, linear_interpolation,
        # quadratic_interpolation, cubic_interpolation, nearest_interpolation, bfill_imputation,
        # ffill_imputation, knn_imputer, mice_imputer (MICE = Multiple Imputations by Chained Equations).
        
        # MICE: Performs multiple regressions over random samples of the data; 
        # Takes the average of multiple regression values; and imputes the missing feature value for the 
        # data point.
        # KNN (K-Nearest Neighbor): Selects K nearest or similar data points using all the 
        # non-missing features. It takes the average of the selected data points to fill in the missing 
        # feature.
        # These are Machine Learning techniques to impute missing values.
        # KNN finds most similar points for imputing.
        # MICE performs multiple regression for imputing. MICE is a very robust model for imputation.
        
        
        # Set a local copy of df to manipulate.
        # The methods used in this function can modify the original object itself. So,
        # here we apply the copy method setting deep = True
        cleaned_df = df.copy(deep = True)
        
        subset_columns_list = [column_to_fill] # only the column indicated.
        total_columns = 1 # keep the homogeneity with the previous function
        
        # Get the list of columns of the dataframe:
        df_cols = list(cleaned_df.columns)
        # Get the index j of the column_to_fill:
        j = df_cols.index(column_to_fill)
        print(f"Filling missing values on column {column_to_fill}. This is the column with index {j} in the original dataframe.\n")

        # Firstly, let's process the timestamp column and save it as x. 
        # That is because datetime objects cannot be directly applied to linear regressions and
        # numeric procedure. We must firstly convert it to an integer scale capable of preserving
        # the distance relationships.
    
        # Check if there is a timestamp_tag_column. If not, make the index the timestamp:
        if (timestamp_tag_column is None):
            
            timestamp_tag_column = column_to_fill + "_index"
            
            # Create the x array
            x = np.array(cleaned_df.index)
            
        else:
            # Run only if there was a timestamp column originally.
            # sort this dataframe by timestamp_tag_column and column_to_fill:
            cleaned_df = cleaned_df.sort_values(by = [timestamp_tag_column, column_to_fill], ascending = [True, True])
            # restart index:
            cleaned_df = cleaned_df.reset_index(drop = True)
            
            # If timestamp_tag_column is an object, the user may be trying to pass a date as x. 
            # So, let's try to convert it to datetime:
            if ((cleaned_df[timestamp_tag_column].dtype == 'O') | (cleaned_df[timestamp_tag_column].dtype == 'object')):

                try:
                    cleaned_df[timestamp_tag_column] = (cleaned_df[timestamp_tag_column]).astype('datetime64[ns]')
                            
                except:
                    # Simply ignore it
                    pass
            
            ts_array = np.array(cleaned_df[timestamp_tag_column])
            
            # Check if the elements from array x are np.datetime64 objects. Pick the first
            # element to check:
            if (type(ts_array[0]) == np.datetime64):
                # In this case, performing the linear regression directly in X will
                # return an error. We must associate a sequential number to each time.
                # to keep the distance between these integers the same as in the original sequence
                # let's define a difference of 1 ns as 1. The 1st timestamp will be zero, and the
                # addition of 1 ns will be an addition of 1 unit. So a timestamp recorded 10 ns
                # after the time zero will have value 10. At the end, we divide every element by
                # 10**9, to obtain the correspondent distance in seconds.
                    
                # start a list for the associated integer timescale. Put the number zero,
                # associated to the first timestamp:
                int_timescale = [0]
                    
                # loop through each element of the array x, starting from index 1:
                for i in range(1, len(ts_array)):
                        
                    # calculate the timedelta between x[i] and x[i-1]:
                    # The delta method from the Timedelta class converts the timedelta to
                    # nanoseconds, guaranteeing the internal compatibility:
                    timedelta = pd.Timedelta(ts_array[i] - ts_array[(i-1)]).delta
                        
                    # Sum this timedelta (integer number of nanoseconds) to the
                    # previous element from int_timescale, and append the result to the list:
                    int_timescale.append((timedelta + int_timescale[(i-1)]))
                    
                # Now convert the new scale (that preserves the distance between timestamps)
                # to NumPy array:
                int_timescale = np.array(int_timescale)
                
                # Divide by 10**9 to obtain the distances in seconds, reducing the order of
                # magnitude of the integer numbers (the division is allowed for arrays).
                # make it the timestamp array ts_array itself:
                ts_array = int_timescale / (10**9)
                # Now, reduce again the order of magnitude through division by (60*60)
                # It will obtain the ts_array in hour:
                ts_array = int_timescale / (60*60)
                
            # make x the ts_array itself:
            x = ts_array
        
        column_data_type = cleaned_df[column_to_fill].dtype
        
        # Pre-process the column if it is categorical
        if ((column_data_type == 'O') | (column_data_type == 'object')):
            
            # Ordinal encoding: let's associate integer sequential numbers to the categorical column
            # to apply the advanced encoding techniques. Even though the one-hot encoding could perform
            # the same task and would, in fact, better, since there may be no ordering relation, the
            # ordinal encoding is simpler and more suitable for this particular task:
            
            # Create Ordinal encoder
            ord_enc = OrdinalEncoder()
            
            # Select non-null values of the column in the dataframe:
            series_on_df = cleaned_df[column_to_fill]
            
            # Reshape series_on_df to shape (-1, 1)
            reshaped_vals = series_on_df.values.reshape(-1, 1)
            
            # Fit the ordinal encoder to the reshaped column_to_fill values:
            encoded_vals = ord_enc.fit_transform(reshaped_vals)
            
            # Finally, store the values to non-null values of the column in dataframe
            cleaned_df.iloc[:,j] = encoded_vals

            # Max and minimum of the encoded range
            max_encoded = max(encoded_vals)
            min_encoded = min(encoded_vals)


        # Start a list of imputations:
        list_of_imputations = []
        
        subset_from_cleaned_df = cleaned_df.copy(deep = True)
        subset_from_cleaned_df = subset_from_cleaned_df[subset_columns_list]

        mean_imputer = SimpleImputer(strategy = 'mean')
        list_of_imputations.append('mean_imputer')
        
        # Now, apply the fit_transform method from the imputer to fit it to the indicated column:
        mean_imputer.fit(subset_from_cleaned_df)
        # If you wanted to obtain constants for all columns, you should not specify a subset:
        # imputer.fit_transform(cleaned_df)
            
        # create a column on the dataframe as 'mean_imputer':
        cleaned_df['mean_imputer'] = mean_imputer.transform(subset_from_cleaned_df)
            
        # Create the median imputer:
        median_imputer = SimpleImputer(strategy = 'median')
        list_of_imputations.append('median_imputer')
        median_imputer.fit(subset_from_cleaned_df)
        cleaned_df['median_imputer'] = median_imputer.transform(subset_from_cleaned_df)
        
        # Create the mode imputer:
        mode_imputer = SimpleImputer(strategy = 'most_frequent')
        list_of_imputations.append('mode_imputer')
        mode_imputer.fit(subset_from_cleaned_df)
        cleaned_df['mode_imputer'] = mode_imputer.transform(subset_from_cleaned_df)
        
        # Create the constant value imputer:
        if (test_value_to_fill is None):
            test_value_to_fill = 0
        
        constant_imputer = SimpleImputer(strategy = 'constant', fill_value = test_value_to_fill)
        list_of_imputations.append('constant_imputer')
        constant_imputer.fit(subset_from_cleaned_df)
        cleaned_df['constant_imputer'] = constant_imputer.transform(subset_from_cleaned_df)
        
        # Make the linear interpolation imputation:
        linear_interpolation_df = cleaned_df[subset_columns_list].copy(deep = True)
        linear_interpolation_df = linear_interpolation_df.interpolate(method = 'linear')
        cleaned_df['linear_interpolation'] = linear_interpolation_df[column_to_fill]
        list_of_imputations.append('linear_interpolation')
            
        # Interpolate 2-nd degree polynomial:
        quadratic_interpolation_df = cleaned_df[subset_columns_list].copy(deep = True)
        quadratic_interpolation_df = quadratic_interpolation_df.interpolate(method = 'polynomial', order = 2)
        cleaned_df['quadratic_interpolation'] = quadratic_interpolation_df[column_to_fill]
        list_of_imputations.append('quadratic_interpolation')
            
        # Interpolate 3-rd degree polynomial:
        cubic_interpolation_df = cleaned_df[subset_columns_list].copy(deep = True)
        cubic_interpolation_df = cubic_interpolation_df.interpolate(method = 'polynomial', order = 3)
        cleaned_df['cubic_interpolation'] = cubic_interpolation_df[column_to_fill]
        list_of_imputations.append('cubic_interpolation')
        
        # Nearest interpolation
        # Similar to bfill and ffill, but uses the nearest
        nearest_interpolation_df = cleaned_df[subset_columns_list].copy(deep = True)
        nearest_interpolation_df = nearest_interpolation_df.interpolate(method = 'nearest')
        cleaned_df['nearest_interpolation'] = nearest_interpolation_df[column_to_fill]
        list_of_imputations.append('nearest_interpolation')
        
        # bfill and ffill:
        bfill_df = cleaned_df[subset_columns_list].copy(deep = True)
        ffill_df = cleaned_df[subset_columns_list].copy(deep = True)
        
        bfill_df = bfill_df.fillna(method = 'bfill')
        cleaned_df['bfill_imputation'] = bfill_df[column_to_fill]
        list_of_imputations.append('bfill_imputation')
        
        ffill_df = ffill_df.fillna(method = 'ffill')
        cleaned_df['ffill_imputation'] = ffill_df[column_to_fill]
        list_of_imputations.append('ffill_imputation')
        
        
        # Now, we can go to the advanced machine learning techniques:
        
        # KNN Imputer:
        # Initialize KNN
        knn_imputer = KNN()
        list_of_imputations.append('knn_imputer')
        cleaned_df['knn_imputer'] = knn_imputer.fit_transform(subset_from_cleaned_df)
        
        # Initialize IterativeImputer
        mice_imputer = IterativeImputer()
        list_of_imputations.append('mice_imputer')
        cleaned_df['mice_imputer'] = mice_imputer.fit_transform(subset_from_cleaned_df)
        
        # Now, let's create linear regressions for compare the performance of different
        # imputation strategies.
        # Firstly, start a dictionary to store
        
        imputation_performance_dict = {}

        # Now, loop through each imputation and calculate the adjusted R²:
        for imputation in list_of_imputations:
            
            y = cleaned_df[imputation]
            
            # fit the linear regression
            slope, intercept, r, p, se = linregress(x, y)
            
            # Get the adjusted R² and add it as the key imputation of the dictionary:
            imputation_performance_dict[imputation] = r**2
        
        # Select best R-squared
        best_imputation = max(imputation_performance_dict, key = imputation_performance_dict.get)
        print(f"The best imputation strategy for the column {column_to_fill} is {best_imputation}.\n")
        
        
        if (show_imputation_comparison_plots & ((column_data_type != 'O') & (column_data_type != 'object'))):
            
            # Firstly, converts the values obtained to closest integer (since we
            # encoded the categorical values as integers, we cannot reconvert
            # decimals):)): # run if it is True
        
            print("Check the Kernel density estimate (KDE) plot for the different imputations.\n")
            labels_list = ['baseline\ncomplete_case']
            y = cleaned_df[column_to_fill]
            X = cleaned_df[timestamp_tag_column] # not the converted scale

            fig = plt.figure(figsize = (12, 8))
            ax = fig.add_subplot()
            
            # Plot graphs of imputed DataFrames and the complete case
            y.plot(kind = 'kde', c = 'red', linewidth = 3)

            for imputation in list_of_imputations:
                
                labels_list.append(imputation)
                y = cleaned_df[imputation]
                y.plot(kind = 'kde')
            
            #ROTATE X AXIS IN XX DEGREES
            plt.xticks(rotation = 0)
            # XX = 0 DEGREES x_axis (Default)
            #ROTATE Y AXIS IN XX DEGREES:
            plt.yticks(rotation = 0)
            # XX = 0 DEGREES y_axis (Default)

            ax.set_title("Kernel_density_estimate_plot_for_each_imputation")
            ax.set_xlabel(column_to_fill)
            ax.set_ylabel("density")

            ax.grid(True) # show grid or not
            ax.legend(loc = 'upper left')
            # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
            # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
            # https://www.statology.org/matplotlib-legend-position/
            plt.show()
            
            print("\n")
            print(f"Now, check the original time series compared with the values obtained through {best_imputation}:\n")
            
            fig = plt.figure(figsize = (12, 8))
            ax = fig.add_subplot()
            
            # Plot the imputed DataFrame in red dotted style
            selected_imputation = cleaned_df[best_imputation]
            ax.plot(X, selected_imputation, color = 'red', marker = 'o', linestyle = 'dotted', label = best_imputation)
            
            # Plot the original DataFrame with title
            # Put a degree of transparency (35%) to highlight the imputation.
            ax.plot(X, y, color = 'darkblue', alpha = 0.65, linestyle = '-', marker = '', label = (column_to_fill + "_original"))
            
            plt.xticks(rotation = 70)
            plt.yticks(rotation = 0)
            ax.set_title(column_to_fill + "_original_vs_imputations")
            ax.set_xlabel(timestamp_tag_column)
            ax.set_ylabel(column_to_fill)

            ax.grid(True) # show grid or not
            ax.legend(loc = 'upper left')
            # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
            # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
            # https://www.statology.org/matplotlib-legend-position/
            plt.show()
            print("\n")
        
                    
        print(f"Returning a dataframe where {best_imputation} strategy was used for filling missing values in {column_to_fill} column.\n")
        
        if (best_imputation == 'mice_imputer'):
            print("MICE = Multiple Imputations by Chained Equations")
            print("MICE: Performs multiple regressions over random samples of the data.")
            print("It takes the average of multiple regression values and imputes the missing feature value for the data point.")
            print("It is a Machine Learning technique to impute missing values.")
            print("MICE performs multiple regression for imputing and is a very robust model for imputation.\n")
        
        elif (best_imputation == 'knn_imputer'):
            print("KNN = K-Nearest Neighbor")
            print("KNN selects K nearest or similar data points using all the non-missing features.")
            print("It takes the average of the selected data points to fill in the missing feature.")
            print("It is a Machine Learning technique to impute missing values.")
            print("KNN finds most similar points for imputing.\n")
        
        # Make all rows from the column j equals to the selected imputer:
        cleaned_df.iloc[:, j] = cleaned_df[best_imputation]
        # If you wanted to make all rows from all columns equal to the imputer, you should declare:
        # cleaned_df.iloc[:, :] = imputer
        
        # Drop all the columns created for storing different imputers:
        # These columns were saved in the list list_of_imputations.
        # Notice that the selected imputations were saved in the original column.
        cleaned_df = cleaned_df.drop(columns = list_of_imputations)
        
        # Finally, let's reverse the ordinal encoding used in the beginning of the code to process object
        # columns:
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        if (column_data_type not in numeric_dtypes):
            
            # Firstly, converts the values obtained to closest integer (since we
            # encoded the categorical values as integers, we cannot reconvert
            # decimals):
            
            cleaned_df[column_to_fill] = (np.rint(cleaned_df[column_to_fill]))
            
            # If a value is above the max_encoded, make it equals to the maximum.
            # If it is below the minimum, make it equals to the minimum:
            for k in range(0, len(cleaned_df)):

                if (cleaned_df.iloc[k,j] > max_encoded):
                    cleaned_df.iloc[k,j] = max_encoded
                    
                elif (cleaned_df.iloc[k,j] < min_encoded):
                    cleaned_df.iloc[k,j] = min_encoded

            new_series = cleaned_df[column_to_fill]
            # We must use the int function to guarantee that the column_to_fill will store an
            # integer number (we cannot have a fraction of an encoding).
            # The int function guarantees that the variable will be stored as an integer.
            # The numpy.rint(a) function rounds elements of the array to the nearest integer.
            # https://numpy.org/doc/stable/reference/generated/numpy.rint.html
            # For values exactly halfway between rounded decimal values, 
            # NumPy rounds to the nearest even value. 
            # Thus 1.5 and 2.5 round to 2.0; -0.5 and 0.5 round to 0.0; etc.
            
            # Reshape series_not_null to shape (-1, 1)
            reshaped_vals = new_series.values.reshape(-1, 1)
            
            # Perform inverse transform of the ordinally encoded columns
            cleaned_df[column_to_fill] = ord_enc.inverse_transform(reshaped_vals)


        print("Check the 10 first rows from the cleaned dataframe:\n")
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(cleaned_df.head(10))
                
        except: # regular mode
            print(cleaned_df.head(10))
        
        return cleaned_df


    def correlation_plot (df, show_masked_plot = True, responses_to_return_corr = None, set_returned_limit = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import seaborn as sns
        
        #show_masked_plot = True - keep as True if you want to see a cleaned version of the plot
        # where a mask is applied.
        
        #responses_to_return_corr - keep as None to return the full correlation tensor.
        # If you want to display the correlations for a particular group of features, input them
        # as a list, even if this list contains a single element. Examples:
        # responses_to_return_corr = ['response1'] for a single response
        # responses_to_return_corr = ['response1', 'response2', 'response3'] for multiple
        # responses. Notice that 'response1',... should be substituted by the name ('string')
        # of a column of the dataset that represents a response variable.
        # WARNING: The returned coefficients will be ordered according to the order of the list
        # of responses. i.e., they will be firstly ordered based on 'response1'
        
        # set_returned_limit = None - This variable will only present effects in case you have
        # provided a response feature to be returned. In this case, keep set_returned_limit = None
        # to return all of the correlation coefficients; or, alternatively, 
        # provide an integer number to limit the total of coefficients returned. 
        # e.g. if set_returned_limit = 10, only the ten highest coefficients will be returned. 
        
        # set a local copy of the dataset to perform the calculations:
        DATASET = df.copy(deep = True)
        
        correlation_matrix = DATASET.corr(method = 'pearson')
        
        if (show_masked_plot == False):
            #Show standard plot
            
            plt.figure(figsize = (12, 8))
            sns.heatmap((correlation_matrix)**2, annot = True, fmt = ".2f")
            
            if (export_png == True):
                # Image will be exported
                import os

                #check if the user defined a directory path. If not, set as the default root path:
                if (directory_to_save is None):
                    #set as the default
                    directory_to_save = ""

                #check if the user defined a file name. If not, set as the default name for this
                # function.
                if (file_name is None):
                    #set as the default
                    file_name = "correlation_plot"

                #check if the user defined an image resolution. If not, set as the default 110 dpi
                # resolution.
                if (png_resolution_dpi is None):
                    #set as 330 dpi
                    png_resolution_dpi = 330

                #Get the new_file_path
                new_file_path = os.path.join(directory_to_save, file_name)

                #Export the file to this new path:
                # The extension will be automatically added by the savefig method:
                plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                #quality could be set from 1 to 100, where 100 is the best quality
                #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                #transparent = True or False
                # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

            plt.show()

        #Oncee the pandas method .corr() calculates R, we raised it to the second power 
        # to obtain R². R² goes from zero to 1, where 1 represents the perfect correlation.
        
        else:
            
            # Show masked (cleaner) plot instead of the standard one
            # Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
            plt.figure(figsize = (12, 8))
            # Mask for the upper triangle
            mask = np.zeros_like((correlation_matrix)**2)

            mask[np.triu_indices_from(mask)] = True

            # Generate a custom diverging colormap
            cmap = sns.diverging_palette(220, 10, as_cmap = True)

            # Heatmap with mask and correct aspect ratio
            sns.heatmap(((correlation_matrix)**2), mask = mask, cmap = cmap, center = 0,
                        linewidths = .5)
            
            if (export_png == True):
                # Image will be exported
                import os

                #check if the user defined a directory path. If not, set as the default root path:
                if (directory_to_save is None):
                    #set as the default
                    directory_to_save = ""

                #check if the user defined a file name. If not, set as the default name for this
                # function.
                if (file_name is None):
                    #set as the default
                    file_name = "correlation_plot"

                #check if the user defined an image resolution. If not, set as the default 110 dpi
                # resolution.
                if (png_resolution_dpi is None):
                    #set as 330 dpi
                    png_resolution_dpi = 330

                #Get the new_file_path
                new_file_path = os.path.join(directory_to_save, file_name)

                #Export the file to this new path:
                # The extension will be automatically added by the savefig method:
                plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                #quality could be set from 1 to 100, where 100 is the best quality
                #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                #transparent = True or False
                # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

            plt.show()

            #Again, the method dataset.corr() calculates R within the variables of dataset.
            #To calculate R², we simply raise it to the second power: (dataset.corr()**2)
        
        #Sort the values of correlation_matrix in Descending order:
        
        if (responses_to_return_corr is not None):
            
            #Select only the desired responses, by passing the list responses_to_return_corr
            # as parameter for column filtering:
            correlation_matrix = correlation_matrix[responses_to_return_corr]
            
            #Now sort the values according to the responses, by passing the list
            # responses_to_return_corr as the parameter
            correlation_matrix = correlation_matrix.sort_values(by = responses_to_return_corr, ascending = False)
            
            # If a limit of coefficients was determined, apply it:
            if (set_returned_limit is not None):
                    
                    correlation_matrix = correlation_matrix.head(set_returned_limit)
                    #Pandas .head(X) method returns the first X rows of the dataframe.
                    # Here, it returns the defined limit of coefficients, set_returned_limit.
                    # The default .head() is X = 5.
        
        print("ATTENTION: The correlation plots show the linear correlations R², which go from 0 (none correlation) to 1 (perfect correlation). Obviously, the main diagonal always shows R² = 1, since the data is perfectly correlated to itself.\n")
        print("The returned correlation matrix, on the other hand, presents the linear coefficients of correlation R, not R². R values go from -1 (perfect negative correlation) to 1 (perfect positive correlation).\n")
        print("None of these coefficients take non-linear relations and the presence of a multiple linear correlation in account. For these cases, it is necessary to calculate R² adjusted, which takes in account the presence of multiple preditors and non-linearities.\n")
        
        print("Correlation matrix - numeric results:\n")
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(correlation_matrix)
                
        except: # regular mode
            print(correlation_matrix)
        
        return correlation_matrix


    def bar_chart (df, categorical_var_name, response_var_name, aggregate_function = 'sum', add_suffix_to_aggregated_col = True, suffix = None, calculate_and_plot_cumulative_percent = True, orientation = 'vertical', limit_of_plotted_categories = None, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, x_axis_rotation = 70, y_axis_rotation = 0, grid = True, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):

        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from scipy import stats
        
        # df: dataframe being analyzed
        
        # categorical_var_name: string (inside quotes) containing the name 
        # of the column to be analyzed. e.g. 
        # categorical_var_name = "column1"
        
        # response_var_name: string (inside quotes) containing the name 
        # of the column that stores the response correspondent to the
        # categories. e.g. response_var_name = "response_feature" 
        
        # aggregate_function = 'sum': String defining the aggregation 
        # method that will be applied. Possible values:
        # 'median', 'mean', 'mode', 'sum', 'min', 'max', 'variance', 'count',
        # 'standard_deviation', '10_percent_quantile', '20_percent_quantile',
        # '25_percent_quantile', '30_percent_quantile', '40_percent_quantile',
        # '50_percent_quantile', '60_percent_quantile', '70_percent_quantile',
        # '75_percent_quantile', '80_percent_quantile', '90_percent_quantile',
        # '95_percent_quantile', 'kurtosis', 'skew', 'interquartile_range',
        # 'mean_standard_error', 'entropy'
        # To use another aggregate function, you can use the .agg method, passing 
        # the aggregate as argument, such as in:
        # .agg(scipy.stats.mode), 
        # where the argument is a Scipy aggregate function.
        # If None or an invalid function is input, 'sum' will be used.
        
        # add_suffix_to_aggregated_col = True will add a suffix to the
        # aggregated column. e.g. 'responseVar_mean'. If add_suffix_to_aggregated_col 
        # = False, the aggregated column will have the original column name.
        
        # suffix = None. Keep it None if no suffix should be added, or if
        # the name of the aggregate function should be used as suffix, after
        # "_". Alternatively, set it as a string. As recommendation, put the
        # "_" sign in the beginning of this string to separate the suffix from
        # the original column name. e.g. if the response variable is 'Y' and
        # suffix = '_agg', the new aggregated column will be named as 'Y_agg'
        
        # calculate_and_plot_cumulative_percent = True to calculate and plot
        # the line of cumulative percent, or 
        # calculate_and_plot_cumulative_percent = False to omit it.
        # This feature is only shown when aggregate_function = 'sum', 'median',
        # 'mean', or 'mode'. So, it will be automatically set as False if 
        # another aggregate is selected.
        
        # orientation = 'vertical' is the standard, and plots vertical bars
        # (perpendicular to the X axis). In this case, the categories are shown
        # in the X axis, and the correspondent responses are in Y axis.
        # Alternatively, orientation = 'horizontal' results in horizontal bars.
        # In this case, categories are in Y axis, and responses in X axis.
        # If None or invalid values are provided, orientation is set as 'vertical'.
        
        # Note: to obtain a Pareto chart, keep aggregate_function = 'sum',
        # plot_cumulative_percent = True, and orientation = 'vertical'.
        
        # limit_of_plotted_categories: integer value that represents
        # the maximum of categories that will be plot. Keep it None to plot
        # all categories. Alternatively, set an integer value. e.g.: if
        # limit_of_plotted_categories = 4, but there are more categories,
        # the dataset will be sorted in descending order and: 1) The remaining
        # categories will be sum in a new category named 'others' if the
        # aggregate function is 'sum'; 2) Or the other categories will be simply
        # omitted from the plot, for other aggregate functions. Notice that
        # it limits only the variables in the plot: all of them will be
        # returned in the dataframe.
        # Use this parameter to obtain a cleaner plot. Notice that the remaining
        # columns will be aggregated as 'others' even if there is a single column
        # beyond the limit.
        
        
        # Create a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        # Before calling the method, we must guarantee that the variables may be
        # used for that aggregate. Some aggregations are permitted only for numeric variables, so calling
        # the methods before selecting the variables may raise warnings or errors.
        
        
        list_of_aggregates = ['median', 'mean', 'mode', 'sum', 'min', 'max', 'variance', 'count',
                            'standard_deviation', '10_percent_quantile', '20_percent_quantile', 
                            '25_percent_quantile', '30_percent_quantile', '40_percent_quantile', 
                            '50_percent_quantile', '60_percent_quantile', '70_percent_quantile', 
                            '75_percent_quantile', '80_percent_quantile', '90_percent_quantile', 
                            '95_percent_quantile', 'kurtosis', 'skew', 'interquartile_range', 
                            'mean_standard_error', 'entropy']
        
        list_of_numeric_aggregates = ['median', 'mean', 'sum', 'min', 'max', 'variance',
                                    'standard_deviation', '10_percent_quantile', '20_percent_quantile', 
                                    '25_percent_quantile', '30_percent_quantile', '40_percent_quantile', 
                                    '50_percent_quantile', '60_percent_quantile', '70_percent_quantile', 
                                    '75_percent_quantile', '80_percent_quantile', '90_percent_quantile',
                                    '95_percent_quantile', 'kurtosis', 'skew', 'interquartile_range', 
                                    'mean_standard_error']
        
        # Check if an invalid or no aggregation function was selected:
        if ((aggregate_function not in (list_of_aggregates)) | (aggregate_function is None)):
            
            aggregate_function = 'sum'
            print("Invalid or no aggregation function input, so using the default \'sum\'.\n")
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        # Check if a numeric aggregate was selected:
        if (aggregate_function in list_of_numeric_aggregates):
            
            column_data_type = DATASET[response_var_name].dtype
            
            if (column_data_type not in numeric_dtypes):
                
                    # If the Pandas series was defined as an object, it means it is categorical
                    # (string, date, etc).
                    print("Numeric aggregate selected, but categorical variable indicated as response variable.")
                    print("Setting aggregate_function = \'mode\', to make aggregate compatible with data type.\n")
                    
                    aggregate_function = 'mode'
        
        else: # categorical aggregate function
            
            column_data_type = DATASET[response_var_name].dtype
            
            if ((column_data_type in numeric_dtypes) & (aggregate_function != 'count')):
                    # count is the only aggregate for categorical that can be used for numerical variables as well.
                    
                    print("Categorical aggregate selected, but numeric variable indicated as response variable.")
                    print("Setting aggregate_function = \'sum\', to make aggregate compatible with data type.\n")
                    
                    aggregate_function = 'sum'
        
        # Before grouping, let's remove the missing values, avoiding the raising of TypeError.
        # Pandas deprecated the automatic dropna with aggregation:
        DATASET = DATASET.dropna(axis = 0)
        
        # Convert categorical_var_name to Pandas 'category' type. If the variable is represented by
        # a number, the dataframe will be grouped in terms of an aggregation of the variable, instead
        # of as a category. It will prevents this to happen:
        DATASET[categorical_var_name] = DATASET[categorical_var_name].astype("category")    
        
        # If an aggregate function different from 'sum', 'mean', 'median' or 'mode' 
        # is used with plot_cumulative_percent = True, 
        # set plot_cumulative_percent = False:
        # (check if aggregate function is not in the list of allowed values):
        if ((aggregate_function not in ['sum', 'mean', 'median', 'mode', 'count']) & (calculate_and_plot_cumulative_percent == True)):
            
            calculate_and_plot_cumulative_percent = False
            print("The cumulative percent is only calculated when aggregate_function = \'sum\', \'mean\', \'median\', \'mode\', or \'count\'. So, plot_cumulative_percent was set as False.")
        
        # Guarantee that the columns from the aggregated dataset have the correct names
        
        # Groupby according to the selection.
        # Here, there is a great gain of performance in not using a dictionary of methods:
        # If using a dictionary of methods, Pandas would calculate the results for each one of the methods.
        
        # Pandas groupby method documentation:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.groupby.html?msclkid=7b3531a6cff211ec9086f4edaddb94ba
        # argument as_index = False: prevents the grouper variable to be set as index of the new dataframe.
        # (default: as_index = True);
        # dropna = False: do not removes the missing values (default: dropna = True, used here to avoid
        # compatibility and version issues)
        
        if (aggregate_function == 'median'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].agg('median')

        elif (aggregate_function == 'mean'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].mean()
        
        elif (aggregate_function == 'mode'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].agg(stats.mode)
        
        elif (aggregate_function == 'sum'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].sum()
        
        elif (aggregate_function == 'count'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].count()

        elif (aggregate_function == 'min'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].min()
        
        elif (aggregate_function == 'max'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].max()
        
        elif (aggregate_function == 'variance'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].var()

        elif (aggregate_function == 'standard_deviation'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].std()
        
        elif (aggregate_function == '10_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.10)
        
        elif (aggregate_function == '20_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.20)
        
        elif (aggregate_function == '25_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.25)
        
        elif (aggregate_function == '30_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.30)
        
        elif (aggregate_function == '40_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.40)
        
        elif (aggregate_function == '50_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.50)

        elif (aggregate_function == '60_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.60)
        
        elif (aggregate_function == '70_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.30)

        elif (aggregate_function == '75_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.75)

        elif (aggregate_function == '80_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.80)
        
        elif (aggregate_function == '90_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.90)
        
        elif (aggregate_function == '95_percent_quantile'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].quantile(0.95)

        elif (aggregate_function == 'kurtosis'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].agg(stats.kurtosis)
        
        elif (aggregate_function == 'skew'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].agg(stats.skew)

        elif (aggregate_function == 'interquartile_range'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].agg(stats.iqr)
        
        elif (aggregate_function == 'mean_standard_error'):
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].agg(stats.sem)
        
        else: # entropy
            
            DATASET = DATASET.groupby(by = categorical_var_name, as_index = False, sort = True)[response_var_name].agg(stats.entropy)

        
        # List of columns of the aggregated dataset:
        list_of_columns = list(DATASET.columns) # convert to a list
        
        if (add_suffix_to_aggregated_col == True):
                
            if (suffix is None):
                    
                suffix = "_" + aggregate_function
                
            new_columns = [(str(name) + suffix) for name in list_of_columns]
            # Do not consider the first element, which is the aggregate function with a suffix.
            # Concatenate the correct name with the columns from the second element of the list:
            new_columns = [categorical_var_name] + new_columns[1:]
            # Make it the new columns:
            DATASET.columns = new_columns
            # Update the list of columns:
            list_of_columns = DATASET.columns
        
        if (aggregate_function == 'mode'):
            
            # The columns was saved as a series of Tuples. Each row contains a tuple like:
            # ([calculated_mode], [counting_of_occurrences]). We want only the calculated mode.
            # On the other hand, if we do column[0], we will get the columns first row. So, we have to
            # go through each column, retrieving only the mode:
            
            # Loop through each column:
            for column in list_of_columns:
                
                # Save the series as a list:
                list_of_modes_arrays = list(DATASET[column])
                # Start a list of modes:
                list_of_modes = []
                
                # Loop through each element from the list of arrays:
                for mode_array in list_of_modes_arrays:
                    # mode array is like:
                    # ModeResult(mode=array([calculated_mode]), count=array([counting_of_occurrences]))
                    # To retrieve only the mode, we must access the element [0][0] from this array:
                    try:
                        list_of_modes.append(mode_array[0][0])
                    
                    except IndexError:
                        # This error is generated when trying to access an array storing no values.
                        # (i.e., with missing values). Since there is no dimension, it is not possible
                        # to access the [0][0] position. In this case, simply append the np.nan 
                        # the (missing value):
                        list_of_modes.append(np.nan)
                
                # Make the list of modes the column itself:
                DATASET[column] = list_of_modes
        
                
        # the name of the response variable is now the second element from the list of column:
        response_var_name = list(DATASET.columns)[1]
        # the categorical variable name was not changed.
        
        # Let's sort the dataframe.
        
        # Order the dataframe in descending order by the response.
        # If there are equal responses, order them by category, in
        # ascending order; put the missing values in the first position
        # To pass multiple columns and multiple types of ordering, we use
        # lists. If there was a single column to order by, we would declare
        # it as a string. If only one order of ascending was used, we would
        # declare it as a simple boolean
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.sort_values.html
        
        DATASET = DATASET.sort_values(by = [response_var_name, categorical_var_name], ascending = [False, True], na_position = 'first')
        
        # Now, reset index positions:
        DATASET = DATASET.reset_index(drop = True)
        
        if (aggregate_function == 'count'):
            
            # Here, the column represents the counting, no matter the variable set as response.
            DATASET.columns = [categorical_var_name, 'count_of_entries']
            response_var_name = 'count_of_entries'
        
        # plot_cumulative_percent = True, create a column to store the
        # cumulative percent:
        if (calculate_and_plot_cumulative_percent): 
            # Run the following code if the boolean value is True (implicity)
            # Only calculates cumulative percent in case aggregate is 'sum' or 'mode'
            
            # Create a column series for the cumulative sum:
            cumsum_col = response_var_name + "_cumsum"
            DATASET[cumsum_col] = DATASET[response_var_name].cumsum()
            
            # total sum is the last element from this series
            # (i.e. the element with index len(DATASET) - 1)
            total_sum = DATASET[cumsum_col][(len(DATASET) - 1)]
            
            # Now, create a column for the accumulated percent
            # by dividing cumsum_col by total_sum and multiplying it by
            # 100 (%):
            cum_pct_col = response_var_name + "_cum_pct"
            DATASET[cum_pct_col] = (DATASET[cumsum_col])/(total_sum) * 100
            print(f"Successfully calculated cumulative sum and cumulative percent correspondent to the response variable {response_var_name}.")
        
        print("Successfully aggregated and ordered the dataset to plot. Check the 10 first rows of this returned dataset:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        # Check if the total of plotted categories is limited:
        if not (limit_of_plotted_categories is None):
            
            # Since the value is not None, we have to limit it
            # Check if the limit is lower than or equal to the length of the dataframe.
            # If it is, we simply copy the columns to the series (there is no need of
            # a memory-consuming loop or of applying the head method to a local copy
            # of the dataframe):
            df_length = len(DATASET)
                
            if (df_length <= limit_of_plotted_categories):
                # Simply copy the columns to the graphic series:
                categories = DATASET[categorical_var_name]
                responses = DATASET[response_var_name]
                # If there is a cum_pct column, copy it to a series too:
                if (calculate_and_plot_cumulative_percent):
                    cum_pct = DATASET[cum_pct_col]
            
            else:
                # The limit is lower than the total of categories,
                # so we actually have to limit the size of plotted df:
            
                # If aggregate_function is not 'sum', we simply apply
                # the head method to obtain the first rows (number of
                # rows input as parameter; if no parameter is input, the
                # number of 5 rows is used):
                
                # Limit to the number limit_of_plotted_categories:
                # create another local copy of the dataframe not to
                # modify the returned dataframe object:
                plotted_df = DATASET.copy(deep = True).head(limit_of_plotted_categories)

                # Create the series of elements to plot:
                categories = list(plotted_df[categorical_var_name])
                responses = list(plotted_df[response_var_name])
                # If the cumulative percent was obtained, create the series for it:
                if (calculate_and_plot_cumulative_percent):
                    cum_pct = list(plotted_df[cum_pct_col])
                
                # Start variable to store the aggregates from the others:
                other_responses = 0
                
                # Loop through each row from DATASET:
                for i in range(0, len(DATASET)):
                    
                    # Check if the category is not in categories:
                    category = DATASET[categorical_var_name][i]
                    
                    if (category not in categories):
                        
                        # sum the value in the response variable to other_responses:
                        other_responses = other_responses + DATASET[response_var_name][i]
                
                # Now we finished the sum of the other responses, let's add these elements to
                # the lists:
                categories.append("others")
                responses.append(other_responses)
                # If there is a cumulative percent, append 100% to the list:
                if (calculate_and_plot_cumulative_percent):
                    cum_pct.append(100)
                    # The final cumulative percent must be the total, 100%
                
                else:

                    # Firstly, copy the elements that will be kept to x, y and (possibly) cum_pct
                    # lists.
                    # Start the lists:
                    categories = []
                    responses = []
                    if (calculate_and_plot_cumulative_percent):
                        cum_pct = [] # start this list only if its needed to save memory

                    for i in range (0, limit_of_plotted_categories):
                        # i goes from 0 (first index) to limit_of_plotted_categories - 1
                        # (index of the last category to be kept):
                        # copy the elements from the DATASET to the list
                        # category is the 1st column (column 0); response is the 2nd (col 1);
                        # and cumulative percent is the 4th (col 3):
                        categories.append(DATASET.iloc[i, 0])
                        responses.append(DATASET.iloc[i, 1])
                        
                        if (calculate_and_plot_cumulative_percent):
                            cum_pct.append(DATASET.iloc[i, 3]) # only if there is something to iloc
                        
                    # Now, i = limit_of_plotted_categories - 1
                    # Create a variable to store the sum of other responses
                    other_responses = 0
                    # loop from i = limit_of_plotted_categories to i = df_length-1, index
                    # of the last element. Notice that this loop may have a single call, if there
                    # is only one element above the limit:
                    for i in range (limit_of_plotted_categories, (df_length - 1)):
                        
                        other_responses = other_responses + (DATASET.iloc[i, 1])
                    
                    # Now, add the last elements to the series:
                    # The last category is named 'others':
                    categories.append('others')
                    # The correspondent aggregated response is the value 
                    # stored in other_responses:
                    responses.append(other_responses)
                    # The cumulative percent is 100%, since this must be the sum of all
                    # elements (the previous ones plus the ones aggregated as 'others'
                    # must totalize 100%).
                    # On the other hand, the cumulative percent is stored only if needed:
                    cum_pct.append(100)
        
        else:
            # This is the situation where there is no limit of plotted categories. So, we
            # simply copy the columns to the plotted series (it is equivalent to the 
            # situation where there is a limit, but the limit is equal or inferior to the
            # size of the dataframe):
            categories = DATASET[categorical_var_name]
            responses = DATASET[response_var_name]
            # If there is a cum_pct column, copy it to a series too:
            if (calculate_and_plot_cumulative_percent):
                cum_pct = DATASET[cum_pct_col]
        
        
        # Now the data is prepared and we only have to plot 
        # categories, responses, and cum_pct:
        
        # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
        # so that the bars do not completely block other views.
        OPACITY = 0.95
        
        # Set labels and titles for the case they are None
        if (plot_title is None):
            
            if (aggregate_function == 'count'):
                # The graph is the same count, no matter the response
                plot_title = f"Bar_chart_count_of_{categorical_var_name}"
            
            else:
                plot_title = f"Bar_chart_for_{response_var_name}_by_{categorical_var_name}"
        
        if (horizontal_axis_title is None):

            horizontal_axis_title = categorical_var_name

        if (vertical_axis_title is None):
            # Notice that response_var_name already has the suffix indicating the
            # aggregation function
            vertical_axis_title = response_var_name
        
        fig, ax1 = plt.subplots(figsize = (12, 8))
        # Set image size (x-pixels, y-pixels) for printing in the notebook's cell:

        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 70 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)
        
        plt.title(plot_title)
        
        if (orientation == 'horizontal'):
            
            # invert the axes in relation to the default (vertical, below)
            ax1.set_ylabel(horizontal_axis_title)
            ax1.set_xlabel(vertical_axis_title, color = 'darkblue')
            
            # Horizontal bars used - barh method (bar horizontal):
            # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.barh.html
            # Now, the categorical variables stored in series categories must be
            # positioned as the vertical axis Y, whereas the correspondent responses
            # must be in the horizontal axis X.
            ax1.barh(categories, responses, color = 'darkblue', alpha = OPACITY, label = categorical_var_name)
            #.barh(y, x, ...)
            
            if (calculate_and_plot_cumulative_percent):
                # Let's plot the line for the cumulative percent
                # Set the grid for the bar chart as False. If it is True, there will
                # be to grids, one for the bars and other for the percents, making 
                # the image difficult to interpretate:
                ax1.grid(False)
                
                # Create the twin plot for the cumulative percent:
                # for the vertical orientation, we use the twinx. Here, we use twiny
                ax2 = ax1.twiny()
                # Here, the x axis must be the cum_pct value, and the Y
                # axis must be categories (it must be correspondent to the
                # bar chart)
                ax2.plot(cum_pct, categories, '-ro', label = "cumulative\npercent")
                #.plot(x, y, ...)
                ax2.tick_params('x', color = 'red')
                ax2.set_xlabel("Cumulative Percent (%)", color = 'red')
                ax2.legend()
                ax2.grid(grid) # shown if user set grid = True
                # If user wants to see the grid, it is shown only for the cumulative line.
            
            else:
                # There is no cumulative line, so the parameter grid must control 
                # the bar chart's grid
                ax1.legend()
                ax1.grid(grid)
            
        else: 
            
            ax1.set_xlabel(horizontal_axis_title)
            ax1.set_ylabel(vertical_axis_title, color = 'darkblue')
            # If None or an invalid orientation was used, set it as vertical
            # Use Matplotlib standard bar method (vertical bar):
            # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.bar.html#matplotlib.pyplot.bar
            
            # In this standard case, the categorical variables (categories) are positioned
            # as X, and the responses as Y:
            ax1.bar(categories, responses, color = 'darkblue', alpha = OPACITY, label = categorical_var_name)
            #.bar(x, y, ...)
            
            if (calculate_and_plot_cumulative_percent):
                # Let's plot the line for the cumulative percent
                # Set the grid for the bar chart as False. If it is True, there will
                # be to grids, one for the bars and other for the percents, making 
                # the image difficult to interpretate:
                ax1.grid(False)
                
                # Create the twin plot for the cumulative percent:
                ax2 = ax1.twinx()
                ax2.plot(cum_pct, categories, '-ro', label = "cumulative\npercent")
                #.plot(x, y, ...)
                ax2.tick_params('y', color = 'red')
                ax2.set_ylabel("Cumulative Percent (%)", color = 'red', rotation = 270)
                # rotate the twin axis so that its label is inverted in relation to the main
                # vertical axis.
                ax2.legend()
                ax2.grid(grid) # shown if user set grid = True
                # If user wants to see the grid, it is shown only for the cumulative line.
            
            else:
                # There is no cumulative line, so the parameter grid must control 
                # the bar chart's grid
                ax1.legend()
                ax1.grid(grid)
        
        # Notice that the .plot method is used for generating the plot for both orientations.
        # It is different from .bar and .barh, which specify the orientation of a bar; or
        # .hline (creation of an horizontal constant line); or .vline (creation of a vertical
        # constant line).
        
        # Now the parameters specific to the configurations are finished, so we can go back
        # to the general code:
        
        if (export_png == True):
            # Image will be exported
            import os
            
            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""
            
            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "bar_chart"
            
            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330
            
            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)
            
            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")
        
        #fig.tight_layout()
        
        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        
        return DATASET


    def calculate_cumulative_stats (df, column_to_analyze, cumulative_statistic = 'sum', new_cum_stats_col_name = None):
        
        import numpy as np
        import pandas as pd
        
        # df: the whole dataframe to be processed.
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # cumulative_statistic: the statistic that will be calculated. The cumulative
        # statistics allowed are: 'sum' (for cumulative sum, cumsum); 'product' 
        # (for cumulative product, cumprod); 'max' (for cumulative maximum, cummax);
        # and 'min' (for cumulative minimum, cummin).
        
        # new_cum_stats_col_name = None or string (inside quotes), 
        # containing the name of the new column created for storing the cumulative statistic
        # calculated. 
        # e.g. new_cum_stats_col_name = "cum_stats" will create a column named as 'cum_stats'.
        # If its None, the new column will be named as column_to_analyze + "_" + [selected
        # cumulative function] ('cumsum', 'cumprod', 'cummax', 'cummin')
        
        
        #WARNING: Use this function to a analyze a single column from a dataframe.
        
        if ((cumulative_statistic not in ['sum', 'product', 'max', 'min']) | (cumulative_statistic is None)):
            
            print("Please, select a valid method for calculating the cumulative statistics: sum, product, max, or min.")
            return "error"
        
        else:
            
            if (new_cum_stats_col_name is None):
                # set the standard name
                # column_to_analyze + "_" + [selected cumulative function] 
                # ('cumsum', 'cumprod', 'cummax', 'cummin')
                # cumulative_statistic variable stores ['sum', 'product', 'max', 'min']
                # we must concatenate "cum" to the left of this string:
                new_cum_stats_col_name = column_to_analyze + "_" + "cum" + cumulative_statistic
            
            # create a local copy of the dataframe to manipulate:
            DATASET = df.copy(deep = True)
            # The series to be analyzed is stored as DATASET[column_to_analyze]
            
            # Now apply the correct method
            # the dictionary dict_of_methods correlates the input cumulative_statistic to the
            # correct Pandas method to be applied to the dataframe column
            dict_of_methods = {
                
                'sum': DATASET[column_to_analyze].cumsum(),
                'product': DATASET[column_to_analyze].cumprod(),
                'max': DATASET[column_to_analyze].cummax(),
                'min': DATASET[column_to_analyze].cummin()
            }
            
            # To access the value (method) correspondent to a given key (input as 
            # cumulative_statistic): dictionary['key'], just as if accessing a column from
            # a dataframe. In this case, the method is accessed as:
            # dict_of_methods[cumulative_statistic], since cumulative_statistic is itself the key
            # of the dictionary of methods.
            
            # store the resultant of the method in a new column of DATASET 
            # named as new_cum_stats_col_name
            DATASET[new_cum_stats_col_name] = dict_of_methods[cumulative_statistic]
            
            print(f"The cumulative {cumulative_statistic} statistic was successfully calculated and added as the column \'{new_cum_stats_col_name}\' of the returned dataframe.\n")
            print("Check the new dataframe's 10 first rows:\n")
            
            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(DATASET.head(10))

            except: # regular mode
                print(DATASET.head(10))
            
            return DATASET


    def scatter_plot_lin_reg (data_in_same_column = False, df = None, column_with_predict_var_x = None, column_with_response_var_y = None, column_with_labels = None, list_of_dictionaries_with_series_to_analyze = [{'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}], x_axis_rotation = 70, y_axis_rotation = 0, show_linear_reg = True, grid = True, add_splines_lines = False, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330): 
        
        import random
        # Python Random documentation:
        # https://docs.python.org/3/library/random.html?msclkid=9d0c34b2d13111ec9cfa8ddaee9f61a1
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import matplotlib.colors as mcolors
        from scipy import stats
        
        # matplotlib.colors documentation:
        # https://matplotlib.org/3.5.0/api/colors_api.html?msclkid=94286fa9d12f11ec94660321f39bf47f
        
        # Matplotlib list of colors:
        # https://matplotlib.org/stable/gallery/color/named_colors.html?msclkid=0bb86abbd12e11ecbeb0a2439e5b0d23
        # Matplotlib colors tutorial:
        # https://matplotlib.org/stable/tutorials/colors/colors.html
        # Matplotlib example of Python code using matplotlib.colors:
        # https://matplotlib.org/stable/_downloads/0843ee646a32fc214e9f09328c0cd008/colors.py
        # Same example as Jupyter Notebook:
        # https://matplotlib.org/stable/_downloads/2a7b13c059456984288f5b84b4b73f45/colors.ipynb
        
            
        # data_in_same_column = False: set as True if all the values to plot are in a same column.
        # If data_in_same_column = True, you must specify the dataframe containing the data as df;
        # the column containing the predict variable (X) as column_with_predict_var_x; the column 
        # containing the responses to plot (Y) as column_with_response_var_y; and the column 
        # containing the labels (subgroup) indication as column_with_labels. 
        # df is an object, so do not declare it in quotes. The other three arguments (columns' names) 
        # are strings, so declare in quotes. 
        
        # Example: suppose you have a dataframe saved as dataset, and two groups A and B to compare. 
        # All the results for both groups are in a column named 'results', wich will be plot against
        # the time, saved as 'time' (X = 'time'; Y = 'results'). If the result is for
        # an entry from group A, then a column named 'group' has the value 'A'. If it is for group B,
        # column 'group' shows the value 'B'. In this example:
        # data_in_same_column = True,
        # df = dataset,
        # column_with_predict_var_x = 'time',
        # column_with_response_var_y = 'results', 
        # column_with_labels = 'group'
        # If you want to declare a list of dictionaries, keep data_in_same_column = False and keep
        # df = None (the other arguments may be set as None, but it is not mandatory: 
        # column_with_predict_var_x = None, column_with_response_var_y = None, column_with_labels = None).
        

        # Parameter to input when DATA_IN_SAME_COLUMN = False:
        # list_of_dictionaries_with_series_to_analyze: if data is already converted to series, lists
        # or arrays, provide them as a list of dictionaries. It must be declared as a list, in brackets,
        # even if there is a single dictionary.
        # Use always the same keys: 'x' for the X-series (predict variables); 'y' for the Y-series
        # (response variables); and 'lab' for the labels. If you do not want to declare a series, simply
        # keep as None, but do not remove or rename a key (ALWAYS USE THE KEYS SHOWN AS MODEL).
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to plot more series.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'x': x_series, 'y': y_series, 'lab': label}, where x_series, y_series and label
        # represents the series and label of the added dictionary (you can pass 'lab': None, but if 
        # 'x' or 'y' are None, the new dictionary will be ignored).
        
        # Examples:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'x': DATASET['X'], 'y': DATASET['Y'], 'lab': 'label'}]
        # will plot a single variable. In turns:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'x': DATASET['X'], 'y': DATASET['Y1'], 'lab': 'label'}, {'x': DATASET['X'], 'y': DATASET['Y2'], 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}]
        # will plot two series, Y1 x X and Y2 x X.
        # Notice that all dictionaries where 'x' or 'y' are None are automatically ignored.
        # If None is provided to 'lab', an automatic label will be generated.
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        if (data_in_same_column == True):
            
            print("Data to be plotted in a same column.\n")
            
            if (df is None):
                
                print("Please, input a valid dataframe as df.\n")
                list_of_dictionaries_with_series_to_analyze = []
                # The code will check the size of this list on the next block.
                # If it is zero, code is simply interrupted.
                # Instead of returning an error, we use this code structure that can be applied
                # on other graphic functions that do not return a summary (and so we should not
                # return a value like 'error' to interrupt the function).
            
            elif (column_with_predict_var_x is None):
                
                print("Please, input a valid column name as column_with_predict_var_x.\n")
                list_of_dictionaries_with_series_to_analyze = []
            
            elif (column_with_response_var_y is None):
                
                print("Please, input a valid column name as column_with_response_var_y.\n")
                list_of_dictionaries_with_series_to_analyze = []
            
            else:
                
                # set a local copy of the dataframe:
                DATASET = df.copy(deep = True)
                
                if (column_with_labels is None):
                
                    print("Using the whole series (column) for correlation.\n")
                    column_with_labels = 'whole_series_' + column_with_response_var_y
                    DATASET[column_with_labels] = column_with_labels
                
                # sort DATASET; by column_with_predict_var_x; by column column_with_labels
                # and by column_with_response_var_y, all in Ascending order
                # Since we sort by label (group), it is easier to separate the groups.
                DATASET = DATASET.sort_values(by = [column_with_predict_var_x, column_with_labels, column_with_response_var_y], ascending = [True, True, True])
                
                # Reset indices:
                DATASET = DATASET.reset_index(drop = True)
                
                # If column_with_predict_var_x is an object, the user may be trying to pass a date as x. 
                # So, let's try to convert it to datetime:
        
                if ((DATASET[column_with_predict_var_x]).dtype not in numeric_dtypes):
                    
                    try:
                        DATASET[column_with_predict_var_x] = (DATASET[column_with_predict_var_x]).astype('datetime64[ns]')
                        print("Variable X successfully converted to datetime64[ns].\n")
                        
                    except:
                        # Simply ignore it
                        pass
                
                # Get a series of unique values of the labels, and save it as a list using the
                # list attribute:
                unique_labels = list(DATASET[column_with_labels].unique())
                print(f"{len(unique_labels)} different labels detected: {unique_labels}.\n")
                
                # Start a list to store the dictionaries containing the keys:
                # 'x': list of predict variables; 'y': list of responses; 'lab': the label (group)
                list_of_dictionaries_with_series_to_analyze = []
                
                # Loop through each possible label:
                for lab in unique_labels:
                    # loop through each element from the list unique_labels, referred as lab
                    
                    # Set a filter for the dataset, to select only rows correspondent to that
                    # label:
                    boolean_filter = (DATASET[column_with_labels] == lab)
                    
                    # Create a copy of the dataset, with entries selected by that filter:
                    ds_copy = (DATASET[boolean_filter]).copy(deep = True)
                    # Sort again by X and Y, to guarantee the results are in order:
                    ds_copy = ds_copy.sort_values(by = [column_with_predict_var_x, column_with_response_var_y], ascending = [True, True])
                    # Restart the index of the copy:
                    ds_copy = ds_copy.reset_index(drop = True)
                    
                    # Re-extract the X and Y series and convert them to NumPy arrays 
                    # (these arrays will be important later in the function):
                    x = np.array(ds_copy[column_with_predict_var_x])
                    y = np.array(ds_copy[column_with_response_var_y])
                
                    # Then, create the dictionary:
                    dict_of_values = {'x': x, 'y': y, 'lab': lab}
                    
                    # Now, append dict_of_values to list_of_dictionaries_with_series_to_analyze:
                    list_of_dictionaries_with_series_to_analyze.append(dict_of_values)
                    
                # Now, we have a list of dictionaries with the same format of the input list.
                
        else:
            
            # The user input a list_of_dictionaries_with_series_to_analyze
            # Create a support list:
            support_list = []
            
            # Loop through each element on the list list_of_dictionaries_with_series_to_analyze:
            
            for i in range (0, len(list_of_dictionaries_with_series_to_analyze)):
                # from i = 0 to i = len(list_of_dictionaries_with_series_to_analyze) - 1, index of the
                # last element from the list
                
                # pick the i-th dictionary from the list:
                dictionary = list_of_dictionaries_with_series_to_analyze[i]
                
                # access 'x', 'y', and 'lab' keys from the dictionary:
                x = dictionary['x']
                y = dictionary['y']
                lab = dictionary['lab']
                # Remember that all this variables are series from a dataframe, so we can apply
                # the astype function:
                # https://www.askpython.com/python/built-in-methods/python-astype?msclkid=8f3de8afd0d411ec86a9c1a1e290f37c
                
                # check if at least x and y are not None:
                if ((x is not None) & (y is not None)):
                    
                    # If column_with_predict_var_x is an object, the user may be trying to pass a date as x. 
                    # So, let's try to convert it to datetime:
                    if (x.dtype not in numeric_dtypes):

                        try:
                            x = (x).astype('datetime64[ns]')
                            print(f"Variable X from {i}-th dictionary successfully converted to datetime64[ns].\n")

                        except:
                            # Simply ignore it
                            pass
                    
                    # Possibly, x and y are not ordered. Firstly, let's merge them into a temporary
                    # dataframe to be able to order them together.
                    # Use the 'list' attribute to guarantee that x and y were read as lists. These lists
                    # are the values for a dictionary passed as argument for the constructor of the
                    # temporary dataframe. When using the list attribute, we make the series independent
                    # from its origin, even if it was created from a Pandas dataframe. Then, we have a
                    # completely independent dataframe that may be manipulated and sorted, without worrying
                    # that it may modify its origin:
                    
                    temp_df = pd.DataFrame(data = {'x': list(x), 'y': list(y)})
                    # sort this dataframe by 'x' and 'y':
                    temp_df = temp_df.sort_values(by = ['x', 'y'], ascending = [True, True])
                    # restart index:
                    temp_df = temp_df.reset_index(drop = True)
                    
                    # Re-extract the X and Y series and convert them to NumPy arrays 
                    # (these arrays will be important later in the function):
                    x = np.array(temp_df['x'])
                    y = np.array(temp_df['y'])
                    
                    # check if lab is None:
                    if (lab is None):
                        # input a default label.
                        # Use the str attribute to convert the integer to string, allowing it
                        # to be concatenated
                        lab = "X" + str(i) + "_x_" + "Y" + str(i)
                        
                    # Then, create the dictionary:
                    dict_of_values = {'x': x, 'y': y, 'lab': lab}
                    
                    # Now, append dict_of_values to support list:
                    support_list.append(dict_of_values)
                
            # Now, support_list contains only the dictionaries with valid entries, as well
            # as labels for each collection of data. The values are independent from their origin,
            # and now they are ordered and in the same format of the data extracted directly from
            # the dataframe.
            # So, make the list_of_dictionaries_with_series_to_analyze the support_list itself:
            list_of_dictionaries_with_series_to_analyze = support_list
            print(f"{len(list_of_dictionaries_with_series_to_analyze)} valid series input.\n")

            
        # Now that both methods of input resulted in the same format of list, we can process both
        # with the same code.
        
        # Each dictionary in list_of_dictionaries_with_series_to_analyze represents a series to
        # plot. So, the total of series to plot is:
        total_of_series = len(list_of_dictionaries_with_series_to_analyze)
        
        if (total_of_series <= 0):
            
            print("No valid series to plot. Please, provide valid arguments.\n")
            return "error" 
            # we return the value because this function always returns an object.
            # In other functions, this return would be omitted.
        
        else:
            
            # Continue to plotting and calculating the fitting.
            # Notice that we sorted the all the lists after they were separated and before
            # adding them to dictionaries. Also, the timestamps were converted to datetime64 variables
            
            # Now we pre-processed the data, we can obtain a final list of dictionaries, containing
            # the linear regression information (it will be plotted only if the user asked to). Start
            # a list to store all predictions:
            list_of_dictionaries_with_series_and_predictions = []
            
            # Loop through each dictionary (element) on the list list_of_dictionaries_with_series_to_analyze:
            for dictionary in list_of_dictionaries_with_series_to_analyze:
                
                x_is_datetime = False
                # boolean that will map if x is a datetime or not. Only change to True when it is.
                
                # Access keys 'x' and 'y' to retrieve the arrays.
                x = dictionary['x']
                y = dictionary['y']
                
                # Check if the elements from array x are np.datetime64 objects. Pick the first
                # element to check:
                
                if (type(x[0]) == np.datetime64):
                    
                    x_is_datetime = True
                    
                if (x_is_datetime):
                    # In this case, performing the linear regression directly in X will
                    # return an error. We must associate a sequential number to each time.
                    # to keep the distance between these integers the same as in the original sequence
                    # let's define a difference of 1 ns as 1. The 1st timestamp will be zero, and the
                    # addition of 1 ns will be an addition of 1 unit. So a timestamp recorded 10 ns
                    # after the time zero will have value 10. At the end, we divide every element by
                    # 10**9, to obtain the correspondent distance in seconds.
                    
                    # start a list for the associated integer timescale. Put the number zero,
                    # associated to the first timestamp:
                    int_timescale = [0]
                    
                    # loop through each element of the array x, starting from index 1:
                    for i in range(1, len(x)):
                        
                        # calculate the timedelta between x[i] and x[i-1]:
                        # The delta method from the Timedelta class converts the timedelta to
                        # nanoseconds, guaranteeing the internal compatibility:
                        timedelta = pd.Timedelta(x[i] - x[(i-1)]).delta
                        
                        # Sum this timedelta (integer number of nanoseconds) to the
                        # previous element from int_timescale, and append the result to the list:
                        int_timescale.append((timedelta + int_timescale[(i-1)]))
                    
                    # Now convert the new scale (that preserves the distance between timestamps)
                    # to NumPy array:
                    int_timescale = np.array(int_timescale)
                    
                    # Divide by 10**9 to obtain the distances in seconds, reducing the order of
                    # magnitude of the integer numbers (the division is allowed for arrays)
                    int_timescale = int_timescale / (10**9)
                    
                    # Finally, use this timescale to obtain the linear regression:
                    lin_reg = stats.linregress(int_timescale, y = y)
                
                else:
                    # Obtain the linear regression object directly from x. Since x is not a
                    # datetime object, we can calculate the regression directly on it:
                    lin_reg = stats.linregress(x, y = y)
                    
                # Retrieve the equation as a string.
                # Access the attributes intercept and slope from the lin_reg object:
                lin_reg_equation = "y = %.2f*x + %.2f" %((lin_reg).slope, (lin_reg).intercept)
                # .2f: float with only two decimals
                    
                # Retrieve R2 (coefficient of correlation) also as a string
                r2_lin_reg = "R²_lin_reg = %.4f" %(((lin_reg).rvalue) ** 2)
                # .4f: 4 decimals. ((lin_reg).rvalue) is the coefficient R. We
                # raise it to the second power by doing **2, where ** is the potentiation.
                    
                # Add these two strings to the dictionary
                dictionary['lin_reg_equation'] = lin_reg_equation
                dictionary['r2_lin_reg'] = r2_lin_reg
                    
                # Now, as final step, let's apply the values x to the linear regression
                # equation to obtain the predicted series used to plot the straight line.
                    
                # The lists cannot perform vector operations like element-wise sum or product, 
                # but numpy arrays can. For example, [1, 2] + 1 would be interpreted as the try
                # for concatenation of two lists, resulting in error. But, np.array([1, 2]) + 1
                # is allowed, resulting in: np.array[2, 3].
                # This and the fact that Scipy and Matplotlib are built on NumPy were the reasons
                # why we converted every list to numpy arrays.
                
                # Save the predicted values as the array y_pred_lin_reg.
                # Access the attributes intercept and slope from the lin_reg object.
                # The equation is y = (slope * x) + intercept
                
                # Notice that again we cannot apply the equation directly to a timestamp.
                # So once again we will apply the integer scale to obtain the predictions
                # if we are dealing with datetime objects:
                if (x_is_datetime):
                    y_pred_lin_reg = ((lin_reg).intercept) + ((lin_reg).slope) * (int_timescale)
                
                else:
                    # x is not a timestamp, so we can directly apply it to the regression
                    # equation:
                    y_pred_lin_reg = ((lin_reg).intercept) + ((lin_reg).slope) * (x)
                
                # Add this array to the dictionary with the key 'y_pred_lin_reg':
                dictionary['y_pred_lin_reg'] = y_pred_lin_reg
                
                if (x_is_datetime):
                
                    print("For performing the linear regression, a sequence of floats proportional to the timestamps was created. In this sequence, check on the returned object a dictionary containing the timestamps and the correspondent integers, that keeps the distance proportion between successive timestamps. The sequence was created by calculating the timedeltas as an integer number of nanoseconds, which were converted to seconds. The first timestamp was considered time = 0.")
                    print("Notice that the regression equation is based on the use of this sequence of floats as X.\n")
                    
                    dictionary['warning'] = "x is a numeric scale that was obtained from datetimes, preserving the distance relationships. It was obtained for allowing the polynomial fitting."
                    dictionary['numeric_to_datetime_correlation'] = {
                        
                        'x = 0': x[0],
                        f'x = {max(int_timescale)}': x[(len(x) - 1)]
                        
                    }
                    
                    dictionary['sequence_of_floats_correspondent_to_timestamps'] = {
                                                                                    'original_timestamps': x,
                                                                                    'sequence_of_floats': int_timescale
                                                                                    }
                    
                # Finally, append this dictionary to list support_list:
                list_of_dictionaries_with_series_and_predictions.append(dictionary)
            
            print("Returning a list of dictionaries. Each one contains the arrays of valid series and labels, and the equations, R² and values predicted by the linear regressions.\n")
            
            # Now we finished the loop, list_of_dictionaries_with_series_and_predictions 
            # contains all series converted to NumPy arrays, with timestamps parsed as datetimes, 
            # and all the information regarding the linear regression, including the predicted 
            # values for plotting.
            # This list will be the object returned at the end of the function. Since it is an
            # JSON-formatted list, we can use the function json_obj_to_pandas_dataframe to convert
            # it to a Pandas dataframe.
            
            
            # Now, we can plot the figure.
            # we set alpha = 0.95 (opacity) to give a degree of transparency (5%), 
            # so that one series do not completely block the visualization of the other.
            
            # Let's retrieve the list of Matplotlib CSS colors:
            css4 = mcolors.CSS4_COLORS
            # css4 is a dictionary of colors: {'aliceblue': '#F0F8FF', 'antiquewhite': '#FAEBD7', ...}
            # Each key of this dictionary is a color name to be passed as argument color on the plot
            # function. So let's retrieve the array of keys, and use the list attribute to convert this
            # array to a list of colors:
            list_of_colors = list(css4.keys())
            
            # In 11 May 2022, this list of colors had 148 different elements
            # Since this list is in alphabetic order, let's create a random order for the colors.
            
            # Function random.sample(input_sequence, number_of_samples): 
            # this function creates a list containing a total of elements equals to the parameter 
            # "number_of_samples", which must be an integer.
            # This list is obtained by ramdomly selecting a total of "number_of_samples" elements from the
            # list "input_sequence" passed as parameter.
            
            # Function random.choices(input_sequence, k = number_of_samples):
            # similarly, randomly select k elements from the sequence input_sequence. This function is
            # newer than random.sample
            # Since we want to simply randomly sort the sequence, we can pass k = len(input_sequence)
            # to obtain the randomly sorted sequence:
            list_of_colors = random.choices(list_of_colors, k = len(list_of_colors))
            # Now, we have a random list of colors to use for plotting the charts
            
            if (add_splines_lines == True):
                LINE_STYLE = '-'

            else:
                LINE_STYLE = ''
            
            # Matplotlib linestyle:
            # https://matplotlib.org/stable/gallery/lines_bars_and_markers/linestyles.html?msclkid=68737f24d16011eca9e9c4b41313f1ad
            
            if (plot_title is None):
                # Set graphic title
                plot_title = f"Y_x_X"

            if (horizontal_axis_title is None):
                # Set horizontal axis title
                horizontal_axis_title = "X"

            if (vertical_axis_title is None):
                # Set vertical axis title
                vertical_axis_title = "Y"
            
            # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
            # so that the bars do not completely block other views.
            OPACITY = 0.95
            
            #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
            fig = plt.figure(figsize = (12, 8))
            ax = fig.add_subplot()

            i = 0 # Restart counting for the loop of colors
            
            # Loop through each dictionary from list_of_dictionaries_with_series_and_predictions:
            for dictionary in list_of_dictionaries_with_series_and_predictions:
                
                # Try selecting a color from list_of_colors:
                try:
                    
                    COLOR = list_of_colors[i]
                    # Go to the next element i, so that the next plot will use a different color:
                    i = i + 1
                
                except IndexError:
                    
                    # This error will be raised if list index is out of range, 
                    # i.e. if i >= len(list_of_colors) - we used all colors from the list (at least 148).
                    # So, return the index to zero to restart the colors from the beginning:
                    i = 0
                    COLOR = list_of_colors[i]
                    i = i + 1
                
                # Access the arrays and label from the dictionary:
                X = dictionary['x']
                Y = dictionary['y']
                LABEL = dictionary['lab']
                
                # Scatter plot:
                ax.plot(X, Y, linestyle = LINE_STYLE, marker = "o", color = COLOR, alpha = OPACITY, label = LABEL)
                # Axes.plot documentation:
                # https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.plot.html?msclkid=42bc92c1d13511eca8634a2c93ab89b5
                
                # x and y are positional arguments: they are specified by their position in function
                # call, not by an argument name like 'marker'.
                
                # Matplotlib markers:
                # https://matplotlib.org/stable/api/markers_api.html?msclkid=36c5eec5d16011ec9583a5777dc39d1f
                
                if (show_linear_reg == True):
                    
                    # Plot the linear regression using the same color.
                    # Access the array of fitted Y's in the dictionary:
                    Y_PRED = dictionary['y_pred_lin_reg']
                    Y_PRED_LABEL = 'lin_reg_' + str(LABEL) # for the case where label is numeric
                    
                    ax.plot(X, Y_PRED,  linestyle = '-', marker = '', color = COLOR, alpha = OPACITY, label = Y_PRED_LABEL)

            # Now we finished plotting all of the series, we can set the general configuration:
            
            #ROTATE X AXIS IN XX DEGREES
            plt.xticks(rotation = x_axis_rotation)
            # XX = 0 DEGREES x_axis (Default)
            #ROTATE Y AXIS IN XX DEGREES:
            plt.yticks(rotation = y_axis_rotation)
            # XX = 0 DEGREES y_axis (Default)
            
            ax.set_title(plot_title)
            ax.set_xlabel(horizontal_axis_title)
            ax.set_ylabel(vertical_axis_title)

            ax.grid(grid) # show grid or not
            ax.legend(loc = 'upper left')
            # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
            # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
            # https://www.statology.org/matplotlib-legend-position/

            if (export_png == True):
                # Image will be exported
                import os

                #check if the user defined a directory path. If not, set as the default root path:
                if (directory_to_save is None):
                    #set as the default
                    directory_to_save = ""

                #check if the user defined a file name. If not, set as the default name for this
                # function.
                if (file_name is None):
                    #set as the default
                    file_name = "scatter_plot_lin_reg"

                #check if the user defined an image resolution. If not, set as the default 110 dpi
                # resolution.
                if (png_resolution_dpi is None):
                    #set as 330 dpi
                    png_resolution_dpi = 330

                #Get the new_file_path
                new_file_path = os.path.join(directory_to_save, file_name)

                #Export the file to this new path:
                # The extension will be automatically added by the savefig method:
                plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                #quality could be set from 1 to 100, where 100 is the best quality
                #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                #transparent = True or False
                # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

            #fig.tight_layout()

            ## Show an image read from an image file:
            ## import matplotlib.image as pltimg
            ## img=pltimg.imread('mydecisiontree.png')
            ## imgplot = plt.imshow(img)
            ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
            ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
            ##  '03_05_END.ipynb'
            plt.show()
            
            if (show_linear_reg == True):
                
                try:
                    # only works in Jupyter Notebook:
                    from IPython.display import display
                except:
                    pass
                
                print("\nLinear regression summaries (equations and R²):\n")
                
                for dictionary in list_of_dictionaries_with_series_and_predictions:
                    
                    print(f"Linear regression summary for {dictionary['lab']}:\n")
                    
                    try:
                        display(dictionary['lin_reg_equation'])
                        display(dictionary['r2_lin_reg'])

                    except: # regular mode                  
                        print(dictionary['lin_reg_equation'])
                        print(dictionary['r2_lin_reg'])
                    
                    print("\n")
            
            
            return list_of_dictionaries_with_series_and_predictions


    def polynomial_fit (data_in_same_column = False, df = None, column_with_predict_var_x = None, column_with_response_var_y = None, column_with_labels = None, list_of_dictionaries_with_series_to_analyze = [{'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}], polynomial_degree = 6, calculate_roots = False, calculate_derivative = False, calculate_integral = False, x_axis_rotation = 70, y_axis_rotation = 0, show_polynomial_reg = True, grid = True, add_splines_lines = False, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import random
        # Python Random documentation:
        # https://docs.python.org/3/library/random.html?msclkid=9d0c34b2d13111ec9cfa8ddaee9f61a1
        import numpy as np
        from numpy.polynomial.polynomial import Polynomial
        import pandas as pd
        import matplotlib.pyplot as plt
        import matplotlib.colors as mcolors
        
        # Check numpy.polynomial class API documentation for other polynomials 
        # (chebyshev, legendre, hermite, etc):
        # https://numpy.org/doc/stable/reference/routines.polynomials.package.html#module-numpy.polynomial
        
        # df: the whole dataframe to be processed.
        
        # data_in_same_column = False: set as True if all the values to plot are in a same column.
        # If data_in_same_column = True, you must specify the dataframe containing the data as df;
        # the column containing the predict variable (X) as column_with_predict_var_x; the column 
        # containing the responses to plot (Y) as column_with_response_var_y; and the column 
        # containing the labels (subgroup) indication as column_with_labels. 
        # df is an object, so do not declare it in quotes. The other three arguments (columns' names) 
        # are strings, so declare in quotes. 
        
        # Example: suppose you have a dataframe saved as dataset, and two groups A and B to compare. 
        # All the results for both groups are in a column named 'results', wich will be plot against
        # the time, saved as 'time' (X = 'time'; Y = 'results'). If the result is for
        # an entry from group A, then a column named 'group' has the value 'A'. If it is for group B,
        # column 'group' shows the value 'B'. In this example:
        # data_in_same_column = True,
        # df = dataset,
        # column_with_predict_var_x = 'time',
        # column_with_response_var_y = 'results', 
        # column_with_labels = 'group'
        # If you want to declare a list of dictionaries, keep data_in_same_column = False and keep
        # df = None (the other arguments may be set as None, but it is not mandatory: 
        # column_with_predict_var_x = None, column_with_response_var_y = None, column_with_labels = None).
        

        # Parameter to input when DATA_IN_SAME_COLUMN = False:
        # list_of_dictionaries_with_series_to_analyze: if data is already converted to series, lists
        # or arrays, provide them as a list of dictionaries. It must be declared as a list, in brackets,
        # even if there is a single dictionary.
        # Use always the same keys: 'x' for the X-series (predict variables); 'y' for the Y-series
        # (response variables); and 'lab' for the labels. If you do not want to declare a series, simply
        # keep as None, but do not remove or rename a key (ALWAYS USE THE KEYS SHOWN AS MODEL).
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to plot more series.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'x': x_series, 'y': y_series, 'lab': label}, where x_series, y_series and label
        # represents the series and label of the added dictionary (you can pass 'lab': None, but if 
        # 'x' or 'y' are None, the new dictionary will be ignored).
        
        # Examples:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'x': DATASET['X'], 'y': DATASET['Y'], 'lab': 'label'}]
        # will plot a single variable. In turns:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'x': DATASET['X'], 'y': DATASET['Y1'], 'lab': 'label'}, {'x': DATASET['X'], 'y': DATASET['Y2'], 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}]
        # will plot two series, Y1 x X and Y2 x X.
        # Notice that all dictionaries where 'x' or 'y' are None are automatically ignored.
        # If None is provided to 'lab', an automatic label will be generated.
        
        # polynomial_degree = integer value representing the degree of the fitted polynomial.
        
        # calculate_derivative = False. Alternatively, set as True to calculate the derivative of the
        #  fitted polynomial and add it as a column of the dataframe.
        
        # calculate_integral = False. Alternatively, set as True to calculate the integral of the
        #  fitted polynomial and add it as a column of the dataframe.
        
        # calculate_roots = False.  Alternatively, set as True to calculate the roots of the
        #  fitted polynomial and return them as a NumPy array.
        
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        if (data_in_same_column == True):
            
            if (df is None):
                
                print("Please, input a valid dataframe as df.\n")
                list_of_dictionaries_with_series_to_analyze = []
                # The code will check the size of this list on the next block.
                # If it is zero, code is simply interrupted.
                # Instead of returning an error, we use this code structure that can be applied
                # on other graphic functions that do not return a summary (and so we should not
                # return a value like 'error' to interrupt the function).
            
            elif (column_with_predict_var_x is None):
                
                print("Please, input a valid column name as column_with_predict_var_x.\n")
                list_of_dictionaries_with_series_to_analyze = []
            
            elif (column_with_response_var_y is None):
                
                print("Please, input a valid column name as column_with_response_var_y.\n")
                list_of_dictionaries_with_series_to_analyze = []
            
            else:
                
                # set a local copy of the dataframe:
                DATASET = df.copy(deep = True)
                
                if (column_with_labels is None):
                
                    print("Using the whole series (column) for correlation.\n")
                    column_with_labels = 'whole_series_' + column_with_response_var_y
                    DATASET[column_with_labels] = column_with_labels
                
                # sort DATASET; by column_with_predict_var_x; by column column_with_labels
                # and by column_with_response_var_y, all in Ascending order
                # Since we sort by label (group), it is easier to separate the groups.
                DATASET = DATASET.sort_values(by = [column_with_predict_var_x, column_with_labels, column_with_response_var_y], ascending = [True, True, True])
                
                # Reset indices:
                DATASET = DATASET.reset_index(drop = True)
                
                # If column_with_predict_var_x is an object, the user may be trying to pass a date as x. 
                # So, let's try to convert it to datetime:
                if ((DATASET[column_with_predict_var_x]).dtype not in numeric_dtypes):
                    
                    try:
                        DATASET[column_with_predict_var_x] = (DATASET[column_with_predict_var_x]).astype('datetime64[ns]')
                        print("Variable X successfully converted to datetime64[ns].\n")
                        
                    except:
                        # Simply ignore it
                        pass
                
                # Get a series of unique values of the labels, and save it as a list using the
                # list attribute:
                unique_labels = list(DATASET[column_with_labels].unique())
                print(f"{len(unique_labels)} different labels detected: {unique_labels}.\n")
                
                # Start a list to store the dictionaries containing the keys:
                # 'x': list of predict variables; 'y': list of responses; 'lab': the label (group)
                list_of_dictionaries_with_series_to_analyze = []
                
                # Loop through each possible label:
                for lab in unique_labels:
                    # loop through each element from the list unique_labels, referred as lab
                    
                    # Set a filter for the dataset, to select only rows correspondent to that
                    # label:
                    boolean_filter = (DATASET[column_with_labels] == lab)
                    
                    # Create a copy of the dataset, with entries selected by that filter:
                    ds_copy = (DATASET[boolean_filter]).copy(deep = True)
                    # Sort again by X and Y, to guarantee the results are in order:
                    ds_copy = ds_copy.sort_values(by = [column_with_predict_var_x, column_with_response_var_y], ascending = [True, True])
                    # Restart the index of the copy:
                    ds_copy = ds_copy.reset_index(drop = True)
                    
                    # Re-extract the X and Y series and convert them to NumPy arrays 
                    # (these arrays will be important later in the function):
                    x = np.array(ds_copy[column_with_predict_var_x])
                    y = np.array(ds_copy[column_with_response_var_y])
                
                    # Then, create the dictionary:
                    dict_of_values = {'x': x, 'y': y, 'lab': lab}
                    
                    # Now, append dict_of_values to list_of_dictionaries_with_series_to_analyze:
                    list_of_dictionaries_with_series_to_analyze.append(dict_of_values)
                    
                # Now, we have a list of dictionaries with the same format of the input list.
                
        else:
            
            # The user input a list_of_dictionaries_with_series_to_analyze
            # Create a support list:
            support_list = []
            
            # Loop through each element on the list list_of_dictionaries_with_series_to_analyze:
            
            for i in range (0, len(list_of_dictionaries_with_series_to_analyze)):
                # from i = 0 to i = len(list_of_dictionaries_with_series_to_analyze) - 1, index of the
                # last element from the list
                
                # pick the i-th dictionary from the list:
                dictionary = list_of_dictionaries_with_series_to_analyze[i]
                
                # access 'x', 'y', and 'lab' keys from the dictionary:
                x = dictionary['x']
                y = dictionary['y']
                lab = dictionary['lab']
                # Remember that all this variables are series from a dataframe, so we can apply
                # the astype function:
                # https://www.askpython.com/python/built-in-methods/python-astype?msclkid=8f3de8afd0d411ec86a9c1a1e290f37c
                
                # check if at least x and y are not None:
                if ((x is not None) & (y is not None)):
                    
                    # If column_with_predict_var_x is an object, the user may be trying to pass a date as x. 
                    # So, let's try to convert it to datetime:
                    
                    if (x.dtype not in numeric_dtypes):

                        try:
                            x = (x).astype('datetime64[ns]')
                            x_is_datetime = True
                            print(f"Variable X from {i}-th dictionary successfully converted to datetime64[ns].\n")

                        except:
                            # Simply ignore it
                            pass
                    
                    # Possibly, x and y are not ordered. Firstly, let's merge them into a temporary
                    # dataframe to be able to order them together.
                    # Use the 'list' attribute to guarantee that x and y were read as lists. These lists
                    # are the values for a dictionary passed as argument for the constructor of the
                    # temporary dataframe. When using the list attribute, we make the series independent
                    # from its origin, even if it was created from a Pandas dataframe. Then, we have a
                    # completely independent dataframe that may be manipulated and sorted, without worrying
                    # that it may modify its origin:
                    
                    temp_df = pd.DataFrame(data = {'x': list(x), 'y': list(y)})
                    # sort this dataframe by 'x' and 'y':
                    temp_df = temp_df.sort_values(by = ['x', 'y'], ascending = [True, True])
                    # restart index:
                    temp_df = temp_df.reset_index(drop = True)
                    
                    # Re-extract the X and Y series and convert them to NumPy arrays 
                    # (these arrays will be important later in the function):
                    x = np.array(temp_df['x'])
                    y = np.array(temp_df['y'])
                    
                    # check if lab is None:
                    if (lab is None):
                        # input a default label.
                        # Use the str attribute to convert the integer to string, allowing it
                        # to be concatenated
                        lab = "X" + str(i) + "_x_" + "Y" + str(i)
                        
                    # Then, create the dictionary:
                    dict_of_values = {'x': x, 'y': y, 'lab': lab}
                    
                    # Now, append dict_of_values to support list:
                    support_list.append(dict_of_values)
                
            # Now, support_list contains only the dictionaries with valid entries, as well
            # as labels for each collection of data. The values are independent from their origin,
            # and now they are ordered and in the same format of the data extracted directly from
            # the dataframe.
            # So, make the list_of_dictionaries_with_series_to_analyze the support_list itself:
            list_of_dictionaries_with_series_to_analyze = support_list
            print(f"{len(list_of_dictionaries_with_series_to_analyze)} valid series input.\n")

            
        # Now that both methods of input resulted in the same format of list, we can process both
        # with the same code.
        
        # Each dictionary in list_of_dictionaries_with_series_to_analyze represents a series to
        # plot. So, the total of series to plot is:
        total_of_series = len(list_of_dictionaries_with_series_to_analyze)
                
        if (total_of_series <= 0):
            
            print("No valid series to fit. Please, provide valid arguments.\n")
            return "error" 
            # we return the value because this function always returns an object.
            # In other functions, this return would be omitted.
        
        else:
            
            # Continue to plotting and calculating the fitting.
            # Notice that we sorted the all the lists after they were separated and before
            # adding them to dictionaries. Also, the timestamps were converted to datetime64 variables
            
            # Now we pre-processed the data, we can obtain a final list of dictionaries, containing
            # the linear regression information (it will be plotted only if the user asked to). Start
            # a list to store all predictions:
            list_of_dictionaries_with_series_and_predictions = []
            
            # Loop through each dictionary (element) on the list list_of_dictionaries_with_series_to_analyze:
            for dictionary in list_of_dictionaries_with_series_to_analyze:
                
                x_is_datetime = False
                # boolean that will map if x is a datetime or not. Only change to True when it is.
                
                # Access keys 'x' and 'y' to retrieve the arrays.
                x = dictionary['x']
                y = dictionary['y']
                lab = dictionary['lab']
                
                # Check if the elements from array x are np.datetime64 objects. Pick the first
                # element to check:
                
                if (type(x[0]) == np.datetime64):
                    
                    x_is_datetime = True
                
                if (x_is_datetime):
                    # In this case, performing the linear regression directly in X will
                    # return an error. We must associate a sequential number to each time.
                    # to keep the distance between these integers the same as in the original sequence
                    # let's define a difference of 1 ns as 1. The 1st timestamp will be zero, and the
                    # addition of 1 ns will be an addition of 1 unit. So a timestamp recorded 10 ns
                    # after the time zero will have value 10. At the end, we divide every element by
                    # 10**9, to obtain the correspondent distance in seconds.
                    
                    # start a list for the associated integer timescale. Put the number zero,
                    # associated to the first timestamp:
                    int_timescale = [0]
                    
                    # loop through each element of the array x, starting from index 1:
                    for i in range(1, len(x)):
                        
                        # calculate the timedelta between x[i] and x[i-1]:
                        # The delta method from the Timedelta class converts the timedelta to
                        # nanoseconds, guaranteeing the internal compatibility:
                        timedelta = pd.Timedelta(x[i] - x[(i-1)]).delta
                        
                        # Sum this timedelta (integer number of nanoseconds) to the
                        # previous element from int_timescale, and append the result to the list:
                        int_timescale.append((timedelta + int_timescale[(i-1)]))
                    
                    # Now convert the new scale (that preserves the distance between timestamps)
                    # to NumPy array:
                    int_timescale = np.array(int_timescale)
                    
                    # Divide by 10**9 to obtain the distances in seconds, reducing the order of
                    # magnitude of the integer numbers (the division is allowed for arrays)
                    int_timescale = int_timescale / (10**9)
                    
                    # Finally, use this timescale to obtain the polynomial fit;
                    # Use the method .fit, passing X, Y, and degree as coefficients
                    # to fit the polynomial to data:
                    # Perform the least squares fit to data:
                    #create an instance (object) named 'pol' from the class Polynomial:
                    fitted_pol = Polynomial.fit(int_timescale, y, deg = polynomial_degree, full = False)
                    
                
                else:
                    # Obtain the polynomial fitting object directly from x. Since x is not a
                    # datetime object, we can calculate the regression directly on it:
                    fitted_pol = Polynomial.fit(x, y, deg = polynomial_degree, full = False)
    
                # when full = True, the [resid, rank, sv, rcond] list is returned
                # check: https://numpy.org/doc/stable/reference/generated/numpy.polynomial.polynomial.Polynomial.fit.html#numpy.polynomial.polynomial.Polynomial.fit

                # This method returned a series named 'fitted_pol', with the values of Y predicted by the
                # polynomial fitting. Now add it to the dictionary as fitted_polynomial_series:
                dictionary['fitted_polynomial'] = fitted_pol
                print(f"{polynomial_degree} degree polynomial successfully fitted to data using the least squares method. The fitting Y ({lab}) values were added to the dataframe as the column \'fitted_polynomial\'.")    
                
                # Get the polynomial coefficients array:
                if (x_is_datetime):
                    coeff_array = Polynomial.fit(int_timescale, y, deg = polynomial_degree, full = False).coef
                
                else:
                    coeff_array = Polynomial.fit(x, y, deg = polynomial_degree, full = False).coef
                
                # Create a coefficient dictionary:
                coeff_dict = {}
                # Loop through each element from the array, and append it to the dictionary:
                full_polynomial_format = "a0"
                
                for i in range(1, len(coeff_array)):
                    
                        full_polynomial_format = full_polynomial_format + " + a" + str(i) + "*x" + str(i)
                
                coeff_dict['full_polynomial_format'] = full_polynomial_format
                
                for i in range(0, len(coeff_array)):
                    
                    # Create a key for the element i:
                    dict_key = "a" + str(i)
                    # Store the correspondent element on the array:
                    coeff_dict[dict_key] = coeff_array[i]
                
                if (x_is_datetime):
                    
                    coeff_dict['warning'] = "x is a numeric scale that was obtained from datetimes, preserving the distance relationships. It was obtained for allowing the polynomial fitting."
                    coeff_dict['numeric_to_datetime_correlation'] = {
                        
                        'x = 0': x[0],
                        f'x = {max(int_timescale)}': x[(len(x) - 1)]
                        
                    }
                    
                    coeff_dict['sequence_of_floats_correspondent_to_timestamps'] = {
                                                                                    'original_timestamps': x,
                                                                                    'sequence_of_floats': int_timescale
                                                                                    }
                
                print("Polynomial summary:\n")
                print(f"Polynomial format = {full_polynomial_format}\n")
                print("Polynomial coefficients:")
                
                for i in range(0, len(coeff_array)):
                    print(f"{str('a') + str(i)} = {coeff_array[i]}")
                
                print(f"Fitted polynomial = {dictionary['fitted_polynomial']}")
                print("\n")
                
                if (x_is_datetime):
                    print(coeff_dict['warning'])
                    print(coeff_dict['numeric_to_datetime_correlation'])
                    print("\n")
                
                # Add it to the main dictionary:
                dictionary['fitted_polynomial_coefficients'] = coeff_dict
                
                # Now, calculate the fitted series. Start a Pandas series as a copy from y:
                fitted_series = y.copy()
                # Make it zero:
                fitted_series = 0
                
                # Now loop through the polynomial coefficients: ai*(x**i):
                for i in range(0, len(coeff_array)):
                    
                    if (x_is_datetime):
                        
                        fitted_series = (coeff_array[i])*(int_timescale**(i))
                    
                    else:
                        fitted_series = (coeff_array[i])*(x**(i))
                
                # Add the series to the dictionary:
                dictionary['fitted_polynomial_series'] = fitted_series
                
                
                if (calculate_derivative == True):
            
                    # Calculate the derivative of the polynomial:
                    if (x_is_datetime):
                        pol_deriv = Polynomial.fit(int_timescale, y, deg = polynomial_degree, full = False).deriv(m = 1)
                    
                    else:
                        pol_deriv = Polynomial.fit(x, y, deg = polynomial_degree, full = False).deriv(m = 1)
                    # m (integer): order of the derivative. If m = 2, the second order is returned.
                    # This method returns a series. Check:
                    # https://numpy.org/doc/stable/reference/generated/numpy.polynomial.polynomial.Polynomial.deriv.html#numpy.polynomial.polynomial.Polynomial.deriv

                    #Add pol_deriv series as a new key from the dictionary:
                    dictionary['fitted_polynomial_derivative'] = pol_deriv
                    print("1st Order derivative of the polynomial successfully calculated and added to the dictionary as the key \'fitted_polynomial_derivative\'.\n")

                if (calculate_integral == True):

                    # Calculate the integral of the polynomial:
                    if (x_is_datetime):
                        pol_integral = Polynomial.fit(int_timescale, y, deg = polynomial_degree, full = False).integ(m = 1)
                    
                    else:
                        pol_integral = Polynomial.fit(x, y, deg = polynomial_degree, full = False).integ(m = 1)
                    # m (integer): The number of integrations to perform.
                    # This method returns a series. Check:
                    # https://numpy.org/doc/stable/reference/generated/numpy.polynomial.polynomial.Polynomial.integ.html#numpy.polynomial.polynomial.Polynomial.integ

                    #Add pol_deriv series as a new key from the dictionary:
                    dictionary['fitted_polynomial_integral'] = pol_integral
                    print("Integral of the polynomial successfully calculated and added to the dictionary as the key \'fitted_polynomial_integral\'.\n")

                if (calculate_roots == True):

                    # Calculate the roots of the polynomial:
                    if (x_is_datetime):
                        roots_array = Polynomial.fit(int_timescale, y, deg = polynomial_degree, full = False).roots()
                    
                    else:
                        roots_array = Polynomial.fit(x, y, deg = polynomial_degree, full = False).roots()
                    # This method returns an array with the polynomial roots. Check:
                    # https://numpy.org/doc/stable/reference/generated/numpy.polynomial.polynomial.Polynomial.roots.html#numpy.polynomial.polynomial.Polynomial.roots

                    #Add it as the key polynomial_roots:
                    dictionary['polynomial_roots'] = roots_array
                    print(f"Roots of the polynomial: {roots_array}.\n")
                    print("Roots added to the dictionary as the key \'polynomial_roots\'.\n")

                # Finally, append this dictionary to list support_list:
                list_of_dictionaries_with_series_and_predictions.append(dictionary)
            
            print("Returning a list of dictionaries. Each one contains the arrays of valid series and labels, as well as the regression statistics obtained.\n")
            
            # Now we finished the loop, list_of_dictionaries_with_series_and_predictions 
            # contains all series converted to NumPy arrays, with timestamps parsed as datetimes, 
            # and all the information regarding the linear regression, including the predicted 
            # values for plotting.
            # This list will be the object returned at the end of the function. Since it is an
            # JSON-formatted list, we can use the function json_obj_to_pandas_dataframe to convert
            # it to a Pandas dataframe.
            
            # Now, we can plot the figure.
            # we set alpha = 0.95 (opacity) to give a degree of transparency (5%), 
            # so that one series do not completely block the visualization of the other.
            
            # Let's retrieve the list of Matplotlib CSS colors:
            css4 = mcolors.CSS4_COLORS
            # css4 is a dictionary of colors: {'aliceblue': '#F0F8FF', 'antiquewhite': '#FAEBD7', ...}
            # Each key of this dictionary is a color name to be passed as argument color on the plot
            # function. So let's retrieve the array of keys, and use the list attribute to convert this
            # array to a list of colors:
            list_of_colors = list(css4.keys())
            
            # In 11 May 2022, this list of colors had 148 different elements
            # Since this list is in alphabetic order, let's create a random order for the colors.
            
            # Function random.sample(input_sequence, number_of_samples): 
            # this function creates a list containing a total of elements equals to the parameter 
            # "number_of_samples", which must be an integer.
            # This list is obtained by ramdomly selecting a total of "number_of_samples" elements from the
            # list "input_sequence" passed as parameter.
            
            # Function random.choices(input_sequence, k = number_of_samples):
            # similarly, randomly select k elements from the sequence input_sequence. This function is
            # newer than random.sample
            # Since we want to simply randomly sort the sequence, we can pass k = len(input_sequence)
            # to obtain the randomly sorted sequence:
            list_of_colors = random.choices(list_of_colors, k = len(list_of_colors))
            # Now, we have a random list of colors to use for plotting the charts
            
            if (add_splines_lines == True):
                LINE_STYLE = '-'

            else:
                LINE_STYLE = ''
            
            # Matplotlib linestyle:
            # https://matplotlib.org/stable/gallery/lines_bars_and_markers/linestyles.html?msclkid=68737f24d16011eca9e9c4b41313f1ad
            
            if (plot_title is None):
                # Set graphic title
                plot_title = f"Y_x_X"

            if (horizontal_axis_title is None):
                # Set horizontal axis title
                horizontal_axis_title = "X"

            if (vertical_axis_title is None):
                # Set vertical axis title
                vertical_axis_title = "Y"
            
            # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
            # so that the bars do not completely block other views.
            OPACITY = 0.95
            
            #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
            fig = plt.figure(figsize = (12, 8))
            ax = fig.add_subplot()

            i = 0 # Restart counting for the loop of colors
            
            # Loop through each dictionary from list_of_dictionaries_with_series_and_predictions:
            for dictionary in list_of_dictionaries_with_series_and_predictions:
                
                # Try selecting a color from list_of_colors:
                try:
                    
                    COLOR = list_of_colors[i]
                    # Go to the next element i, so that the next plot will use a different color:
                    i = i + 1
                
                except IndexError:
                    
                    # This error will be raised if list index is out of range, 
                    # i.e. if i >= len(list_of_colors) - we used all colors from the list (at least 148).
                    # So, return the index to zero to restart the colors from the beginning:
                    i = 0
                    COLOR = list_of_colors[i]
                    i = i + 1
                
                # Access the arrays and label from the dictionary:
                X = dictionary['x']
                Y = dictionary['y']
                LABEL = dictionary['lab']
                
                # Scatter plot:
                ax.plot(X, Y, linestyle = LINE_STYLE, marker = "o", color = COLOR, alpha = OPACITY, label = LABEL)
                # Axes.plot documentation:
                # https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.plot.html?msclkid=42bc92c1d13511eca8634a2c93ab89b5
                
                # x and y are positional arguments: they are specified by their position in function
                # call, not by an argument name like 'marker'.
                
                # Matplotlib markers:
                # https://matplotlib.org/stable/api/markers_api.html?msclkid=36c5eec5d16011ec9583a5777dc39d1f
                
                if (show_polynomial_reg == True):
                    
                    # Plot the linear regression using the same color.
                    # Access the array of fitted Y's in the dictionary:
                    Y_PRED = dictionary['fitted_polynomial_series']
                    Y_PRED_LABEL = 'fitted_pol_' + str(LABEL) # for the case where label is numeric
                    
                    ax.plot(X, Y_PRED,  linestyle = '-', marker = '', color = COLOR, alpha = OPACITY, label = Y_PRED_LABEL)

            # Now we finished plotting all of the series, we can set the general configuration:
            
            #ROTATE X AXIS IN XX DEGREES
            plt.xticks(rotation = x_axis_rotation)
            # XX = 0 DEGREES x_axis (Default)
            #ROTATE Y AXIS IN XX DEGREES:
            plt.yticks(rotation = y_axis_rotation)
            # XX = 0 DEGREES y_axis (Default)
            
            ax.set_title(plot_title)
            ax.set_xlabel(horizontal_axis_title)
            ax.set_ylabel(vertical_axis_title)

            ax.grid(grid) # show grid or not
            ax.legend(loc = 'upper left')
            # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
            # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
            # https://www.statology.org/matplotlib-legend-position/

            if (export_png == True):
                # Image will be exported
                import os

                #check if the user defined a directory path. If not, set as the default root path:
                if (directory_to_save is None):
                    #set as the default
                    directory_to_save = ""

                #check if the user defined a file name. If not, set as the default name for this
                # function.
                if (file_name is None):
                    #set as the default
                    file_name = "polynomial_fitting"

                #check if the user defined an image resolution. If not, set as the default 110 dpi
                # resolution.
                if (png_resolution_dpi is None):
                    #set as 330 dpi
                    png_resolution_dpi = 330

                #Get the new_file_path
                new_file_path = os.path.join(directory_to_save, file_name)

                #Export the file to this new path:
                # The extension will be automatically added by the savefig method:
                plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                #quality could be set from 1 to 100, where 100 is the best quality
                #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                #transparent = True or False
                # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

            #fig.tight_layout()

            ## Show an image read from an image file:
            ## import matplotlib.image as pltimg
            ## img=pltimg.imread('mydecisiontree.png')
            ## imgplot = plt.imshow(img)
            ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
            ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
            ##  '03_05_END.ipynb'
            plt.show()
            
            return list_of_dictionaries_with_series_and_predictions


    def time_series_vis (data_in_same_column = False, df = None, column_with_predict_var_x = None, column_with_response_var_y = None, column_with_labels = None, list_of_dictionaries_with_series_to_analyze = [{'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}], x_axis_rotation = 70, y_axis_rotation = 0, grid = True, add_splines_lines = True, add_scatter_dots = False, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import random
        # Python Random documentation:
        # https://docs.python.org/3/library/random.html?msclkid=9d0c34b2d13111ec9cfa8ddaee9f61a1
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import matplotlib.colors as mcolors
        
        # matplotlib.colors documentation:
        # https://matplotlib.org/3.5.0/api/colors_api.html?msclkid=94286fa9d12f11ec94660321f39bf47f
        
        # Matplotlib list of colors:
        # https://matplotlib.org/stable/gallery/color/named_colors.html?msclkid=0bb86abbd12e11ecbeb0a2439e5b0d23
        # Matplotlib colors tutorial:
        # https://matplotlib.org/stable/tutorials/colors/colors.html
        # Matplotlib example of Python code using matplotlib.colors:
        # https://matplotlib.org/stable/_downloads/0843ee646a32fc214e9f09328c0cd008/colors.py
        # Same example as Jupyter Notebook:
        # https://matplotlib.org/stable/_downloads/2a7b13c059456984288f5b84b4b73f45/colors.ipynb
        
            
        # data_in_same_column = False: set as True if all the values to plot are in a same column.
        # If data_in_same_column = True, you must specify the dataframe containing the data as df;
        # the column containing the predict variable (X) as column_with_predict_var_x; the column 
        # containing the responses to plot (Y) as column_with_response_var_y; and the column 
        # containing the labels (subgroup) indication as column_with_labels. 
        # df is an object, so do not declare it in quotes. The other three arguments (columns' names) 
        # are strings, so declare in quotes. 
        
        # Example: suppose you have a dataframe saved as dataset, and two groups A and B to compare. 
        # All the results for both groups are in a column named 'results', wich will be plot against
        # the time, saved as 'time' (X = 'time'; Y = 'results'). If the result is for
        # an entry from group A, then a column named 'group' has the value 'A'. If it is for group B,
        # column 'group' shows the value 'B'. In this example:
        # data_in_same_column = True,
        # df = dataset,
        # column_with_predict_var_x = 'time',
        # column_with_response_var_y = 'results', 
        # column_with_labels = 'group'
        # If you want to declare a list of dictionaries, keep data_in_same_column = False and keep
        # df = None (the other arguments may be set as None, but it is not mandatory: 
        # column_with_predict_var_x = None, column_with_response_var_y = None, column_with_labels = None).
        

        # Parameter to input when DATA_IN_SAME_COLUMN = False:
        # list_of_dictionaries_with_series_to_analyze: if data is already converted to series, lists
        # or arrays, provide them as a list of dictionaries. It must be declared as a list, in brackets,
        # even if there is a single dictionary.
        # Use always the same keys: 'x' for the X-series (predict variables); 'y' for the Y-series
        # (response variables); and 'lab' for the labels. If you do not want to declare a series, simply
        # keep as None, but do not remove or rename a key (ALWAYS USE THE KEYS SHOWN AS MODEL).
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to plot more series.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'x': x_series, 'y': y_series, 'lab': label}, where x_series, y_series and label
        # represents the series and label of the added dictionary (you can pass 'lab': None, but if 
        # 'x' or 'y' are None, the new dictionary will be ignored).
        
        # Examples:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'x': DATASET['X'], 'y': DATASET['Y'], 'lab': 'label'}]
        # will plot a single variable. In turns:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'x': DATASET['X'], 'y': DATASET['Y1'], 'lab': 'label'}, {'x': DATASET['X'], 'y': DATASET['Y2'], 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}]
        # will plot two series, Y1 x X and Y2 x X.
        # Notice that all dictionaries where 'x' or 'y' are None are automatically ignored.
        # If None is provided to 'lab', an automatic label will be generated.
        
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        if (data_in_same_column == True):
            
            print("Data to be plotted in a same column.\n")
            
            if (df is None):
                
                print("Please, input a valid dataframe as df.\n")
                list_of_dictionaries_with_series_to_analyze = []
                # The code will check the size of this list on the next block.
                # If it is zero, code is simply interrupted.
                # Instead of returning an error, we use this code structure that can be applied
                # on other graphic functions that do not return a summary (and so we should not
                # return a value like 'error' to interrupt the function).
            
            elif (column_with_predict_var_x is None):
                
                print("Please, input a valid column name as column_with_predict_var_x.\n")
                list_of_dictionaries_with_series_to_analyze = []
            
            elif (column_with_response_var_y is None):
                
                print("Please, input a valid column name as column_with_response_var_y.\n")
                list_of_dictionaries_with_series_to_analyze = []
            
            else:
                
                # set a local copy of the dataframe:
                DATASET = df.copy(deep = True)
                
                if (column_with_labels is None):
                
                    print("Using the whole series (column) for correlation.\n")
                    column_with_labels = 'whole_series_' + column_with_response_var_y
                    DATASET[column_with_labels] = column_with_labels
                
                # sort DATASET; by column_with_predict_var_x; by column column_with_labels
                # and by column_with_response_var_y, all in Ascending order
                # Since we sort by label (group), it is easier to separate the groups.
                DATASET = DATASET.sort_values(by = [column_with_predict_var_x, column_with_labels, column_with_response_var_y], ascending = [True, True, True])
                
                # Reset indices:
                DATASET = DATASET.reset_index(drop = True)
                
                # If column_with_predict_var_x is an object, the user may be trying to pass a date as x. 
                # So, let's try to convert it to datetime:
                if ((DATASET[column_with_predict_var_x]).dtype not in numeric_dtypes):
                    
                    try:
                        DATASET[column_with_predict_var_x] = (DATASET[column_with_predict_var_x]).astype('datetime64[ns]')
                        print("Variable X successfully converted to datetime64[ns].\n")
                        
                    except:
                        # Simply ignore it
                        pass
                
                # Get a series of unique values of the labels, and save it as a list using the
                # list attribute:
                unique_labels = list(DATASET[column_with_labels].unique())
                print(f"{len(unique_labels)} different labels detected: {unique_labels}.\n")
                
                # Start a list to store the dictionaries containing the keys:
                # 'x': list of predict variables; 'y': list of responses; 'lab': the label (group)
                list_of_dictionaries_with_series_to_analyze = []
                
                # Loop through each possible label:
                for lab in unique_labels:
                    # loop through each element from the list unique_labels, referred as lab
                    
                    # Set a filter for the dataset, to select only rows correspondent to that
                    # label:
                    boolean_filter = (DATASET[column_with_labels] == lab)
                    
                    # Create a copy of the dataset, with entries selected by that filter:
                    ds_copy = (DATASET[boolean_filter]).copy(deep = True)
                    # Sort again by X and Y, to guarantee the results are in order:
                    ds_copy = ds_copy.sort_values(by = [column_with_predict_var_x, column_with_response_var_y], ascending = [True, True])
                    # Restart the index of the copy:
                    ds_copy = ds_copy.reset_index(drop = True)
                    
                    # Re-extract the X and Y series and convert them to NumPy arrays 
                    # (these arrays will be important later in the function):
                    x = np.array(ds_copy[column_with_predict_var_x])
                    y = np.array(ds_copy[column_with_response_var_y])
                
                    # Then, create the dictionary:
                    dict_of_values = {'x': x, 'y': y, 'lab': lab}
                    
                    # Now, append dict_of_values to list_of_dictionaries_with_series_to_analyze:
                    list_of_dictionaries_with_series_to_analyze.append(dict_of_values)
                    
                # Now, we have a list of dictionaries with the same format of the input list.
                
        else:
            
            # The user input a list_of_dictionaries_with_series_to_analyze
            # Create a support list:
            support_list = []
            
            # Loop through each element on the list list_of_dictionaries_with_series_to_analyze:
            
            for i in range (0, len(list_of_dictionaries_with_series_to_analyze)):
                # from i = 0 to i = len(list_of_dictionaries_with_series_to_analyze) - 1, index of the
                # last element from the list
                
                # pick the i-th dictionary from the list:
                dictionary = list_of_dictionaries_with_series_to_analyze[i]
                
                # access 'x', 'y', and 'lab' keys from the dictionary:
                x = dictionary['x']
                y = dictionary['y']
                lab = dictionary['lab']
                # Remember that all this variables are series from a dataframe, so we can apply
                # the astype function:
                # https://www.askpython.com/python/built-in-methods/python-astype?msclkid=8f3de8afd0d411ec86a9c1a1e290f37c
                
                # check if at least x and y are not None:
                if ((x is not None) & (y is not None)):
                    
                    # If column_with_predict_var_x is an object, the user may be trying to pass a date as x. 
                    # So, let's try to convert it to datetime:
                    if (x.dtype not in numeric_dtypes):

                        try:
                            x = (x).astype('datetime64[ns]')
                            print(f"Variable X from {i}-th dictionary successfully converted to datetime64[ns].\n")

                        except:
                            # Simply ignore it
                            pass
                    
                    # Possibly, x and y are not ordered. Firstly, let's merge them into a temporary
                    # dataframe to be able to order them together.
                    # Use the 'list' attribute to guarantee that x and y were read as lists. These lists
                    # are the values for a dictionary passed as argument for the constructor of the
                    # temporary dataframe. When using the list attribute, we make the series independent
                    # from its origin, even if it was created from a Pandas dataframe. Then, we have a
                    # completely independent dataframe that may be manipulated and sorted, without worrying
                    # that it may modify its origin:
                    
                    temp_df = pd.DataFrame(data = {'x': list(x), 'y': list(y)})
                    # sort this dataframe by 'x' and 'y':
                    temp_df = temp_df.sort_values(by = ['x', 'y'], ascending = [True, True])
                    # restart index:
                    temp_df = temp_df.reset_index(drop = True)
                    
                    # Re-extract the X and Y series and convert them to NumPy arrays 
                    # (these arrays will be important later in the function):
                    x = np.array(temp_df['x'])
                    y = np.array(temp_df['y'])
                    
                    # check if lab is None:
                    if (lab is None):
                        # input a default label.
                        # Use the str attribute to convert the integer to string, allowing it
                        # to be concatenated
                        lab = "X" + str(i) + "_x_" + "Y" + str(i)
                        
                    # Then, create the dictionary:
                    dict_of_values = {'x': x, 'y': y, 'lab': lab}
                    
                    # Now, append dict_of_values to support list:
                    support_list.append(dict_of_values)
                
            # Now, support_list contains only the dictionaries with valid entries, as well
            # as labels for each collection of data. The values are independent from their origin,
            # and now they are ordered and in the same format of the data extracted directly from
            # the dataframe.
            # So, make the list_of_dictionaries_with_series_to_analyze the support_list itself:
            list_of_dictionaries_with_series_to_analyze = support_list
            print(f"{len(list_of_dictionaries_with_series_to_analyze)} valid series input.\n")

            
        # Now that both methods of input resulted in the same format of list, we can process both
        # with the same code.
        
        # Each dictionary in list_of_dictionaries_with_series_to_analyze represents a series to
        # plot. So, the total of series to plot is:
        total_of_series = len(list_of_dictionaries_with_series_to_analyze)
        
        if (total_of_series <= 0):
            
            print("No valid series to plot. Please, provide valid arguments.\n")
        
        else:
            
            # Continue to plotting and calculating the fitting.
            # Notice that we sorted the all the lists after they were separated and before
            # adding them to dictionaries. Also, the timestamps were converted to datetime64 variables
            # Now we finished the loop, list_of_dictionaries_with_series_to_analyze 
            # contains all series converted to NumPy arrays, with timestamps parsed as datetimes.
            # This list will be the object returned at the end of the function. Since it is an
            # JSON-formatted list, we can use the function json_obj_to_pandas_dataframe to convert
            # it to a Pandas dataframe.
            
            
            # Now, we can plot the figure.
            # we set alpha = 0.95 (opacity) to give a degree of transparency (5%), 
            # so that one series do not completely block the visualization of the other.
            
            # Let's retrieve the list of Matplotlib CSS colors:
            css4 = mcolors.CSS4_COLORS
            # css4 is a dictionary of colors: {'aliceblue': '#F0F8FF', 'antiquewhite': '#FAEBD7', ...}
            # Each key of this dictionary is a color name to be passed as argument color on the plot
            # function. So let's retrieve the array of keys, and use the list attribute to convert this
            # array to a list of colors:
            list_of_colors = list(css4.keys())
            
            # In 11 May 2022, this list of colors had 148 different elements
            # Since this list is in alphabetic order, let's create a random order for the colors.
            
            # Function random.sample(input_sequence, number_of_samples): 
            # this function creates a list containing a total of elements equals to the parameter 
            # "number_of_samples", which must be an integer.
            # This list is obtained by ramdomly selecting a total of "number_of_samples" elements from the
            # list "input_sequence" passed as parameter.
            
            # Function random.choices(input_sequence, k = number_of_samples):
            # similarly, randomly select k elements from the sequence input_sequence. This function is
            # newer than random.sample
            # Since we want to simply randomly sort the sequence, we can pass k = len(input_sequence)
            # to obtain the randomly sorted sequence:
            list_of_colors = random.choices(list_of_colors, k = len(list_of_colors))
            # Now, we have a random list of colors to use for plotting the charts
            
            if (add_splines_lines == True):
                LINE_STYLE = '-'

            else:
                LINE_STYLE = ''
            
            if (add_scatter_dots == True):
                MARKER = 'o'
                
            else:
                MARKER = ''
            
            # Matplotlib linestyle:
            # https://matplotlib.org/stable/gallery/lines_bars_and_markers/linestyles.html?msclkid=68737f24d16011eca9e9c4b41313f1ad
            
            if (plot_title is None):
                # Set graphic title
                plot_title = f"Y_x_timestamp"

            if (horizontal_axis_title is None):
                # Set horizontal axis title
                horizontal_axis_title = "timestamp"

            if (vertical_axis_title is None):
                # Set vertical axis title
                vertical_axis_title = "Y"
            
            # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
            # so that the bars do not completely block other views.
            OPACITY = 0.95
            
            #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
            fig = plt.figure(figsize = (12, 8))
            ax = fig.add_subplot()

            i = 0 # Restart counting for the loop of colors
            
            # Loop through each dictionary from list_of_dictionaries_with_series_and_predictions:
            for dictionary in list_of_dictionaries_with_series_to_analyze:
                
                # Try selecting a color from list_of_colors:
                try:
                    
                    COLOR = list_of_colors[i]
                    # Go to the next element i, so that the next plot will use a different color:
                    i = i + 1
                
                except IndexError:
                    
                    # This error will be raised if list index is out of range, 
                    # i.e. if i >= len(list_of_colors) - we used all colors from the list (at least 148).
                    # So, return the index to zero to restart the colors from the beginning:
                    i = 0
                    COLOR = list_of_colors[i]
                    i = i + 1
                
                # Access the arrays and label from the dictionary:
                X = dictionary['x']
                Y = dictionary['y']
                LABEL = dictionary['lab']
                
                # Scatter plot:
                ax.plot(X, Y, linestyle = LINE_STYLE, marker = MARKER, color = COLOR, alpha = OPACITY, label = LABEL)
                # Axes.plot documentation:
                # https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.plot.html?msclkid=42bc92c1d13511eca8634a2c93ab89b5
                
                # x and y are positional arguments: they are specified by their position in function
                # call, not by an argument name like 'marker'.
                
                # Matplotlib markers:
                # https://matplotlib.org/stable/api/markers_api.html?msclkid=36c5eec5d16011ec9583a5777dc39d1f
                
            # Now we finished plotting all of the series, we can set the general configuration:
            
            #ROTATE X AXIS IN XX DEGREES
            plt.xticks(rotation = x_axis_rotation)
            # XX = 0 DEGREES x_axis (Default)
            #ROTATE Y AXIS IN XX DEGREES:
            plt.yticks(rotation = y_axis_rotation)
            # XX = 0 DEGREES y_axis (Default)

            ax.set_title(plot_title)
            ax.set_xlabel(horizontal_axis_title)
            ax.set_ylabel(vertical_axis_title)

            ax.grid(grid) # show grid or not
            ax.legend(loc = 'upper left')
            # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
            # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
            # https://www.statology.org/matplotlib-legend-position/

            if (export_png == True):
                # Image will be exported
                import os

                #check if the user defined a directory path. If not, set as the default root path:
                if (directory_to_save is None):
                    #set as the default
                    directory_to_save = ""

                #check if the user defined a file name. If not, set as the default name for this
                # function.
                if (file_name is None):
                    #set as the default
                    file_name = "time_series_vis"

                #check if the user defined an image resolution. If not, set as the default 110 dpi
                # resolution.
                if (png_resolution_dpi is None):
                    #set as 330 dpi
                    png_resolution_dpi = 330

                #Get the new_file_path
                new_file_path = os.path.join(directory_to_save, file_name)

                #Export the file to this new path:
                # The extension will be automatically added by the savefig method:
                plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                #quality could be set from 1 to 100, where 100 is the best quality
                #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                #transparent = True or False
                # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

            #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
            #plt.figure(figsize = (12, 8))
            #fig.tight_layout()

            ## Show an image read from an image file:
            ## import matplotlib.image as pltimg
            ## img=pltimg.imread('mydecisiontree.png')
            ## imgplot = plt.imshow(img)
            ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
            ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
            ##  '03_05_END.ipynb'
            plt.show()


    def histogram (df, column_to_analyze, total_of_bins = 10, normal_curve_overlay = True, x_axis_rotation = 0, y_axis_rotation = 0, grid = True, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        
        # column_to_analyze: string with the name of the column that will be analyzed.
        # column_to_analyze = 'col1' obtain a histogram from column 1.
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        # Sort by the column to analyze (ascending order) and reset the index:
        DATASET = DATASET.sort_values(by = column_to_analyze, ascending = True)
        
        DATASET = DATASET.reset_index(drop = True)
        
        # Create an instance (object) from class capability_analysis:
        capability_obj = capability_analysis(df = DATASET, column_with_variable_to_be_analyzed = column_to_analyze, specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': None}, total_of_bins = total_of_bins)
        
        # Get histogram array:
        capability_obj = capability_obj.get_histogram_array()
        # Attribute .histogram_dict: dictionary with keys 'list_of_bins' and 'list_of_counts'.
        
        # Get fitted normal:
        capability_obj = capability_obj.get_fitted_normal()
        # Now the .specification_limits attribute contains the nested dict desired_normal = {'x': x_of_normal, 'y': y_normal}
        # in key 'fitted_normal'.
        
        # Get the actual probability density function (PDF):
        capability_obj = capability_obj.get_actual_pdf()
        # Now the dictionary in the attribute .specification_limits has the nested dict actual_pdf = {'x': array_to_analyze, 'y': array_of_probs}
        # in key 'actual_pdf'.
        
        # Retrieve general statistics:
        stats_dict = {
            
            'sample_size': capability_obj.sample_size,
            'mu': capability_obj.mu,
            'median': capability_obj.median,
            'sigma': capability_obj.sigma,
            'lowest': capability_obj.lowest,
            'highest': capability_obj.highest
        }
        
        # Retrieve the histogram dict:
        histogram_dict = capability_obj.histogram_dict
        
        # Retrieve the specification limits dictionary updated:
        specification_limits = capability_obj.specification_limits
        # Retrieve the desired normal and actual PDFs dictionaries:
        fitted_normal = specification_limits['fitted_normal']
        actual_pdf = specification_limits['actual_pdf']
        
        string_for_title = " - $\mu = %.2f$, $\sigma = %.2f$" %(stats_dict['mu'], stats_dict['sigma'])
        
        if not (plot_title is None):
            plot_title = plot_title + string_for_title
            # %.2f: the number between . and f indicates the number of printed decimal cases
            # the notation $\ - Latex code for printing formatted equations and symbols.
        
        else:
            # Set graphic title
            plot_title = f"histogram_of_{column_to_analyze}" + string_for_title

        if (horizontal_axis_title is None):
            # Set horizontal axis title
            horizontal_axis_title = column_to_analyze

        if (vertical_axis_title is None):
            # Set vertical axis title
            vertical_axis_title = "Counting/Frequency"
            
        # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
        # so that the bars do not completely block other views.
        OPACITY = 0.95
        
        y_hist = DATASET[column_to_analyze]
        
        # Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
        fig = plt.figure(figsize = (12, 8))
        ax = fig.add_subplot()
        
        #STANDARD MATPLOTLIB METHOD:
        #bins = number of bins (intervals) of the histogram. Adjust it manually
        #increasing bins will increase the histogram's resolution, but height of bars
        
        ax.hist(y_hist, bins = total_of_bins, alpha = OPACITY, label = f'counting_of\n{column_to_analyze}', color = 'darkblue')
        #ax.hist(y, bins=20, width = bar_width, label=xlabel, color='blue')
        #IF GRAPHIC IS NOT SHOWN: THAT IS BECAUSE THE DISTANCES BETWEEN VALUES ARE LOW, AND YOU WILL
        #HAVE TO USE THE STANDARD HISTOGRAM METHOD FROM MATPLOTLIB.
        #TO DO THAT, UNMARK LINE ABOVE: ax.hist(y, bins=20, width = bar_width, label=xlabel, color='blue')
        #AND MARK LINE BELOW AS COMMENT: ax.bar(xhist, yhist, width = bar_width, label=xlabel, color='blue')
        
        #IF YOU WANT TO CREATE GRAPHIC AS A BAR CHART BASED ON THE CALCULATED DISTRIBUTION TABLE, 
        #MARK THE LINE ABOVE AS COMMENT AND UNMARK LINE BELOW:
        #ax.bar(x_hist, y_hist, label = f'counting_of\n{column_to_analyze}', color = 'darkblue')
        #ajuste manualmente a largura, width, para deixar as barras mais ou menos proximas
        
        # Plot the probability density function for the data:
        pdf_x = actual_pdf['x']
        pdf_y = actual_pdf['y']
        
        ax.plot(pdf_x, pdf_y, color = 'darkgreen', linestyle = '-', alpha = OPACITY, label = 'probability\ndensity')
        
        if (normal_curve_overlay == True):
            
            # Check if a normal curve was obtained:
            x_of_normal = fitted_normal['x']
            y_normal = fitted_normal['y']

            if (len(x_of_normal) > 0):
                # Non-empty list, add the normal curve:
                ax.plot(x_of_normal, y_normal, color = 'crimson', linestyle = 'dashed', alpha = OPACITY, label = 'expected\nnormal_curve')

        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 0 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)

        ax.set_title(plot_title)
        ax.set_xlabel(horizontal_axis_title)
        ax.set_ylabel(vertical_axis_title)

        ax.grid(grid) # show grid or not
        ax.legend(loc = 'upper right')
        # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
        # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
        # https://www.statology.org/matplotlib-legend-position/

        if (export_png == True):
            # Image will be exported
            import os

            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""

            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "histogram"

            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330

            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)

            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

        #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
        #plt.figure(figsize = (12, 8))
        #fig.tight_layout()

        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        
        stats_dict = {
                    'statistics': ['mean', 'median', 'standard_deviation', f'lowest_{column_to_analyze}', 
                                    f'highest_{column_to_analyze}', 'count_of_values', 'number_of_bins', 
                                    'bin_size', 'bin_of_max_proba', 'count_on_bin_of_max_proba'],
                    'value': [stats_dict['mu'], stats_dict['median'], stats_dict['sigma'], 
                                stats_dict['lowest'], stats_dict['highest'], stats_dict['sample_size'], 
                                histogram_dict['number_of_bins'], histogram_dict['bin_size'], 
                                histogram_dict['bin_of_max_proba'], histogram_dict['max_count']]
                    }
        
        # Convert it to a Pandas dataframe setting the list 'statistics' as the index:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
        general_stats = pd.DataFrame(data = stats_dict)
        
        # Set the column 'statistics' as the index of the dataframe, using set_index method:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.set_index.html
        
        # If inplace = True, modifies the DataFrame in place (do not create a new object).
        # Then, we do not create an object equal to the expression. We simply apply the method (so,
        # None is returned from the method):
        general_stats.set_index(['statistics'], inplace = True)
        
        print("Check the general statistics from the analyzed variable:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(general_stats)
                
        except: # regular mode
            print(general_stats)
        
        print("\n")
        print("Check the frequency table:\n")
        
        freq_table = histogram_dict['df']
        
        try:    
            display(freq_table)    
        except:
            print(freq_table)

        return general_stats, freq_table


    def test_data_normality (df, column_to_analyze, column_with_labels_to_test_subgroups = None, alpha = 0.10, show_probability_plot = True, x_axis_rotation = 0, y_axis_rotation = 0, grid = True, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from statsmodels.stats import diagnostic
        from scipy import stats
        # Check https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.probplot.html#scipy.stats.probplot
        # Check https://docs.scipy.org/doc/scipy/tutorial/stats.html
        # Check https://docs.scipy.org/doc/scipy-1.8.0/html-scipyorg/reference/generated/scipy.stats.normaltest.html
        
        # WARNING: The statistical tests require at least 20 samples
        
        # column_to_analyze: column (variable) of the dataset that will be tested. Declare as a string,
        # in quotes.
        # e.g. column_to_analyze = 'col1' will analyze a column named 'col1'.
        
        # column_with_labels_to_test_subgroups: if there is a column with labels or
        # subgroup indication, and the normality should be tested separately for each label, indicate
        # it here as a string (in quotes). e.g. column_with_labels_to_test_subgroups = 'col2' 
        # will retrieve the labels from 'col2'.
        # Keep column_with_labels_to_test_subgroups = None if a single series (the whole column)
        # will be tested.
        
        # Confidence level = 1 - ALPHA. For ALPHA = 0.10, we get a 0.90 = 90% confidence
        # Set ALPHA = 0.05 to get 0.95 = 95% confidence in the analysis.
        # Notice that, when less trust is needed, we can increase ALPHA to get less restrictive
        # results.
        
        print("WARNING: The statistical tests require at least 20 samples.\n")
        print("Interpretation:")
        print("p-value: probability that data is described by the normal distribution.")
        print("Criterion: the series is not described by normal if p < alpha = %.3f." %(alpha))
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        # Start a list to store the different Pandas series to test:
        list_of_dicts = []
        
        if not (column_with_labels_to_test_subgroups is None):
            
            # 1. Get the unique values from column_with_labels_to_test_subgroups
            # and save it as the list labels_list:
            labels_list = list(DATASET[column_with_labels_to_test_subgroups].unique())
            
            # 2. Loop through each element from labels_list:
            for label in labels_list:
                
                # 3. Create a copy of the DATASET, filtering for entries where 
                # column_with_labels_to_test_subgroups == label:
                filtered_df = (DATASET[DATASET[column_with_labels_to_test_subgroups] == label]).copy(deep = True)
                # 4. Reset index of the copied dataframe:
                filtered_df = filtered_df.reset_index(drop = True)
                # 5. Create a dictionary, with an identification of the series, and the series
                # that will be tested:
                series_dict = {'series_id': (column_to_analyze + "_" + label), 
                            'series': filtered_df[column_to_analyze],
                            'total_elements_to_test': len(filtered_df[column_to_analyze])}
                
                # 6. Append this dictionary to the list of series:
                list_of_dicts.append(series_dict)
            
        else:
            # In this case, the only series is the column itself. So, let's create a dictionary with
            # same structure:
            series_dict = {'series_id': column_to_analyze, 'series': DATASET[column_to_analyze],
                        'total_elements_to_test': len(DATASET[column_to_analyze])}
            
            # Append this dictionary to the list of series:
            list_of_dicts.append(series_dict)
        
        
        # Now, loop through each element from the list of series:
        
        for series_dict in list_of_dicts:
            
            # start a support list:
            support_list = []
            
            # Check if there are at least 20 samples to test:
            series_id = series_dict['series_id']
            total_elements_to_test = series_dict['total_elements_to_test']
            
            # Create an instance (object) from class capability_analysis:
            capability_obj = capability_analysis(df = DATASET, column_with_variable_to_be_analyzed = series_id, specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': None}, alpha = alpha)
            
            # Check data normality:
            capability_obj = capability_obj.check_data_normality()
            # Attribute .normality_dict: dictionary with results from normality tests
            
            # Retrieve the normality dictionary:
            normality_dict = capability_obj.normality_dict
            # Nest it in series_dict:
            series_dict['normality_dict'] = normality_dict
            
            # Finally, append the series dictionary to the support list:
            support_list.append(series_dict)
            
            if ((total_elements_to_test >= 20) & (show_probability_plot == True)):
                
                y = series_dict['series']
            
                print("\n")
                #Obtain the probability plot  
                fig, ax = plt.subplots(figsize = (12, 8))

                ax.set_title(f"probability_plot_of_{series_id}_for_normal_distribution")
                
                plot_results = stats.probplot(y, dist = 'norm', fit = True, plot = ax)
                #This function resturns a tuple, so we must store it into res
                
                ax.grid(grid)
                #ROTATE X AXIS IN XX DEGREES
                plt.xticks(rotation = x_axis_rotation)
                # XX = 70 DEGREES x_axis (Default)
                #ROTATE Y AXIS IN XX DEGREES:
                plt.yticks(rotation = y_axis_rotation)
                # XX = 0 DEGREES y_axis (Default)   
                
                # Other distributions to check, see scipy Stats documentation. 
                # you could test dist=stats.loggamma, where stats was imported from scipy
                # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.probplot.html#scipy.stats.probplot

                if (export_png == True):
                    # Image will be exported
                    import os

                    #check if the user defined a directory path. If not, set as the default root path:
                    if (directory_to_save is None):
                        #set as the default
                        directory_to_save = ""

                    #check if the user defined a file name. If not, set as the default name for this
                    # function.
                    if (file_name is None):
                        #set as the default
                        file_name = "probability_plot_normal"

                    #check if the user defined an image resolution. If not, set as the default 110 dpi
                    # resolution.
                    if (png_resolution_dpi is None):
                        #set as 330 dpi
                        png_resolution_dpi = 330

                    #Get the new_file_path
                    new_file_path = os.path.join(directory_to_save, file_name)

                    #Export the file to this new path:
                    # The extension will be automatically added by the savefig method:
                    plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                    #quality could be set from 1 to 100, where 100 is the best quality
                    #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                    #transparent = True or False
                    # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                    print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

                #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
                #plt.figure(figsize = (12, 8))
                #fig.tight_layout()
                ## Show an image read from an image file:
                ## import matplotlib.image as pltimg
                ## img=pltimg.imread('mydecisiontree.png')
                ## imgplot = plt.imshow(img)
                ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
                ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
                ##  '03_05_END.ipynb'
                plt.show()
                    
                print("\n")
                
        # Now we left the for loop, make the list of dicts support list itself:
        list_of_dicts = support_list
        
        print("\n")
        print("Finished normality tests. Returning a list of dictionaries, where each dictionary contains the series analyzed and the p-values obtained.\n")
        print("Now, check general statistics of the data distribution:\n")
        
        # Now, let's obtain general statistics for all of the series, even those without the normality
        # test results.
        
        # start a support list:
        support_list = []
        
        for series_dict in list_of_dicts:
            
            y = series_dict['series']
            # Guarantee it is still a pandas series:
            y = pd.Series(y)
            # Calculate data skewness and kurtosis
        
            # Skewness
            data_skew = stats.skew(y)
            # skewness = 0 : normally distributed.
            # skewness > 0 : more weight in the left tail of the distribution.
            # skewness < 0 : more weight in the right tail of the distribution.
            # https://www.geeksforgeeks.org/scipy-stats-skew-python/

            # Kurtosis
            data_kurtosis = stats.kurtosis(y, fisher = True)
            # scipy.stats.kurtosis(array, axis=0, fisher=True, bias=True) function 
            # calculates the kurtosis (Fisher or Pearson) of a data set. It is the the fourth 
            # central moment divided by the square of the variance. 
            # It is a measure of the “tailedness” i.e. descriptor of shape of probability 
            # distribution of a real-valued random variable. 
            # In simple terms, one can say it is a measure of how heavy tail is compared 
            # to a normal distribution.
            # fisher parameter: fisher : Bool; Fisher’s definition is used (normal 0.0) if True; 
            # else Pearson’s definition is used (normal 3.0) if set to False.
            # https://www.geeksforgeeks.org/scipy-stats-kurtosis-function-python/
            print("A normal distribution should present no skewness (distribution distortion); and no kurtosis (long-tail).\n")
            print("For the data analyzed:\n")
            print(f"skewness = {data_skew}")
            print(f"kurtosis = {data_kurtosis}\n")

            if (data_skew < 0):

                print(f"Skewness = {data_skew} < 0: more weight in the left tail of the distribution.")

            elif (data_skew > 0):

                print(f"Skewness = {data_skew} > 0: more weight in the right tail of the distribution.")

            else:

                print(f"Skewness = {data_skew} = 0: no distortion of the distribution.")
                    

            if (data_kurtosis == 0):

                print("Data kurtosis = 0. No long-tail effects detected.\n")

            else:

                print(f"The kurtosis different from zero indicates long-tail effects on the distribution.\n")

            #Calculate the mode of the distribution:
            # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.mode.html
            data_mode = stats.mode(y, axis = None)[0][0]
            # returns an array of arrays. The first array is called mode=array and contains the mode.
            # Axis: Default is 0. If None, compute over the whole array.
            # we set axis = None to compute the general mode.

            #Create general statistics dictionary:
            general_statistics_dict = {

                "series_mean": y.mean(),
                "series_variance": y.var(),
                "series_standard_deviation": y.std(),
                "series_skewness": data_skew,
                "series_kurtosis": data_kurtosis,
                "series_mode": data_mode

            }
            
            # Add this dictionary to the series dictionary:
            series_dict['general_statistics'] = general_statistics_dict
            
            # Append the dictionary to support list:
            support_list.append(series_dict)
        
        # Now, make the list of dictionaries support_list itself:
        list_of_dicts = support_list

        return list_of_dicts


    def test_stat_distribution (df, column_to_analyze, column_with_labels_to_test_subgroups = None, statistical_distribution_to_test = 'lognormal'):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from statsmodels.stats import diagnostic
        from scipy import stats
        # Check https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.probplot.html#scipy.stats.probplot
        # Check https://docs.scipy.org/doc/scipy/tutorial/stats.html
        # Check https://docs.scipy.org/doc/scipy-1.8.0/html-scipyorg/reference/generated/scipy.stats.normaltest.html
        
        # column_to_analyze: column (variable) of the dataset that will be tested. Declare as a string,
        # in quotes.
        # e.g. column_to_analyze = 'col1' will analyze a column named 'col1'.
        
        # column_with_labels_to_test_subgroups: if there is a column with labels or
        # subgroup indication, and the normality should be tested separately for each label, indicate
        # it here as a string (in quotes). e.g. column_with_labels_to_test_subgroups = 'col2' 
        # will retrieve the labels from 'col2'.
        # Keep column_with_labels_to_test_subgroups = None if a single series (the whole column)
        # will be tested.
        
        # Attention: if you want to test a normal distribution, use the function 
        # test_data_normality. Function test_data_normality tests normality through 4 methods 
        # and compare them: D’Agostino and Pearson’s; Shapiro-Wilk; Lilliefors; 
        # and Anderson-Darling tests.
        # The calculus of the p-value from the Anderson-Darling statistic is available only 
        # for some distributions. The function specific for the normality calculates these 
        # probabilities of following the normal.
        # Here, the function is destined to test a variety of distributions, and so only the 
        # Anderson-Darling test is performed.
            
        # statistical_distribution: string (inside quotes) containing the tested statistical 
        # distribution.
        # Notice: if data Y follow a 'lognormal', log(Y) follow a normal
        # Poisson is a special case from 'gamma' distribution.
        ## There are 91 accepted statistical distributions:
        # 'alpha', 'anglit', 'arcsine', 'beta', 'beta_prime', 'bradford', 'burr', 'burr12', 
        # 'cauchy', 'chi', 'chi-squared', 'cosine', 'double_gamma', 
        # 'double_weibull', 'erlang', 'exponential', 'exponentiated_weibull', 'exponential_power',
        # 'fatigue_life_birnbaum-saunders', 'fisk_log_logistic', 'folded_cauchy', 'folded_normal',
        # 'F', 'gamma', 'generalized_logistic', 'generalized_pareto', 'generalized_exponential', 
        # 'generalized_extreme_value', 'generalized_gamma', 'generalized_half-logistic', 
        # 'generalized_inverse_gaussian', 'generalized_normal', 
        # 'gilbrat', 'gompertz_truncated_gumbel', 'gumbel', 'gumbel_left-skewed', 'half-cauchy', 
        # 'half-normal', 'half-logistic', 'hyperbolic_secant', 'gauss_hypergeometric', 
        # 'inverted_gamma', 'inverse_normal', 'inverted_weibull', 'johnson_SB', 'johnson_SU', 
        # 'KSone', 'KStwobign', 'laplace', 'left-skewed_levy', 
        # 'levy', 'logistic', 'log_laplace', 'log_gamma', 'lognormal', 'log-uniform', 'maxwell', 
        # 'mielke_Beta-Kappa', 'nakagami', 'noncentral_chi-squared', 'noncentral_F', 
        # 'noncentral_t', 'normal', 'normal_inverse_gaussian', 'pareto', 'lomax', 
        # 'power_lognormal', 'power_normal', 'power-function', 'R', 'rayleigh', 'rice', 
        # 'reciprocal_inverse_gaussian', 'semicircular', 'student-t', 
        # 'triangular', 'truncated_exponential', 'truncated_normal', 'tukey-lambda',
        # 'uniform', 'von_mises', 'wald', 'weibull_maximum_extreme_value', 
        # 'weibull_minimum_extreme_value', 'wrapped_cauchy'
        
        print("WARNING: The statistical tests require at least 20 samples.\n")
        print("Attention: if you want to test a normal distribution, use the function test_data_normality.")
        print("Function test_data_normality tests normality through 4 methods and compare them: D’Agostino and Pearson’s; Shapiro-Wilk; Lilliefors; and Anderson-Darling tests.")
        print("The calculus of the p-value from the Anderson-Darling statistic is available only for some distributions.")
        print("The function which specifically tests the normality calculates these probabilities that data follows the normal.")
        print("Here, the function is destined to test a variety of distributions, and so only the Anderson-Darling test is performed.\n")
        
        print("If a compilation error is shown below, please update your Scipy version. Declare and run the following code into a separate cell:")
        print("! pip install scipy --upgrade\n")
        
        # Lets define the statistic distributions:
        # This are the callable Scipy objects which can be tested through Anderson-Darling test:
        # They are listed and explained in: 
        # https://docs.scipy.org/doc/scipy/tutorial/stats/continuous.html
        
        # This dictionary correlates the input name of the distribution to the correct scipy.stats
        # callable object
        # There are 91 possible statistical distributions:
        
        callable_statistical_distributions_dict = {
            
            'alpha': stats.alpha, 'anglit': stats.anglit, 'arcsine': stats.arcsine,
            'beta': stats.beta, 'beta_prime': stats.betaprime, 'bradford': stats.bradford,
            'burr': stats.burr, 'burr12': stats.burr12, 'cauchy': stats.cauchy,
            'chi': stats.chi, 'chi-squared': stats.chi2,
            'cosine': stats.cosine, 'double_gamma': stats.dgamma, 'double_weibull': stats.dweibull,
            'erlang': stats.erlang, 'exponential': stats.expon, 'exponentiated_weibull': stats.exponweib,
            'exponential_power': stats.exponpow, 'fatigue_life_birnbaum-saunders': stats.fatiguelife,
            'fisk_log_logistic': stats.fisk, 'folded_cauchy': stats.foldcauchy,
            'folded_normal': stats.foldnorm, 'F': stats.f, 'gamma': stats.gamma,
            'generalized_logistic': stats.genlogistic, 'generalized_pareto': stats.genpareto,
            'generalized_exponential': stats.genexpon, 'generalized_extreme_value': stats.genextreme,
            'generalized_gamma': stats.gengamma, 'generalized_half-logistic': stats.genhalflogistic,
            'generalized_inverse_gaussian': stats.geninvgauss,
            'generalized_normal': stats.gennorm, 'gilbrat': stats.gilbrat,
            'gompertz_truncated_gumbel': stats.gompertz, 'gumbel': stats.gumbel_r,
            'gumbel_left-skewed': stats.gumbel_l, 'half-cauchy': stats.halfcauchy, 'half-normal': stats.halfnorm,
            'half-logistic': stats.halflogistic, 'hyperbolic_secant': stats.hypsecant,
            'gauss_hypergeometric': stats.gausshyper, 'inverted_gamma': stats.invgamma,
            'inverse_normal': stats.invgauss, 'inverted_weibull': stats.invweibull,
            'johnson_SB': stats.johnsonsb, 'johnson_SU': stats.johnsonsu, 'KSone': stats.ksone,
            'KStwobign': stats.kstwobign, 'laplace': stats.laplace,
            'left-skewed_levy': stats.levy_l,
            'levy': stats.levy, 'logistic': stats.logistic, 'log_laplace': stats.loglaplace,
            'log_gamma': stats.loggamma, 'lognormal': stats.lognorm, 'log-uniform': stats.loguniform,
            'maxwell': stats.maxwell, 'mielke_Beta-Kappa': stats.mielke, 'nakagami': stats.nakagami,
            'noncentral_chi-squared': stats.ncx2, 'noncentral_F': stats.ncf, 'noncentral_t': stats.nct,
            'normal': stats.norm, 'normal_inverse_gaussian': stats.norminvgauss, 'pareto': stats.pareto,
            'lomax': stats.lomax, 'power_lognormal': stats.powerlognorm, 'power_normal': stats.powernorm,
            'power-function': stats.powerlaw, 'R': stats.rdist, 'rayleigh': stats.rayleigh,
            'rice': stats.rayleigh, 'reciprocal_inverse_gaussian': stats.recipinvgauss,
            'semicircular': stats.semicircular,
            'student-t': stats.t, 'triangular': stats.triang,
            'truncated_exponential': stats.truncexpon, 'truncated_normal': stats.truncnorm,
            'tukey-lambda': stats.tukeylambda, 'uniform': stats.uniform, 'von_mises': stats.vonmises,
            'wald': stats.wald, 'weibull_maximum_extreme_value': stats.weibull_max,
            'weibull_minimum_extreme_value': stats.weibull_min, 'wrapped_cauchy': stats.wrapcauchy
                    
        }
        
        # Get a list of keys from this dictionary, to compare with the selected string:
        list_of_dictionary_keys = callable_statistical_distributions_dict.keys()
        
        #check if an invalid string was provided using the in method:
        # The string must be in the list of dictionary keys
        boolean_filter = statistical_distribution_to_test in list_of_dictionary_keys
        # if it is the list, boolean_filter == True. If it is not, boolean_filter == False
        
        if (boolean_filter == False):
            
            print(f"Please, select a valid statistical distribution to test: {list_of_dictionary_keys}")
            return "error"
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        # Start a list to store the different Pandas series to test:
        list_of_dicts = []
        
        if not (column_with_labels_to_test_subgroups is None):
            
            # 1. Get the unique values from column_with_labels_to_test_subgroups
            # and save it as the list labels_list:
            labels_list = list(DATASET[column_with_labels_to_test_subgroups].unique())
            
            # 2. Loop through each element from labels_list:
            for label in labels_list:
                
                # 3. Create a copy of the DATASET, filtering for entries where 
                # column_with_labels_to_test_subgroups == label:
                filtered_df = (DATASET[DATASET[column_with_labels_to_test_subgroups] == label]).copy(deep = True)
                # 4. Reset index of the copied dataframe:
                filtered_df = filtered_df.reset_index(drop = True)
                # 5. Create a dictionary, with an identification of the series, and the series
                # that will be tested:
                series_dict = {'series_id': (column_to_analyze + "_" + label), 
                            'series': filtered_df[column_to_analyze],
                            'total_elements_to_test': filtered_df[column_to_analyze].count()}
                
                # 6. Append this dictionary to the list of series:
                list_of_dicts.append(series_dict)
            
        else:
            # In this case, the only series is the column itself. So, let's create a dictionary with
            # same structure:
            series_dict = {'series_id': column_to_analyze, 'series': DATASET[column_to_analyze],
                        'total_elements_to_test': DATASET[column_to_analyze].count()}
            
            # Append this dictionary to the list of series:
            list_of_dicts.append(series_dict)
        
        
        # Now, loop through each element from the list of series:
        
        for series_dict in list_of_dicts:
            
            # start a support list:
            support_list = []
            
            # Check if there are at least 20 samples to test:
            series_id = series_dict['series_id']
            total_elements_to_test = series_dict['total_elements_to_test']
            
            if (total_elements_to_test < 20):
                
                print(f"Unable to test series {series_id}: at least 20 samples are needed, but found only {total_elements_to_test} entries for this series.\n")
                # Add a warning to the dictionary:
                series_dict['WARNING'] = "Series without the minimum number of elements (20) required to test the normality."
                # Append it to the support list:
                support_list.append(series_dict)
                
            else:
                # Let's test the series.
                y = series_dict['series']
                
                # Calculate data skewness and kurtosis
                # Skewness
                data_skew = stats.skew(y)
                # skewness = 0 : normally distributed.
                # skewness > 0 : more weight in the left tail of the distribution.
                # skewness < 0 : more weight in the right tail of the distribution.
                # https://www.geeksforgeeks.org/scipy-stats-skew-python/

                # Kurtosis
                data_kurtosis = stats.kurtosis(y, fisher = True)
                # scipy.stats.kurtosis(array, axis=0, fisher=True, bias=True) function 
                # calculates the kurtosis (Fisher or Pearson) of a data set. It is the the fourth 
                # central moment divided by the square of the variance. 
                # It is a measure of the “tailedness” i.e. descriptor of shape of probability 
                # distribution of a real-valued random variable. 
                # In simple terms, one can say it is a measure of how heavy tail is compared 
                # to a normal distribution.
                # fisher parameter: fisher : Bool; Fisher’s definition is used (normal 0.0) if True; 
                # else Pearson’s definition is used (normal 3.0) if set to False.
                # https://www.geeksforgeeks.org/scipy-stats-kurtosis-function-python/
                print("A normal distribution should present no skewness (distribution distortion); and no kurtosis (long-tail).\n")
                print("For the data analyzed:\n")
                print(f"skewness = {data_skew}")
                print(f"kurtosis = {data_kurtosis}\n")

                if (data_skew < 0):

                    print(f"Skewness = {data_skew} < 0: more weight in the left tail of the distribution.")

                elif (data_skew > 0):

                    print(f"Skewness = {data_skew} > 0: more weight in the right tail of the distribution.")

                else:

                    print(f"Skewness = {data_skew} = 0: no distortion of the distribution.")
                    

                if (data_kurtosis == 0):

                    print("Data kurtosis = 0. No long-tail effects detected.\n")

                else:

                    print(f"The kurtosis different from zero indicates long-tail effects on the distribution.\n")

                #Calculate the mode of the distribution:
                # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.mode.html
                data_mode = stats.mode(y, axis = None)[0][0]
                # returns an array of arrays. The first array is called mode=array and contains the mode.
                # Axis: Default is 0. If None, compute over the whole array.
                # we set axis = None to compute the general mode.
                
                # Access the object correspondent to the distribution provided. To do so,
                # simply access dict['key1'], where 'key1' is a key from a dictionary dict ={"key1": 'val1'}
                # Access just as accessing a column from a dataframe.

                anderson_darling_statistic = diagnostic.anderson_statistic(y, dist = (callable_statistical_distributions_dict[statistical_distribution_to_test]), fit = True)
                
                print(f"Anderson-Darling statistic for the distribution {statistical_distribution_to_test} = {anderson_darling_statistic}\n")
                print("The Anderson–Darling test assesses whether a sample comes from a specified distribution.")
                print("It makes use of the fact that, when given a hypothesized underlying distribution and assuming the data does arise from this distribution, the cumulative distribution function (CDF) of the data can be assumed to follow a uniform distribution.")
                print("Then, data can be tested for uniformity using an appropriate distance test (Shapiro 1980).\n")
                # source: https://en.wikipedia.org/wiki/Anderson%E2%80%93Darling_test

                # Fit the distribution and get its parameters
                # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.rv_continuous.fit.html

                distribution_parameters = callable_statistical_distributions_dict[statistical_distribution_to_test].fit(y)
                
                # With method="MLE" (default), the fit is computed by minimizing the negative 
                # log-likelihood function. A large, finite penalty (rather than infinite negative 
                # log-likelihood) is applied for observations beyond the support of the distribution. 
                # With method="MM", the fit is computed by minimizing the L2 norm of the relative errors 
                # between the first k raw (about zero) data moments and the corresponding distribution 
                # moments, where k is the number of non-fixed parameters. 

                # distribution_parameters: Estimates for any shape parameters (if applicable), 
                # followed by those for location and scale.
                print(f"Distribution shape parameters calculated for {statistical_distribution_to_test} = {distribution_parameters}\n")
                
                print("ATTENTION:\n")
                print("The critical values for the Anderson-Darling test are dependent on the specific distribution that is being tested.")
                print("Tabulated values and formulas have been published (Stephens, 1974, 1976, 1977, 1979) for a few specific distributions: normal, lognormal, exponential, Weibull, logistic, extreme value type 1.")
                print("The test consists on an one-sided test of the hypothesis that the distribution is of a specific form.")
                print("The hypothesis is rejected if the test statistic, A, is greater than the critical value for that particular distribution.")
                print("Note that, for a given distribution, the Anderson-Darling statistic may be multiplied by a constant which usually depends on the sample size, n).")
                print("These constants are given in the various papers by Stephens, and may be simply referred as the \'adjusted Anderson-Darling statistic\'.")
                print("This adjusted statistic is what should be compared against the critical values.")
                print("Also, be aware that different constants (and therefore critical values) have been published.")
                print("Therefore, you just need to be aware of what constant was used for a given set of critical values (the needed constant is typically given with the correspondent critical values).")
                print("To learn more about the Anderson-Darling statistics and the check the full references of Stephens, go to the webpage from the National Institute of Standards and Technology (NIST):\n")
                print("https://itl.nist.gov/div898/handbook/eda/section3/eda3e.htm")
                print("\n")
                
                #Create general statistics dictionary:
                general_statistics_dict = {

                    "series_mean": y.mean(),
                    "series_variance": y.var(),
                    "series_standard_deviation": y.std(),
                    "series_skewness": data_skew,
                    "series_kurtosis": data_kurtosis,
                    "series_mode": data_mode,
                    "AndersonDarling_statistic_A": anderson_darling_statistic,
                    "distribution_parameters": distribution_parameters

                }

                # Add this dictionary to the series dictionary:
                series_dict['general_statistics'] = general_statistics_dict
    
                # Now, append the series dictionary to the support list:
                support_list.append(series_dict)
            
        # Now we left the for loop, make the list of dicts support list itself:
        list_of_dicts = support_list
        
        print("General statistics successfully returned in the list \'list_of_dicts\'.\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(list_of_dicts)
                
        except: # regular mode
            print(list_of_dicts)
        
        print("\n")
        
        print("Note: the obtention of the probability plot specific for each distribution requires shape parameters.")
        print("Check Scipy documentation for additional information:")
        print("https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.probplot.html")
        
        return list_of_dicts


    def select_order_or_rename_columns (df, columns_list, mode = 'select_or_order_columns'):
        
        import numpy as np
        import pandas as pd
        
        # MODE = 'select_or_order_columns' for filtering only the list of columns passed as columns_list,
        # and setting a new column order. In this mode, you can pass the columns in any order: 
        # the order of elements on the list will be the new order of columns.

        # MODE = 'rename_columns' for renaming the columns with the names passed as columns_list. In this
        # mode, the list must have same length and same order of the columns of the dataframe. That is because
        # the columns will sequentially receive the names in the list. So, a mismatching of positions
        # will result into columns with incorrect names.
        
        # columns_list = list of strings containing the names (headers) of the columns to select
        # (filter); or to be set as the new columns' names, according to the selected mode.
        # For instance: columns_list = ['col1', 'col2', 'col3'] will 
        # select columns 'col1', 'col2', and 'col3' (or rename the columns with these names). 
        # Declare the names inside quotes.
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        print(f"Original columns in the dataframe:\n{DATASET.columns}\n")
        
        if ((columns_list is None) | (columns_list == np.nan)):
            # empty list
            columns_list = []
        
        if (len(columns_list) == 0):
            print("Please, input a valid list of columns.\n")
            return DATASET
        
        if (mode == 'select_or_order_columns'):
            
            #filter the dataframe so that it will contain only the cols_list.
            DATASET = DATASET[columns_list]
            print("Dataframe filtered according to the list provided.\n")
            print("Check the new dataframe:\n")
            
            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(DATASET)

            except: # regular mode
                print(DATASET)
            
        elif (mode == 'rename_columns'):
            
            # Check if the number of columns of the dataset is equal to the number of elements
            # of the new list. It will avoid raising an exception error.
            boolean_filter = (len(columns_list) == len(DATASET.columns))
            
            if (boolean_filter == False):
                #Impossible to rename, number of elements are different.
                print("The number of columns of the dataframe is different from the number of elements of the list. Please, provide a list with number of elements equals to the number of columns.\n")
                return DATASET
            
            else:
                #Same number of elements, so that we can update the columns' names.
                DATASET.columns = columns_list
                print("Dataframe columns renamed according to the list provided.\n")
                print("Warning: the substitution is element-wise: the first element of the list is now the name of the first column, and so on, ..., so that the last element is the name of the last column.\n")
                print("Check the new dataframe:\n")
                try:
                    # only works in Jupyter Notebook:
                    from IPython.display import display
                    display(DATASET)

                except: # regular mode
                    print(DATASET)
            
        else:
            print("Enter a valid mode: \'select_or_order_columns\' or \'rename_columns\'.")
            return DATASET
        
        return DATASET


    def rename_or_clean_columns_labels (df, mode = 'set_new_names', substring_to_be_replaced = ' ', new_substring_for_replacement = '_', trailing_substring = None, list_of_columns_labels = [{'column_name': None, 'new_column_name': None}, {'column_name': None, 'new_column_name': None}, {'column_name': None, 'new_column_name': None}, {'column_name': None, 'new_column_name': None}, {'column_name': None, 'new_column_name': None}, {'column_name': None, 'new_column_name': None}, {'column_name': None, 'new_column_name': None}, {'column_name': None, 'new_column_name': None}]):
        
        import numpy as np
        import pandas as pd
        # Pandas .rename method:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.rename.html
        
        # mode = 'set_new_names' will change the columns according to the specifications in
        # list_of_columns_labels.
        
        # list_of_columns_labels = [{'column_name': None, 'new_column_name': None}]
        # This is a list of dictionaries, where each dictionary contains two key-value pairs:
        # the first one contains the original column name; and the second one contains the new name
        # that will substitute the original one. The function will loop through all dictionaries in
        # this list, access the values of the keys 'column_name', and it will be replaced (switched) 
        # by the correspondent value in key 'new_column_name'.
        
        # The object list_of_columns_labels must be declared as a list, 
        # in brackets, even if there is a single dictionary.
        # Use always the same keys: 'column_name' for the original label; 
        # and 'new_column_name', for the correspondent new label.
        # Notice that this function will not search substrings: it will substitute a value only when
        # there is perfect correspondence between the string in 'column_name' and one of the columns
        # labels. So, the cases (upper or lower) must be the same.
        
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to replace more
        # values.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'column_name': original_col, 'new_column_name': new_col}, 
        # where original_col and new_col represent the strings for searching and replacement 
        # (If one of the keys contains None, the new dictionary will be ignored).
        # Example: list_of_columns_labels = [{'column_name': 'col1', 'new_column_name': 'col'}] will
        # rename 'col1' as 'col'.
        
        
        # mode = 'capitalize_columns' will capitalize all columns names (i.e., they will be put in
        # upper case). e.g. a column named 'column' will be renamed as 'COLUMN'
        
        # mode = 'lowercase_columns' will lower the case of all columns names. e.g. a column named
        # 'COLUMN' will be renamed as 'column'.
        
        # mode = 'replace_substring' will search on the columns names (strings) for the 
        # substring_to_be_replaced (which may be a character or a string); and will replace it by 
        # new_substring_for_replacement (which again may be either a character or a string). 
        # Numbers (integers or floats) will be automatically converted into strings.
        # As an example, consider the default situation where we search for a whitespace ' ' 
        # and replace it by underscore '_': 
        # substring_to_be_replaced = ' ', new_substring_for_replacement = '_'  
        # In this case, a column named 'new column' will be renamed as 'new_column'.
        
        # mode = 'trim' will remove all trailing or leading whitespaces from column names.
        # e.g. a column named as ' col1 ' will be renamed as 'col1'; 'col2 ' will be renamed as
        # 'col2'; and ' col3' will be renamed as 'col3'.
        
        # mode = 'eliminate_trailing_characters' will eliminate a defined trailing and leading 
        # substring from the columns' names. 
        # The substring must be indicated as trailing_substring, and its default, when no value
        # is provided, is equivalent to mode = 'trim' (eliminate white spaces). 
        # e.g., if trailing_substring = '_test' and you have a column named 'col_test', it will be 
        # renamed as 'col'.
        
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        # Guarantee that the columns were read as strings:
        DATASET.columns = (DATASET.columns).astype(str)
        # dataframe.columns is a Pandas Index object, so it has the dtype attribute as other Pandas
        # objects. So, we can use the astype method to set its type as str or 'object' (or "O").
        # Notice that there are situations with int Index, when integers are used as column names or
        # as row indices. So, this portion guarantees that we can call the str attribute to apply string
        # methods.
        
        if (mode == 'set_new_names'):
            
            # Start a mapping dictionary:
            mapping_dict = {}
            # This dictionary will be in the format required by .rename method: old column name as key,
            # and new name as value.

            # Loop through each element from list_of_columns_labels:
            for dictionary in list_of_columns_labels:

                # Access the values in keys:
                column_name = dictionary['column_name']
                new_column_name = dictionary['new_column_name']

                # Check if neither is None:
                if ((column_name is not None) & (new_column_name is not None)):
                    
                    # Guarantee that both were read as strings:
                    column_name = str(column_name)
                    new_column_name = str(new_column_name)

                    # Add it to the mapping dictionary setting column_name as key, and the new name as the
                    # value:
                    mapping_dict[column_name] = new_column_name

            # Now, the dictionary is in the correct format for the method. Let's apply it:
            DATASET.rename(columns = mapping_dict, inplace = True)
        
        elif (mode == 'capitalize_columns'):
            
            DATASET.rename(str.upper, axis = 'columns', inplace = True)
        
        elif (mode == 'lowercase_columns'):
            
            DATASET.rename(str.lower, axis = 'columns', inplace = True)
        
        elif (mode == 'replace_substring'):
            
            if (substring_to_be_replaced is None):
                # set as the default (whitespace):
                substring_to_be_replaced = ' '
            
            if (new_substring_for_replacement is None):
                # set as the default (underscore):
                new_substring_for_replacement = '_'
            
            # Apply the str attribute to guarantee that numbers were read as strings:
            substring_to_be_replaced = str(substring_to_be_replaced)
            new_substring_for_replacement = str(new_substring_for_replacement)
            # Replace the substrings in the columns' names:
            substring_replaced_series = (pd.Series(DATASET.columns)).str.replace(substring_to_be_replaced, new_substring_for_replacement)
            # The Index object is not callable, and applying the str attribute to a np.array or to a list
            # will result in a single string concatenating all elements from the array. So, we convert
            # the columns index to a pandas series for performing a element-wise string replacement.
            
            # Now, convert the columns to the series with the replaced substrings:
            DATASET.columns = substring_replaced_series
            
        elif (mode == 'trim'):
            # Use the strip method from str attribute with no argument, correspondening to the
            # Trim function.
            DATASET.rename(str.strip, axis = 'columns', inplace = True)
        
        elif (mode == 'eliminate_trailing_characters'):
            
            if ((trailing_substring is None) | (trailing_substring == np.nan)):
                # Apply the str.strip() with no arguments:
                DATASET.rename(str.strip, axis = 'columns', inplace = True)
            
            else:
                # Apply the str attribute to guarantee that numbers were read as strings:
                trailing_substring = str(trailing_substring)

                # Apply the strip method:
                stripped_series = (pd.Series(DATASET.columns)).str.strip(trailing_substring)
                # The Index object is not callable, and applying the str attribute to a np.array or to a list
                # will result in a single string concatenating all elements from the array. So, we convert
                # the columns index to a pandas series for performing a element-wise string replacement.

                # Now, convert the columns to the series with the stripped strings:
                DATASET.columns = stripped_series
        
        else:
            print("Select a valid mode: \'set_new_names\', \'capitalize_columns\', \'lowercase_columns\', \'replace_substrings\', \'trim\', or \'eliminate_trailing_characters\'.\n")
            return "error"
        
        print("Finished renaming dataframe columns.\n")
        print("Check the new dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET)
                
        except: # regular mode
            print(DATASET)
            
        return DATASET


    def trim_spaces_or_characters (df, column_to_analyze, new_variable_type = None, method = 'trim', substring_to_eliminate = None, create_new_column = True, new_column_suffix = "_trim"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # new_variable_type = None. String (in quotes) that represents a given data type for the column
        # after transformation. Set:
        # - new_variable_type = 'int' to convert the column to integer type after the transform;
        # - new_variable_type = 'float' to convert the column to float (decimal number);
        # - new_variable_type = 'datetime' to convert it to date or timestamp;
        # - new_variable_type = 'category' to convert it to Pandas categorical variable.
        
        # method = 'trim' will eliminate trailing and leading white spaces from the strings in
        # column_to_analyze.
        # method = 'substring' will eliminate a defined trailing and leading substring from
        # column_to_analyze.
        
        # substring_to_eliminate = None. Set as a string (in quotes) if method = 'substring'.
        # e.g. suppose column_to_analyze contains time information: each string ends in " min":
        # "1 min", "2 min", "3 min", etc. If substring_to_eliminate = " min", this portion will be
        # eliminated, resulting in: "1", "2", "3", etc. If new_variable_type = None, these values will
        # continue to be strings. By setting new_variable_type = 'int' or 'float', the series will be
        # converted to a numeric type.
        
        # create_new_column = True
        # Alternatively, set create_new_column = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_trim"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_column_suffix. Then, if the original
        # column was "column1" and the suffix is "_trim", the new column will be named as
        # "column1_trim".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        if (method == 'substring'):
            
            if (substring_to_eliminate is None):
                
                method = 'trim'
                print("No valid substring input. Modifying method to \'trim\'.\n")
        
        if (method == 'substring'):
            
            print("ATTENTION: Operations of string strip (removal) or replacement are all case-sensitive. There must be correct correspondence between cases and spaces for the strings being removed or replaced.\n")
            # For manipulating strings, call the str attribute and, then, the method to be applied:
            new_series = new_series.str.strip(substring_to_eliminate)
        
        else:
            
            new_series = new_series.str.strip()
        
        # Check if a the series type should be modified:
        if (new_variable_type is not None):
            
            if (new_variable_type == 'int'):

                new_type = np.int64

            elif (new_variable_type == 'float'):
                
                new_type = np.float64
            
            elif (new_variable_type == 'datetime'):
                
                new_type = np.datetime64
            
            elif (new_variable_type == 'category'):
                
                new_type = new_variable_type
            
            # Try converting the type:
            try:
                new_series = new_series.astype(new_type)
                print(f"Successfully converted the series to the type {new_variable_type}.\n")
            
            except:
                pass

        if (create_new_column):
            
            if (new_column_suffix is None):
                new_column_suffix = "_trim"
                    
            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:
            
            DATASET[column_to_analyze] = new_series
        
        # Now, we are in the main code.
        print("Finished removing leading and trailing spaces or characters (substrings).")
        print("Check the 10 first elements from the series:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))
                
        except: # regular mode
            print(new_series.head(10))
        
        return DATASET


    def capitalize_or_lower_string_case (df, column_to_analyze, method = 'lowercase', create_new_column = True, new_column_suffix = "_homogenized"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # method = 'capitalize' will capitalize all letters from the input string 
        # (turn them to upper case).
        # method = 'lowercase' will make the opposite: turn all letters to lower case.
        # e.g. suppose column_to_analyze contains strings such as 'String One', 'STRING 2',  and
        # 'string3'. If method = 'capitalize', the output will contain the strings: 
        # 'STRING ONE', 'STRING 2', 'STRING3'. If method = 'lowercase', the outputs will be:
        # 'string one', 'string 2', 'string3'.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_homogenized"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_column_suffix. Then, if the original
        # column was "column1" and the suffix is "_homogenized", the new column will be named as
        # "column1_homogenized".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        if (method == 'capitalize'):
            
            print("Capitalizing the string (moving all characters to upper case).\n")
            # For manipulating strings, call the str attribute and, then, the method to be applied:
            new_series = new_series.str.upper()
        
        else:
            
            print("Lowering the string case (moving all characters to lower case).\n")
            new_series = new_series.str.lower()
            
        if (create_new_column):
            
            if (new_column_suffix is None):
                new_column_suffix = "_homogenized"
                    
            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:
            
            DATASET[column_to_analyze] = new_series
        
        # Now, we are in the main code.
        print(f"Finished homogenizing the string case of {column_to_analyze}, giving value consistency.")
        print("Check the 10 first elements from the series:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))
                
        except: # regular mode
            print(new_series.head(10))
        
        return DATASET


    def add_contractions_to_library (list_of_contractions = [{'contracted_expression': None, 'correct_expression': None}, {'contracted_expression': None, 'correct_expression': None}, {'contracted_expression': None, 'correct_expression': None}, {'contracted_expression': None, 'correct_expression': None}]):
        
        import contractions
        # contractions library: https://github.com/kootenpv/contractions
        
        # list_of_contractions = 
        # [{'contracted_expression': None, 'correct_expression': None}]
        # This is a list of dictionaries, where each dictionary contains two key-value pairs:
        # the first one contains the form as the contraction is usually observed; and the second one 
        # contains the correct (full) string that will replace it.
        # Since contractions can cause issues when processing text, we can expand them with these functions.
        
        # The object list_of_contractions must be declared as a list, 
        # in brackets, even if there is a single dictionary.
        # Use always the same keys: 'contracted_expression' for the contraction; and 'correct_expression', 
        # for the strings with the correspondent correction.
        
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you want to add more elements
        # to the contractions library.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'contracted_expression': original_str, 'correct_expression': new_str}, 
        # where original_str and new_str represent the contracted and expanded strings
        # (If one of the keys contains None, the new dictionary will be ignored).
        
        # Example:
        # list_of_contractions = [{'contracted_expression': 'mychange', 'correct_expression': 'my change'}]
        
        
        for dictionary in list_of_contractions:
            
            contraction = dictionary['contracted_expression']
            correction = dictionary['correct_expression']
            
            if ((contraction is not None) & (correction is not None)):
        
                contractions.add(contraction, correction)
                print(f"Successfully included the contracted expression {contraction} to the contractions library.")

        print("Now, the function for contraction correction will be able to process it within the strings.\n")


    def correct_contracted_strings (df, column_to_analyze, create_new_column = True, new_column_suffix = "_contractionsFixed"):
        
        import numpy as np
        import pandas as pd
        import contractions
        
        # contractions library: https://github.com/kootenpv/contractions
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
    
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_contractionsFixed"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_contractionsFixed", the new column will be named as
        # "column1_contractionsFixed".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        # Contractions operate at one string at once:
        correct_contractions_list = [contractions.fix(new_series[i], slang = True) for i in range (0, len(DATASET))]
        
        # Make this list the new_series itself:
        new_series = pd.Series(correct_contractions_list)
        
        if (create_new_column):
                
            if (new_column_suffix is None):
                new_column_suffix = "_contractionsFixed"

            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:

            DATASET[column_to_analyze] = new_series

        # Now, we are in the main code.
        print(f"Finished correcting the contracted strings from column {column_to_analyze}.")
        print("Check the 10 first elements (10 lists) from the series:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))

        except: # regular mode
            print(new_series.head(10))

        return DATASET


    def replace_substring (df, column_to_analyze, substring_to_be_replaced = None, new_substring_for_replacement = '', create_new_column = True, new_column_suffix = "_substringReplaced"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # substring_to_be_replaced = None; new_substring_for_replacement = ''. 
        # Strings (in quotes): when the sequence of characters substring_to_be_replaced was
        # found in the strings from column_to_analyze, it will be substituted by the substring
        # new_substring_for_replacement. If None is provided to one of these substring arguments,
        # it will be substituted by the empty string: ''
        # e.g. suppose column_to_analyze contains the following strings, with a spelling error:
        # "my collumn 1", 'his collumn 2', 'her column 3'. We may correct this error by setting:
        # substring_to_be_replaced = 'collumn' and new_substring_for_replacement = 'column'. The
        # function will search for the wrong group of characters and, if it finds it, will substitute
        # by the correct sequence: "my column 1", 'his column 2', 'her column 3'.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_substringReplaced"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_column_suffix. Then, if the original
        # column was "column1" and the suffix is "_substringReplaced", the new column will be named as
        # "column1_substringReplaced".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        print("ATTENTION: Operations of string strip (removal) or replacement are all case-sensitive. There must be correct correspondence between cases and spaces for the strings being removed or replaced.\n")
            
        # If one of the input substrings is None, make it the empty string:
        if (substring_to_be_replaced is None):
            substring_to_be_replaced = ''
        
        if (new_substring_for_replacement is None):
            new_substring_for_replacement = ''
        
        # Guarantee that both were read as strings (they may have been improperly read as 
        # integers or floats):
        substring_to_be_replaced = str(substring_to_be_replaced)
        new_substring_for_replacement = str(new_substring_for_replacement)
        
        # For manipulating strings, call the str attribute and, then, the method to be applied:
        new_series = new_series.str.replace(substring_to_be_replaced, new_substring_for_replacement)
            
        if (create_new_column):
            
            if (new_column_suffix is None):
                new_column_suffix = "_substringReplaced"
                    
            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:
            
            DATASET[column_to_analyze] = new_series
        
        # Now, we are in the main code.
        print(f"Finished replacing the substring {substring_to_be_replaced} by {new_substring_for_replacement}.")
        print("Check the 10 first elements from the series:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))
                
        except: # regular mode
            print(new_series.head(10))
        
        return DATASET


    def invert_strings (df, column_to_analyze, create_new_column = True, new_column_suffix = "_stringInverted"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_stringInverted"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_stringInverted", the new column will be named as
        # "column1_stringInverted".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        # Pandas slice: start from -1 (last character) and go to the last element with -1 step
        # walk through the string 'backwards':
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.slice.html
        
        new_series = new_series.str.slice(start = -1, step = -1)
        
        if (create_new_column):
                
            if (new_column_suffix is None):
                new_column_suffix = "_stringInverted"

            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:

            DATASET[column_to_analyze] = new_series

        # Now, we are in the main code.
        print(f"Finished inversion of the strings.")
        print("Check the 10 first elements from the series:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))

        except: # regular mode
            print(new_series.head(10))

        return DATASET


    def slice_strings (df, column_to_analyze, first_character_index = None, last_character_index = None, step = 1, create_new_column = True, new_column_suffix = "_slicedString"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_slicedString"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_slicedString", the new column will be named as
        # "column1_slicedString".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        # first_character_index = None - integer representing the index of the first character to be
        # included in the new strings. If None, slicing will start from first character.
        # Indexing of strings always start from 0. The last index can be represented as -1, the index of
        # the character before as -2, etc (inverse indexing starts from -1).
        # example: consider the string "idsw", which contains 4 characters. We can represent the indices as:
        # 'i': index 0; 'd': 1, 's': 2, 'w': 3. Alternatively: 'w': -1, 's': -2, 'd': -3, 'i': -4.
        
        # last_character_index = None - integer representing the index of the last character to be
        # included in the new strings. If None, slicing will go until the last character.
        # Attention: this is effectively the last character to be added, and not the next index after last
        # character.
        
        # in the 'idsw' example, if we want a string as 'ds', we want the first_character_index = 1 and
        # last_character_index = 2.
        
        # step = 1 - integer representing the slicing step. If step = 1, all characters will be added.
        # If step = 2, then the slicing will pick one element of index i and the element with index (i+2)
        # (1 index will be 'jumped'), and so on.
        # If step is negative, then the order of the new strings will be inverted.
        # Example: step = -1, and the start and finish indices are None: the output will be the inverted
        # string, 'wsdi'.
        # first_character_index = 1, last_character_index = 2, step = 1: output = 'ds';
        # first_character_index = None, last_character_index = None, step = 2: output = 'is';
        # first_character_index = None, last_character_index = None, step = 3: output = 'iw';
        # first_character_index = -1, last_character_index = -2, step = -1: output = 'ws';
        # first_character_index = -1, last_character_index = None, step = -2: output = 'wd';
        # first_character_index = -1, last_character_index = None, step = 1: output = 'w'
        # In this last example, the function tries to access the next element after the character of index
        # -1. Since -1 is the last character, there are no other characters to be added.
        # first_character_index = -2, last_character_index = -1, step = 1: output = 'sw'.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        # Pandas slice:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.slice.html
        
        
        if (step is None):
            # set as 1
            step = 1
        
        if (last_character_index is not None):
            if (last_character_index == -1):
                # In this case, we cannot sum 1, because it would result in index 0 (1st character).
                # So, we will proceed without last index definition, to stop only at the end.
                last_character_index = None
        
        # Now, make the checking again:
                
        if ((first_character_index is None) & (last_character_index is None)):
            
            new_series = new_series.str.slice(step = step)
            
        elif (first_character_index is None):
            # Only this is None:
            new_series = new_series.str.slice(stop = (last_character_index + 1), step = step)
        
        elif (last_character_index is None):
            
            new_series = new_series.str.slice(start = first_character_index, step = step)
        
        else:
            
            new_series = new_series.str.slice(start = first_character_index, stop = (last_character_index + 1), step = step)
        
        # Slicing from index i to index j includes index i, but does not include 
        # index j (ends in j-1). So, we add 1 to the last index to include it.
        # automatically included.

        if (create_new_column):
                
            if (new_column_suffix is None):
                new_column_suffix = "_slicedString"

            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:

            DATASET[column_to_analyze] = new_series

        # Now, we are in the main code.
        print(f"Finished slicing the strings from character {first_character_index} to character {last_character_index}.")
        print("Check the 10 first elements from the series:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))

        except: # regular mode
            print(new_series.head(10))

        return DATASET


    def left_characters (df, column_to_analyze, number_of_characters_to_retrieve = 1, new_variable_type = None, create_new_column = True, new_column_suffix = "_leftChars"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_leftChars"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_leftChars", the new column will be named as
        # "column1_leftChars".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        # number_of_characters_to_retrieve = 1 - integer representing the total of characters that will
        # be retrieved. Here, we will retrieve the leftest characters. If number_of_characters_to_retrieve = 1,
        # only the leftest (last) character will be retrieved.
        # Consider the string 'idsw'.
        # number_of_characters_to_retrieve = 1 - output: 'w';
        # number_of_characters_to_retrieve = 2 - output: 'sw'.
        
        # new_variable_type = None. String (in quotes) that represents a given data type for the column
        # after transformation. Set:
        # - new_variable_type = 'int' to convert the extracted column to integer;
        # - new_variable_type = 'float' to convert the column to float (decimal number);
        # - new_variable_type = 'datetime' to convert it to date or timestamp;
        # - new_variable_type = 'category' to convert it to Pandas categorical variable.
        
        # So, if the last part of the strings is a number, you can use this argument to directly extract
        # this part as numeric variable.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        # Pandas slice:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.slice.html
        
        if (number_of_characters_to_retrieve is None):
            # set as 1
            number_of_characters_to_retrieve = 1
        
        # last_character_index = -1 would be the index of the last character.
        # If we want the last N = 2 characters, we should go from index -2 to -1, -2 = -1 - (N-1);
        # If we want the last N = 3 characters, we should go from index -3 to -1, -2 = -1 - (N-1);
        # If we want only the last (N = 1) character, we should go from -1 to -1, -1 = -1 - (N-1).
        
        # N = number_of_characters_to_retrieve
        first_character_index = -1 - (number_of_characters_to_retrieve - 1)
        
        # Perform the slicing without setting the limit, to slice until the end of the string:
        new_series = new_series.str.slice(start = first_character_index, step = 1)
        
        # Check if a the series type should be modified:
        if (new_variable_type is not None):
            
            if (new_variable_type == 'int'):

                new_type = np.int64

            elif (new_variable_type == 'float'):
                
                new_type = np.float64
            
            elif (new_variable_type == 'datetime'):
                
                new_type = np.datetime64
            
            elif (new_variable_type == 'category'):
                
                new_type = new_variable_type
            
            # Try converting the type:
            try:
                new_series = new_series.astype(new_type)
                print(f"Successfully converted the series to the type {new_variable_type}.\n")
            
            except:
                pass
        
        
        if (create_new_column):
                
            if (new_column_suffix is None):
                new_column_suffix = "_leftChars"

            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:

            DATASET[column_to_analyze] = new_series

        # Now, we are in the main code.
        print(f"Finished extracting the {number_of_characters_to_retrieve} leftest characters.")
        print("Check the 10 first elements from the series:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))

        except: # regular mode
            print(new_series.head(10))

        return DATASET


    def right_characters (df, column_to_analyze, number_of_characters_to_retrieve = 1, new_variable_type = None, create_new_column = True, new_column_suffix = "_rightChars"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_rightChars"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_rightChars", the new column will be named as
        # "column1_rightChars".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        # number_of_characters_to_retrieve = 1 - integer representing the total of characters that will
        # be retrieved. Here, we will retrieve the rightest characters. If number_of_characters_to_retrieve = 1,
        # only the rightest (first) character will be retrieved.
        # Consider the string 'idsw'.
        # number_of_characters_to_retrieve = 1 - output: 'i';
        # number_of_characters_to_retrieve = 2 - output: 'id'.
        
        # new_variable_type = None. String (in quotes) that represents a given data type for the column
        # after transformation. Set:
        # - new_variable_type = 'int' to convert the extracted column to integer;
        # - new_variable_type = 'float' to convert the column to float (decimal number);
        # - new_variable_type = 'datetime' to convert it to date or timestamp;
        # - new_variable_type = 'category' to convert it to Pandas categorical variable.
        
        # So, if the first part of the strings is a number, you can use this argument to directly extract
        # this part as numeric variable.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        # Pandas slice:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.slice.html
        
        if (number_of_characters_to_retrieve is None):
            # set as 1
            number_of_characters_to_retrieve = 1
        
        # first_character_index = 0 would be the index of the first character.
        # If we want the last N = 2 characters, we should go from index 0 to 1, 1 = (N-1);
        # If we want the last N = 3 characters, we should go from index 0 to 2, 2 = (N-1);
        # If we want only the last (N = 1) character, we should go from 0 to 0, 0 = (N-1).
        
        # N = number_of_characters_to_retrieve
        last_character_index = number_of_characters_to_retrieve - 1
        
        # Perform the slicing without setting the limit, to slice from the 1st character:
        new_series = new_series.str.slice(stop = (last_character_index + 1), step = 1)
        
        # Check if a the series type should be modified:
        if (new_variable_type is not None):
            
            if (new_variable_type == 'int'):

                new_type = np.int64

            elif (new_variable_type == 'float'):
                
                new_type = np.float64
            
            elif (new_variable_type == 'datetime'):
                
                new_type = np.datetime64
            
            elif (new_variable_type == 'category'):
                
                new_type = new_variable_type
            
            # Try converting the type:
            try:
                new_series = new_series.astype(new_type)
                print(f"Successfully converted the series to the type {new_variable_type}.\n")
            
            except:
                pass
        
        
        if (create_new_column):
                
            if (new_column_suffix is None):
                new_column_suffix = "_rightChars"

            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:

            DATASET[column_to_analyze] = new_series

        # Now, we are in the main code.
        print(f"Finished extracting the {number_of_characters_to_retrieve} rightest characters.")
        print("Check the 10 first elements from the series:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))

        except: # regular mode
            print(new_series.head(10))

        return DATASET


    def join_strings_from_column (df, column_to_analyze, separator = " "):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # separator = " " - string containing the separator. Suppose the column contains the
        # strings: 'a', 'b', 'c', 'd'. If the separator is the empty string '', the output will be:
        # 'abcd' (no separation). If separator = " " (simple whitespace), the output will be 'a b c d'
        
        
        if (separator is None):
            # make it a whitespace:
            separator = " "
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        concat_string = separator.join(new_series)
        # sep.join(list_of_strings) method: join all the strings, separating them by sep.

        # Now, we are in the main code.
        print(f"Finished joining strings from column {column_to_analyze}.")
        print("Check the 10 first characters from the new string:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(concat_string[:10])

        except: # regular mode
            print(concat_string[:10])

        return concat_string


    def join_string_columns (df, list_of_columns_to_join, separator = " ", new_column_suffix = "_stringConcat"):
        
        import numpy as np
        import pandas as pd
        
        # list_of_columns_to_join: list of strings (inside quotes), 
        # containing the name of the columns with strings to be joined.
        # Attention: the strings will be joined row by row, i.e. only strings in the same rows will
        # be concatenated. To join strings from the same column, use function join_strings_from_column
        # e.g. list_of_columns_to_join = ["column1", "column2"] will join strings from "column1" with
        # the correspondent strings from "column2".
        # Notice that you can concatenate any kind of columns: numeric, dates, texts ,..., but the output
        # will be a string column.
        
        # separator = " " - string containing the separator. Suppose the columns contain the
        # strings: 'a', 'b', 'c', 'd' on a given row. If the separator is the empty string '', 
        # the output will be: 'abcd' (no separation). If separator = " " (simple whitespace), 
        # the output will be 'a b c d'
        
        # new_column_suffix = "_stringConcat"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_stringConcat", the new column will be named as
        # "column1_stringConcat".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        if (separator is None):
            # make it a whitespace:
            separator = " "
            
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        
        # Start a string pandas series from DATASET, but without connections with it. It will contain
        # only empty strings.
        second_copy_df = DATASET.copy(deep = True)
        second_copy_df['concat_string'] = ''
        # Also, create a separator series from it, and make it constant and equals to the separator:
        second_copy_df['separator'] = separator
        
        new_series = second_copy_df['concat_string']
        sep_series = second_copy_df['separator']
        
        col = list_of_columns_to_join[0]
        new_series = new_series + (DATASET[col]).astype(str)
        
        # Now, loop through the columns in the list:
        for i in range(1, len(list_of_columns_to_join)):
            # We already picked the 1st column (index 0). Now, we pick the second one and go
            # until len(list_of_columns_to_join) - 1, index of the last column of the list.
            
            # concatenate the column with new_series, adding the separator to the left.
            # As we add the separator before, there will be no extra separator after the last string.
            # Convert the columns to strings for concatenation.
            new_series = new_series + sep_series + (DATASET[col]).astype(str)
            # The sep.join(list_of_strings) method can only be applied to array-like objects. It cannot
            # be used for this operation.
                
        if (new_column_suffix is None):
            new_column_suffix = "_stringConcat"

        # Add the suffix to the name of the first column
        new_column_name = list_of_columns_to_join[0] + new_column_suffix
        DATASET[new_column_name] = new_series
        
        # Now, we are in the main code.
        print(f"Finished concatenating strings from columns {list_of_columns_to_join}.")
        print("Check the 10 first elements from the series:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))

        except: # regular mode
            print(new_series.head(10))

        return DATASET


    def split_strings (df, column_to_analyze, separator = " ", create_new_column = True, new_column_suffix = "_stringSplitted"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
    
        # separator = " " - string containing the separator. Suppose the column contains the
        # string: 'a b c d' on a given row. If the separator is whitespace ' ', 
        # the output will be a list: ['a', 'b', 'c', 'd']: the function splits the string into a list
        # of strings (one list per row) every time it finds the separator.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_stringSplitted"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_stringSplitted", the new column will be named as
        # "column1_stringSplitted".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        if (separator is None):
            # make it a whitespace:
            separator = " "
            
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        # Split the strings from new_series, getting a list of strings per column:
        new_series = new_series.str.split(separator)
        
        if (create_new_column):
                
            if (new_column_suffix is None):
                new_column_suffix = "_stringSplitted"

            new_column_name = column_to_analyze + new_column_suffix
            DATASET[new_column_name] = new_series
                
        else:

            DATASET[column_to_analyze] = new_series

        # Now, we are in the main code.
        print(f"Finished splitting strings from column {column_to_analyze}.")
        print("Check the 10 first elements (10 lists) from the series:\n")

        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_series.head(10))

        except: # regular mode
            print(new_series.head(10))

        return DATASET


    def switch_strings (df, column_to_analyze, list_of_dictionaries_with_original_strings_and_replacements = [{'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}, {'original_string': None, 'new_string': None}], create_new_column = True, new_column_suffix = "_stringReplaced"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # list_of_dictionaries_with_original_strings_and_replacements = 
        # [{'original_string': None, 'new_string': None}]
        # This is a list of dictionaries, where each dictionary contains two key-value pairs:
        # the first one contains the original string; and the second one contains the new string
        # that will substitute the original one. The function will loop through all dictionaries in
        # this list, access the values of the keys 'original_string', and search these values on the strings
        # in column_to_analyze. When the value is found, it will be replaced (switched) by the correspondent
        # value in key 'new_string'.
        
        # The object list_of_dictionaries_with_original_strings_and_replacements must be declared as a list, 
        # in brackets, even if there is a single dictionary.
        # Use always the same keys: 'original_string' for the original strings to search on the column 
        # column_to_analyze; and 'new_string', for the strings that will replace the original ones.
        # Notice that this function will not search substrings: it will substitute a value only when
        # there is perfect correspondence between the string in 'column_to_analyze' and 'original_string'.
        # So, the cases (upper or lower) must be the same.
        
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to replace more
        # values.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'original_string': original_str, 'new_string': new_str}, 
        # where original_str and new_str represent the strings for searching and replacement 
        # (If one of the keys contains None, the new dictionary will be ignored).
        
        # Example:
        # Suppose the column_to_analyze contains the values 'sunday', 'monday', 'tuesday', 'wednesday',
        # 'thursday', 'friday', 'saturday', but you want to obtain data labelled as 'weekend' or 'weekday'.
        # Set: list_of_dictionaries_with_original_strings_and_replacements = 
        # [{'original_string': 'sunday', 'new_string': 'weekend'},
        # {'original_string': 'saturday', 'new_string': 'weekend'},
        # {'original_string': 'monday', 'new_string': 'weekday'},
        # {'original_string': 'tuesday', 'new_string': 'weekday'},
        # {'original_string': 'wednesday', 'new_string': 'weekday'},
        # {'original_string': 'thursday', 'new_string': 'weekday'},
        # {'original_string': 'friday', 'new_string': 'weekday'}]
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_stringReplaced"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_stringReplaced", the new column will be named as
        # "column1_stringReplaced".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()
        
        print("ATTENTION: Operations of string strip (removal) or replacement are all case-sensitive. There must be correct correspondence between cases and spaces for the strings being removed or replaced.\n")
        
        # Create the mapping dictionary for the str.replace method:
        mapping_dict = {}
        # The key of the mapping dict must be an string, whereas the value must be the new string
        # that will replace it.
            
        # Loop through each element on the list list_of_dictionaries_with_original_strings_and_replacements:
        
        for i in range (0, len(list_of_dictionaries_with_original_strings_and_replacements)):
            # from i = 0 to i = len(list_of_dictionaries_with_original_strings_and_replacements) - 1, index of the
            # last element from the list
                
            # pick the i-th dictionary from the list:
            dictionary = list_of_dictionaries_with_original_strings_and_replacements[i]
                
            # access 'original_string' and 'new_string' keys from the dictionary:
            original_string = dictionary['original_string']
            new_string = dictionary['new_string']
            
            # check if they are not None:
            if ((original_string is not None) & (new_string is not None)):
                
                #Guarantee that both are read as strings:
                original_string = str(original_string)
                new_string = str(new_string)
                
                # add them to the mapping dictionary, using the original_string as key and
                # new_string as the correspondent value:
                mapping_dict[original_string] = new_string
        
        # Now, the input list was converted into a dictionary with the correct format for the method.
        # Check if there is at least one key in the dictionary:
        if (len(mapping_dict) > 0):
            # len of a dictionary returns the amount of key:value pairs stored. If nothing is stored,
            # len = 0. dictionary.keys() method (no arguments in parentheses) returns an array containing
            # the keys; whereas dictionary.values() method returns the arrays of the values.
            
            new_series = new_series.replace(mapping_dict)
            # For replacing the whole strings using a mapping dictionary, do not call the str
            # attribute
        
            if (create_new_column):
                
                if (new_column_suffix is None):
                    new_column_suffix = "_substringReplaced"

                new_column_name = column_to_analyze + new_column_suffix
                DATASET[new_column_name] = new_series
                
            else:

                DATASET[column_to_analyze] = new_series

            # Now, we are in the main code.
            print(f"Finished replacing the substrings accordingly to the mapping: {mapping_dict}.")
            print("Check the 10 first elements from the series:\n")

            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(new_series.head(10))

            except: # regular mode
                print(new_series.head(10))

            return DATASET
        
        else:
            print("Input at least one dictionary containing a pair of original string, in the key \'original_string\', and the correspondent new string as key \'new_string\'.")
            print("The dictionaries must be elements from the list list_of_dictionaries_with_original_strings_and_replacements.\n")
            
            return "error"


    def string_replacement_ml (df, column_to_analyze, mode = 'find_and_replace', threshold_for_percent_of_similarity = 80.0, list_of_dictionaries_with_standard_strings_for_replacement = [{'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}, {'standard_string': None}], create_new_column = True, new_column_suffix = "_stringReplaced"):
        
        import numpy as np
        import pandas as pd
        from fuzzywuzzy import process
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # mode = 'find_and_replace' will find similar strings; and switch them by one of the
        # standard strings if the similarity between them is higher than or equals to the threshold.
        # Alternatively: mode = 'find' will only find the similar strings by calculating the similarity.
        
        # threshold_for_percent_of_similarity = 80.0 - 0.0% means no similarity and 100% means equal strings.
        # The threshold_for_percent_of_similarity is the minimum similarity calculated from the
        # Levenshtein (minimum edit) distance algorithm. This distance represents the minimum number of
        # insertion, substitution or deletion of characters operations that are needed for making two
        # strings equal.
        
        # list_of_dictionaries_with_standard_strings_for_replacement =
        # [{'standard_string': None}]
        # This is a list of dictionaries, where each dictionary contains a single key-value pair:
        # the key must be always 'standard_string', and the value will be one of the standard strings 
        # for replacement: if a given string on the column_to_analyze presents a similarity with one 
        # of the standard string equals or higher than the threshold_for_percent_of_similarity, it will be
        # substituted by this standard string.
        # For instance, suppose you have a word written in too many ways, making it difficult to use
        # the function switch_strings: "EU" , "eur" , "Europ" , "Europa" , "Erope" , "Evropa" ...
        # You can use this function to search strings similar to "Europe" and replace them.
        
        # The function will loop through all dictionaries in
        # this list, access the values of the keys 'standard_string', and search these values on the strings
        # in column_to_analyze. When the value is found, it will be replaced (switched) if the similarity
        # is sufficiently high.
        
        # The object list_of_dictionaries_with_standard_strings_for_replacement must be declared as a list, 
        # in brackets, even if there is a single dictionary.
        # Use always the same keys: 'standard_string'.
        # Notice that this function performs fuzzy matching, so it MAY SEARCH substrings and strings
        # written with different cases (upper or lower) when this portions or modifications make the
        # strings sufficiently similar to each other.
        
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to replace more
        # values.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same key: {'standard_string': other_std_str}, 
        # where other_std_str represents the string for searching and replacement 
        # (If the key contains None, the new dictionary will be ignored).
        
        # Example:
        # Suppose the column_to_analyze contains the values 'California', 'Cali', 'Calefornia', 
        # 'Calefornie', 'Californie', 'Calfornia', 'Calefernia', 'New York', 'New York City', 
        # but you want to obtain data labelled as the state 'California' or 'New York'.
        # Set: list_of_dictionaries_with_standard_strings_for_replacement = 
        # [{'standard_string': 'California'},
        # {'standard_string': 'New York'}]
        
        # ATTENTION: It is advisable for previously searching the similarity to find the best similarity
        # threshold; set it as high as possible, avoiding incorrect substitutions in a gray area; and then
        # perform the replacement. It will avoid the repetition of original incorrect strings in the
        # output dataset, as well as wrong replacement (replacement by one of the standard strings which
        # is not the correct one).
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_stringReplaced"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_stringReplaced", the new column will be named as
        # "column1_stringReplaced".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        print("Performing fuzzy replacement based on the Levenshtein (minimum edit) distance algorithm.")
        print("This distance represents the minimum number of insertion, substitution or deletion of characters operations that are needed for making two strings equal.\n")
        
        print("This means that substrings or different cases (upper or higher) may be searched and replaced, as long as the similarity threshold is reached.\n")
        
        print("ATTENTION!\n")
        print("It is advisable for previously searching the similarity to find the best similarity threshold.\n")
        print("Set the threshold as high as possible, and only then perform the replacement.\n")
        print("It will avoid the repetition of original incorrect strings in the output dataset, as well as wrong replacement (replacement by one of the standard strings which is not the correct one.\n")
        
        # Set a local copy of dataframe to manipulate
        DATASET = df.copy(deep = True)
        # Guarantee that the column to analyze was read as string:
        DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
        new_series = DATASET[column_to_analyze].copy()

        # Get the unique values present in column_to_analyze:
        unique_types = new_series.unique()
        
        # Create the summary_list:
        summary_list = []
            
        # Loop through each element on the list list_of_dictionaries_with_original_strings_and_replacements:
        
        for i in range (0, len(list_of_dictionaries_with_standard_strings_for_replacement)):
            # from i = 0 to i = len(list_of_dictionaries_with_standard_strings_for_replacement) - 1, index of the
            # last element from the list
                
            # pick the i-th dictionary from the list:
            dictionary = list_of_dictionaries_with_standard_strings_for_replacement[i]
                
            # access 'standard_string' key from the dictionary:
            standard_string = dictionary['standard_string']
            
            # check if it is not None:
            if (standard_string is not None):
                
                # Guarantee that it was read as a string:
                standard_string = str(standard_string)
                
                # Calculate the similarity between each one of the unique_types and standard_string:
                similarity_list = process.extract(standard_string, unique_types, limit = len(unique_types))
                
                # Add the similarity list to the dictionary:
                dictionary['similarity_list'] = similarity_list
                # This is a list of tuples with the format (tested_string, percent_of_similarity_with_standard_string)
                # e.g. ('asiane', 92) for checking similarity with string 'asian'
                
                if (mode == 'find_and_replace'):
                    
                    # If an invalid value was set for threshold_for_percent_of_similarity, correct it to 80% standard:
                    
                    if(threshold_for_percent_of_similarity is None):
                        threshold_for_percent_of_similarity = 80.0
                    
                    if((threshold_for_percent_of_similarity == np.nan) | (threshold_for_percent_of_similarity < 0)):
                        threshold_for_percent_of_similarity = 80.0
                    
                    list_of_replacements = []
                    # Let's replace the matches in the series by the standard_string:
                    # Iterate through the list of matches
                    for match in similarity_list:
                        # Check whether the similarity score is greater than or equal to threshold_for_percent_of_similarity.
                        # The similarity score is the second element (index 1) from the tuples:
                        if (match[1] >= threshold_for_percent_of_similarity):
                            # If it is, select all rows where the column_to_analyze is spelled as
                            # match[0] (1st Tuple element), and set it to standard_string:
                            boolean_filter = (new_series == match[0])
                            new_series.loc[boolean_filter] = standard_string
                            print(f"Found {match[1]}% of similarity between {match[0]} and {standard_string}.")
                            print(f"Then, {match[0]} was replaced by {standard_string}.\n")
                            
                            # Add match to the list of replacements:
                            list_of_replacements.append(match)
                    
                    # Add the list_of_replacements to the dictionary, if its length is higher than zero:
                    if (len(list_of_replacements) > 0):
                        dictionary['list_of_replacements_by_std_str'] = list_of_replacements
                
                # Add the dictionary to the summary_list:
                summary_list.append(dictionary)
        
        # Now, let's replace the original column or create a new one if mode was set as replace:
        if (mode == 'find_and_replace'):
        
            if (create_new_column):
                
                if (new_column_suffix is None):
                    new_column_suffix = "_substringReplaced"

                new_column_name = column_to_analyze + new_column_suffix
                DATASET[new_column_name] = new_series
                
            else:

                DATASET[column_to_analyze] = new_series

            # Now, we are in the main code.
            print(f"Finished replacing the strings by the provided standards. Returning the new dataset and a summary list.\n")
            print("In summary_list, you can check the calculated similarities in keys \'similarity_list\' from the dictionaries.\n")
            print("The similarity list is a list of tuples, where the first element is the string compared against the value on key \'standard_string\'; and the second element is the similarity score, the percent of similarity between the tested and the standard string.\n")
            print("Check the 10 first elements from the new series, with strings replaced:\n")
            
            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(new_series.head(10))

            except: # regular mode
                print(new_series.head(10))
        
        else:
            
            print("Finished mapping similarities. Returning the original dataset and a summary list.\n")
            print("Check the similarities below, in keys \'similarity_list\' from the dictionaries.\n")
            print("The similarity list is a list of tuples, where the first element is the string compared against the value on key \'standard_string\'; and the second element is the similarity score, the percent of similarity between the tested and the standard string.\n")
            
            try:
                display(summary_list)
            except:
                print(summary_list)
        
        return DATASET, summary_list


    def regex_search (df, column_to_analyze, regex_to_search = r"", show_regex_helper = False, create_new_column = True, new_column_suffix = "_regex"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # regex_to_search = r"" - string containing the regular expression (regex) that will be searched
        # within each string from the column. Declare it with the r before quotes, indicating that the
        # 'raw' string should be read. That is because the regex contain special characters, such as \,
        # which should not be read as scape characters.
        # example of regex: r'st\d\s\w{3,10}'
        # Use the regex helper to check: basic theory and most common metacharacters; regex quantifiers;
        # regex anchoring and finding; regex greedy and non-greedy search; regex grouping and capturing;
        # regex alternating and non-capturing groups; regex backreferences; and regex lookaround.
        
        ## ATTENTION: This function returns ONLY the capturing groups from the regex, i.e., portions of the
        # regex explicitly marked with parentheses (check the regex helper for more details, including how
        # to convert parentheses into non-capturing groups). If no groups are marked as capturing, the
        # function will raise an error.

        # show_regex_helper: set show_regex_helper = True to show a helper guide to the construction of
        # the regular expression. After finishing the helper, the original dataset itself will be returned
        # and the function will not proceed. Use it in case of not knowing or not certain on how to input
        # the regex.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_regex"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_regex", the new column will be named as
        # "column1_regex".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        if (show_regex_helper): # run if True
            
            # Create an instance (object) from class regex_help:
            helper = regex_help()
            # Run helper object:
            helper = helper.show_screen()
            print("Interrupting the function and returning the dataframe itself.")
            print("Use the regex helper instructions to obtain the regex.")
            print("Do not forget to declare it as r'regex', with the r before quotes.")
            print("It indicates a raw expression. It is important for not reading the regex metacharacters as regular string scape characters.")
            print("Also, notice that this function returns only the capturing groups (marked with parentheses).")
            print("If no groups are marked as capturing groups (with parentheses) within the regex, the function will raise an error.\n")
            
            return df
        
        else:
            
            # Set a local copy of dataframe to manipulate
            DATASET = df.copy(deep = True)
            DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
            new_series = DATASET[column_to_analyze].copy()
            
            # Search for the regex within new_series:
            new_series = new_series.str.extract(regex_to_search, expand = True)
            
            # https://pandas.pydata.org/docs/reference/api/pandas.Series.str.extract.html
            # setting expand = True returns a dataframe with one column per capture group, if the
            # regex contains more than 1 capture group.
            
            # The shape attribute is a tuple (N,) for a Pandas Series, and (N, M) for a dataframe,
            # where N is the number of rows, and M is the number of columns.
            # Let's try to access the number of columns. It will only be possible if the object is a
            # dataframe (index 1 from shape tuple):
            try:
                
                total_new_cols = new_series.shape[1]
                
                if (new_column_suffix is None):
                    new_column_suffix = "_regex"
                
                new_column_suffix = str(column_to_analyze) + new_column_suffix + "_group_"
                
                # In the regex, the group 0 is the expression itself, whereas the first group is group 1.
                # range (0, total_new_cols) goes from 0 to total_new_cols-1;
                # range (1, total_new_cols + 1) goes from group 1 to group total_new_cols
                # (both cases result in total_new_cols elements):
                
                # Create a list of columns:
                new_columns_list = [(new_column_suffix + str(i)) for i in range (1, (total_new_cols + 1))]
                
                # Make this list the new columns' names:
                new_series.columns = new_columns_list
                
                # Concatenate this dataframe to the original one (add columns to the right of DATASET):
                DATASET = pd.concat([DATASET, new_series], axis = 1, join = "inner")
            
            
            except IndexError:
                
                # There is no second dimension, because it is a series.
                # The regex finds a single group
                
                if (create_new_column):

                    if (new_column_suffix is None):
                        new_column_suffix = "_regex"

                    new_column_name = column_to_analyze + new_column_suffix
                    DATASET[new_column_name] = new_series

                else:

                    DATASET[column_to_analyze] = new_series

            # Now, we are in the main code.
            print(f"Finished searching the regex {regex_to_search} within {column_to_analyze}.")
            print("Check the 10 first elements from the output:\n")

            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(new_series.head(10))

            except: # regular mode
                print(new_series.head(10))

            return DATASET


    def regex_replacement (df, column_to_analyze, regex_to_search = r"", string_for_replacement = "", show_regex_helper = False, create_new_column = True, new_column_suffix = "_regex"):
        
        import numpy as np
        import pandas as pd
        
        # column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # regex_to_search = r"" - string containing the regular expression (regex) that will be searched
        # within each string from the column. Declare it with the r before quotes, indicating that the
        # 'raw' string should be read. That is because the regex contain special characters, such as \,
        # which should not be read as scape characters.
        # example of regex: r'st\d\s\w{3,10}'
        # Use the regex helper to check: basic theory and most common metacharacters; regex quantifiers;
        # regex anchoring and finding; regex greedy and non-greedy search; regex grouping and capturing;
        # regex alternating and non-capturing groups; regex backreferences; and regex lookaround.
        
        # string_for_replacement = "" - regular string that will replace the regex_to_search: 
        # whenever regex_to_search is found in the string, it is replaced (substituted) by 
        # string_or_regex_for_replacement. 
        # Example string_for_replacement = " " (whitespace).
        # If string_for_replacement = None, the empty string will be used for replacement.
        
        ## ATTENTION: This function process a single regex by call.
        
        # show_regex_helper: set show_regex_helper = True to show a helper guide to the construction of
        # the regular expression. After finishing the helper, the original dataset itself will be returned
        # and the function will not proceed. Use it in case of not knowing or not certain on how to input
        # the regex.
        
        # create_new_column = True
        # Alternatively, set create_new_columns = True to store the transformed data into a new
        # column. Or set create_new_column = False to overwrite the existing column.
        
        # new_column_suffix = "_regex"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_regex", the new column will be named as
        # "column1_regex".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        if (show_regex_helper): # run if True
            
            # Create an instance (object) from class regex_help:
            helper = regex_help()
            # Run helper object:
            helper = helper.show_screen()
            print("Interrupting the function and returning the dataframe itself.")
            print("Use the regex helper instructions to obtain the regex.")
            print("Do not forget to declare it as r'regex', with the r before quotes.")
            print("It indicates a raw expression. It is important for not reading the regex metacharacters as regular string scape characters.\n")
            
            return df
        
        else:
            
            if (string_for_replacement is None):
                # make it the empty string
                string_for_replacement = ""
            
            # Set a local copy of dataframe to manipulate
            DATASET = df.copy(deep = True)
            DATASET[column_to_analyze] = (DATASET[column_to_analyze]).astype(str)
            new_series = DATASET[column_to_analyze].copy()
            
            new_series = new_series.str.replace(regex_to_search, string_for_replacement, regex = True)
            # set regex = True to replace a regular expression
            # https://pandas.pydata.org/docs/reference/api/pandas.Series.str.replace.html
                
            if (create_new_column):

                if (new_column_suffix is None):
                    new_column_suffix = "_regex"

                new_column_name = column_to_analyze + new_column_suffix
                DATASET[new_column_name] = new_series

            else:

                DATASET[column_to_analyze] = new_series

            # Now, we are in the main code.
            print(f"Finished searching the regex {regex_to_search} within {column_to_analyze}.")
            print("Check the 10 first elements from the output:\n")

            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(new_series.head(10))

            except: # regular mode
                print(new_series.head(10))

            return DATASET


    def log_transform (df, subset = None, create_new_columns = True, new_columns_suffix = "_log"):
        
        import numpy as np
        import pandas as pd
        
        #### WARNING: This function will eliminate rows where the selected variables present 
        #### values lower or equal to zero (condition for the logarithm to be applied).
        
        # subset = None
        # Set subset = None to transform the whole dataset. Alternatively, pass a list with 
        # columns names for the transformation to be applied. For instance:
        # subset = ['col1', 'col2', 'col3'] will apply the transformation to the columns named
        # as 'col1', 'col2', and 'col3'. Declare the names inside quotes.
        # Declaring the full list of columns is equivalent to setting subset = None.
        
        # create_new_columns = True
        # Alternatively, set create_new_columns = True to store the transformed data into new
        # columns. Or set create_new_columns = False to overwrite the existing columns
        
        # new_columns_suffix = "_log"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_log", the new column will be named as
        # "column1_log".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        # Start a local copy of the dataframe:
        DATASET = df.copy(deep = True)
        
        # Check if a subset was defined. If so, make columns_list = subset 
        if not (subset is None):
            
            columns_list = subset
        
        else:
            #There is no declared subset. Then, make columns_list equals to the list of
            # numeric columns of the dataframe.
            columns_list = list(DATASET.columns)
            
        # Let's check if there are categorical columns in columns_list. Only numerical
        # columns should remain
        # Start a support list:
        support_list = []
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        # Loop through each column in columns_list:
        for column in columns_list:
            
            # Check the Pandas series (column) data type:
            column_type = DATASET[column].dtype
                
            # If it is not categorical (object), append it to the support list:
            if (column_type in numeric_dtypes):
                    
                support_list.append(column)
        
        # Finally, make the columns_list support_list itself:
        columns_list = support_list
        
        #Loop through each column to apply the transform:
        for column in columns_list:
            #access each element in the list column_list. The element is named 'column'.
            
            #boolean filter to check if the entry is higher than zero, condition for the log
            # to be applied
            boolean_filter = (DATASET[column] > 0)
            #This filter is equals True only for the rows where the column is higher than zero.
            
            #Apply the boolean filter to the dataframe, removing the entries where the column
            # cannot be log transformed.
            # The boolean_filter selects only the rows for which the filter values are True.
            DATASET = DATASET[boolean_filter]
            
            #Check if a new column will be created, or if the original column should be
            # substituted.
            if (create_new_columns == True):
                # Create a new column.
                
                # The new column name will be set as column + new_columns_suffix
                new_column_name = column + new_columns_suffix
            
            else:
                # Overwrite the existing column. Simply set new_column_name as the value 'column'
                new_column_name = column
            
            # Calculate the column value as the log transform of the original series (column)
            DATASET[new_column_name] = np.log(DATASET[column])
        
        # Reset the index:
        DATASET.reset_index(drop = True)
        
        print("The columns were successfully log-transformed. Check the 10 first rows of the new dataset:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        return DATASET

        # One curve derived from the normal is the log-normal.
        # If the values Y follow a log-normal distribution, their log follow a normal.
        # A log normal curve resembles a normal, but with skewness (distortion); 
        # and kurtosis (long-tail).

        # Applying the log is a methodology for normalizing the variables: 
        # the sample space gets shrinkled after the transformation, making the data more 
        # adequate for being processed by Machine Learning algorithms. Preferentially apply 
        # the transformation to the whole dataset, so that all variables will be of same order 
        # of magnitude.
        # Obviously, it is not necessary for variables ranging from -100 to 100 in numerical 
        # value, where most outputs from the log transformation are.


    def reverse_log_transform (df, subset = None, create_new_columns = True, new_columns_suffix = "_originalScale"):
        
        import numpy as np
        import pandas as pd
        
        #### WARNING: This function will eliminate rows where the selected variables present 
        #### values lower or equal to zero (condition for the logarithm to be applied).
        
        # subset = None
        # Set subset = None to transform the whole dataset. Alternatively, pass a list with 
        # columns names for the transformation to be applied. For instance:
        # subset = ['col1', 'col2', 'col3'] will apply the transformation to the columns named
        # as 'col1', 'col2', and 'col3'. Declare the names inside quotes.
        # Declaring the full list of columns is equivalent to setting subset = None.
        
        # create_new_columns = True
        # Alternatively, set create_new_columns = True to store the transformed data into new
        # columns. Or set create_new_columns = False to overwrite the existing columns
        
        # new_columns_suffix = "_originalScale"
        # This value has effect only if create_new_column = True.
        # The new column name will be set as column + new_columns_suffix. Then, if the original
        # column was "column1" and the suffix is "_originalScale", the new column will be named 
        # as "column1_originalScale".
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name.
        
        
        # Start a local copy of the dataframe:
        DATASET = df.copy(deep = True)
        
        # Check if a subset was defined. If so, make columns_list = subset 
        if not (subset is None):
            
            columns_list = subset
        
        else:
            #There is no declared subset. Then, make columns_list equals to the list of
            # numeric columns of the dataframe.
            columns_list = list(DATASET.columns)
            
        # Let's check if there are categorical columns in columns_list. Only numerical
        # columns should remain
        # Start a support list:
        support_list = []
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]

        # Loop through each column in columns_list:
        for column in columns_list:
            
            # Check the Pandas series (column) data type:
            column_type = DATASET[column].dtype
                
            # If it is not categorical (object), append it to the support list:
            if (column_type in numeric_dtypes):
                    
                support_list.append(column)
        
        # Finally, make the columns_list support_list itself:
        columns_list = support_list
        
        #Loop through each column to apply the transform:
        for column in columns_list:
            #access each element in the list column_list. The element is named 'column'.
            
            # The exponential transformation can be applied to zero and negative values,
            # so we remove the boolean filter.
            
            #Check if a new column will be created, or if the original column should be
            # substituted.
            if (create_new_columns == True):
                # Create a new column.
                
                # The new column name will be set as column + new_columns_suffix
                new_column_name = column + new_columns_suffix
            
            else:
                # Overwrite the existing column. Simply set new_column_name as the value 'column'
                new_column_name = column
            
            # Calculate the column value as the log transform of the original series (column)
            DATASET[new_column_name] = np.exp(DATASET[column])
        
        print("The log_transform was successfully reversed through the exponential transformation. Check the 10 first rows of the new dataset:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        return DATASET


    def box_cox_transform (df, column_to_transform, mode = 'calculate_and_apply', lambda_boxcox = None, suffix = '_BoxCoxTransf', specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': None}):
        
        import numpy as np
        import pandas as pd
        from statsmodels.stats import diagnostic
        from scipy import stats
        
        # This function will process a single column column_to_transform 
        # of the dataframe df per call.
        
        # Check https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.boxcox.html
        ## Box-Cox transform is given by:
        ## y = (x**lmbda - 1) / lmbda,  for lmbda != 0
        ## log(x),                  for lmbda = 0
        
        # column_to_transform must be a string with the name of the column.
        # e.g. column_to_transform = 'column1' to transform a column named as 'column1'
        
        # mode = 'calculate_and_apply'
        # Aternatively, mode = 'calculate_and_apply' to calculate lambda and apply Box-Cox
        # transform; mode = 'apply_only' to apply the transform for a known lambda.
        # To 'apply_only', lambda_box must be provided.
        
        # lambda_boxcox must be a float value. e.g. lamda_boxcox = 1.7
        # If you calculated lambda from the function box_cox_transform and saved the
        # transformation data summary dictionary as data_sum_dict, simply set:
        # lambda_boxcox = data_sum_dict['lambda_boxcox']
        # This will access the value on the key 'lambda_boxcox' of the dictionary, which
        # contains the lambda. 
        
        # Analogously, spec_lim_dict['Inf_spec_lim_transf'] access
        # the inferior specification limit transformed; and spec_lim_dict['Sup_spec_lim_transf'] 
        # access the superior specification limit transformed.
        
        # If lambda_boxcox is None, 
        # the mode will be automatically set as 'calculate_and_apply'.
        
        # suffix: string (inside quotes).
        # How the transformed column will be identified in the returned data_transformed_df.
        # If y_label = 'Y' and suffix = '_BoxCoxTransf', the transformed column will be
        # identified as 'Y_BoxCoxTransf'.
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name
        
        # specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': None}
        # If there are specification limits, input them in this dictionary. Do not modify the keys,
        # simply substitute None by the lower and/or the upper specification.
        # e.g. Suppose you have a tank that cannot have more than 10 L. So:
        # specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': 10}, there is only
        # an upper specification equals to 10 (do not add units);
        # Suppose a temperature cannot be lower than 10 ºC, but there is no upper specification. So,
        # specification_limits = {'lower_spec_lim': 10, 'upper_spec_lim': None}. Finally, suppose
        # a liquid which pH must be between 6.8 and 7.2:
        # specification_limits = {'lower_spec_lim': 6.8, 'upper_spec_lim': 7.2}
        
        if not (suffix is None):
            #only if a suffix was declared
            #concatenate the column name to the suffix
            new_col = column_to_transform + suffix
        
        else:
            #concatenate the column name to the standard '_BoxCoxTransf' suffix
            new_col = column_to_transform + '_BoxCoxTransf'
        
        boolean_check = (mode != 'calculate_and_apply') & (mode != 'apply_only')
        # & is the 'and' operator. != is the 'is different from' operator.
        #Check if neither 'calculate_and_apply' nor 'apply_only' were selected
        
        if ((lambda_boxcox is None) & (mode == 'apply_only')):
            print("Invalid value set for \'lambda_boxcox'\. Setting mode to \'calculate_and_apply\'.\n")
            mode = 'calculate_and_apply'
        
        elif (boolean_check == True):
            print("Invalid value set for \'mode'\. Setting mode to \'calculate_and_apply\'.\n")
            mode = 'calculate_and_apply'
        
        
        # Start a local copy of the dataframe:
        DATASET = df.copy(deep = True)
        
        print("Box-Cox transformation must be applied only to values higher than zero.\n")
        print("That is because it is a logarithmic transformation.\n")
        print(f"So, filtering out all values from {column_to_transform} lower than or equal to zero.\n")
        DATASET = DATASET[DATASET[column_to_transform] > 0]
        DATASET = DATASET.reset_index(drop = True)
        
        y = DATASET[column_to_transform]
        
        
        if (mode == 'calculate_and_apply'):
            # Calculate lambda_boxcox
            lambda_boxcox = stats.boxcox_normmax(y, method = 'pearsonr')
            #calcula o lambda da transformacao box-cox utilizando o metodo da maxima verossimilhanca
            #por meio da maximizacao do coeficiente de correlacao de pearson da funcao
            #y = boxcox(x), onde boxcox representa a transformacao
        
        # For other cases, we will apply the lambda_boxcox set as the function parameter.

        #Calculo da variavel transformada
        y_transform = stats.boxcox(y, lmbda = lambda_boxcox, alpha = None)
        #Calculo da transformada
        
        DATASET[new_col] = y_transform
        #dataframe contendo os dados transformados
        
        print("Data successfully transformed. Check the 10 first transformed rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
            
        print("\n") #line break
        
        # Start a dictionary to store the summary results of the transform and the normality
        # tests:
        data_sum_dict = {'lambda_boxcox': lambda_boxcox}
        
        # Test normality of the transformed variable:
        # Scipy.stats’ normality test
        # It is based on D’Agostino and Pearson’s test that combines 
        # skew and kurtosis to produce an omnibus test of normality.
        _, scipystats_test_pval = stats.normaltest(y_transform) 
        # add this test result to the dictionary:
        data_sum_dict['dagostino_pearson_p_val'] = scipystats_test_pval
                
        # Scipy.stats’ Shapiro-Wilk test
        shapiro_test = stats.shapiro(y_transform)
        data_sum_dict['shapiro_wilk_p_val'] = shapiro_test[1]
        
        # Lilliefors’ normality test
        lilliefors_test = diagnostic.kstest_normal(y_transform, dist = 'norm', pvalmethod = 'table')
        data_sum_dict['lilliefors_p_val'] = lilliefors_test[1]
        
        # Anderson-Darling normality test
        ad_test = diagnostic.normal_ad(y_transform, axis = 0)
        data_sum_dict['anderson_darling_p_val'] = ad_test[1]
        
        print("Box-Cox Transformation Summary:\n")
        try:
            display(data_sum_dict)     
        except:
            print(data_sum_dict)
        
        print("\n") #line break
        
        if not ((specification_limits['lower_spec_lim'] is None) & (specification_limits['upper_spec_lim'] is None)):
            # Convert it to a list of specs:
            list_of_specs = []
            
            if not (specification_limits['lower_spec_lim'] is None):
                
                if (specification_limits['lower_spec_lim'] <= 0):
                    print("Box-Cox transform can only be applied to values higher than zero. So, ignoring the lower specification.\n")
                
                else:
                    list_of_specs.append(specification_limits['lower_spec_lim'])
            
            if not (specification_limits['upper_spec_lim'] is None):
                
                if (specification_limits['upper_spec_lim'] <= 0):
                    print("Box-Cox transform can only be applied to values higher than zero. So, ignoring the upper specification.\n")
                
                else:
                    list_of_specs.append(specification_limits['upper_spec_lim'])
            
            # Notice that the list may have 1 or 2 elements.
            
            # Convert the list of specifications into a NumPy array:
            spec_lim_array = np.array(list_of_specs)
            
            # If the array has a single element, we cannot apply stats method. So, let's transform
            # manually:
            ## y = (x**lmbda - 1) / lmbda,  for lmbda != 0
            ## log(x),                  for lmbda = 0
            if (lambda_boxcox == 0):
                
                spec_lim_array = np.log(spec_lim_array)
            
            else:
                spec_lim_array = ((spec_lim_array**lambda_boxcox) - 1)/(lambda_boxcox)
            
            # Start a dictionary to store the transformed limits:
            spec_lim_dict = {}
            
            if not (specification_limits['lower_spec_lim'] is None):
                # First element of the array is the lower specification. Add it to the
                # dictionary:
                spec_lim_dict['lower_spec_lim_transf'] = spec_lim_array[0]
                
                if not (specification_limits['upper_spec_lim'] is None):
                    # Second element of the array is the upper specification. Add it to the
                    # dictionary:
                    spec_lim_dict['upper_spec_lim_transf'] = spec_lim_array[1]
            
            else:
                # The array contains only one element, which is the upper specification. Add
                # it to the dictionary:
                spec_lim_dict['upper_spec_lim_transf'] = spec_lim_array[0]
            
            print("New specification limits successfully obtained:\n")
            try:
                display(spec_lim_dict)     
            except:
                print(spec_lim_dict)
            
            # Add spec_lim_dict as a new element from data_sum_dict:
            data_sum_dict['spec_lim_dict'] = spec_lim_dict
        
        
        return DATASET, data_sum_dict


    def reverse_box_cox (df, column_to_transform, lambda_boxcox, suffix = '_ReversedBoxCox'):
        
        import numpy as np
        import pandas as pd
        
        # This function will process a single column column_to_transform 
        # of the dataframe df per call.
        
        # Check https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.boxcox.html
        ## Box-Cox transform is given by:
        ## y = (x**lmbda - 1) / lmbda,  for lmbda != 0
        ## log(x),                  for lmbda = 0
        
        # column_to_transform must be a string with the name of the column.
        # e.g. column_to_transform = 'column1' to transform a column named as 'column1'
        
        # lambda_boxcox must be a float value. e.g. lamda_boxcox = 1.7
        # If you calculated lambda from the function box_cox_transform and saved the
        # transformation data summary dictionary as data_sum_dict, simply set:
        # lambda_boxcox = data_sum_dict['lambda_boxcox']
        # This will access the value on the key 'lambda_boxcox' of the dictionary, which
        # contains the lambda. 
        
        # Analogously, spec_lim_dict['Inf_spec_lim_transf'] access
        # the inferior specification limit transformed; and spec_lim_dict['Sup_spec_lim_transf'] 
        # access the superior specification limit transformed.
        
        #suffix: string (inside quotes).
        # How the transformed column will be identified in the returned data_transformed_df.
        # If y_label = 'Y' and suffix = '_ReversedBoxCox', the transformed column will be
        # identified as '_ReversedBoxCox'.
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name
        
        
        # Start a local copy of the dataframe:
        DATASET = df.copy(deep = True)

        y = DATASET[column_to_transform]
        
        if (lambda_boxcox == 0):
            #ytransf = np.log(y), according to Box-Cox definition. Then
            #y_retransform = np.exp(y)
            #In the case of this function, ytransf is passed as the argument y.
            y_transform = np.exp(y)
        
        else:
            #apply Box-Cox function:
            #y_transf = (y**lmbda - 1) / lmbda. Then,
            #y_retransf ** (lmbda) = (y_transf * lmbda) + 1
            #y_retransf = ((y_transf * lmbda) + 1) ** (1/lmbda), where ** is the potentiation
            #In the case of this function, ytransf is passed as the argument y.
            y_transform = ((y * lambda_boxcox) + 1) ** (1/lambda_boxcox)
        
        if not (suffix is None):
            #only if a suffix was declared
            #concatenate the column name to the suffix
            new_col = column_to_transform + suffix
        
        else:
            #concatenate the column name to the standard '_ReversedBoxCox' suffix
            new_col = column_to_transform + '_ReversedBoxCox'
        
        DATASET[new_col] = y_transform
        #dataframe contendo os dados transformados
        
        print("Data successfully retransformed. Check the 10 first retransformed rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        print("\n") #line break
    
        return DATASET


    def OneHotEncode_df (df, subset_of_features_to_be_encoded):

        import numpy as np
        import pandas as pd
        from sklearn.preprocessing import OneHotEncoder
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html
        
        # df: the whole dataframe to be processed.
        
        # subset_of_features_to_be_encoded: list of strings (inside quotes), 
        # containing the names of the columns with the categorical variables that will be 
        # encoded. If a single column will be encoded, declare this parameter as list with
        # only one element e.g.subset_of_features_to_be_encoded = ["column1"] 
        # will analyze the column named as 'column1'; 
        # subset_of_features_to_be_encoded = ["col1", 'col2', 'col3'] will analyze 3 columns
        # with categorical variables: 'col1', 'col2', and 'col3'.
        
        #Start an encoding list empty (it will be a JSON object):
        encoding_list = []
        
        # Start a copy of the original dataframe. This copy will be updated to create the new
        # transformed dataframe. Then, we avoid manipulating the original object.
        new_df = df.copy(deep = True)
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display  
        except:
            pass
        
        #loop through each column of the subset:
        for column in subset_of_features_to_be_encoded:
            
            # Start two empty dictionaries:
            encoding_dict = {}
            nested_dict = {}
            
            # Add the column to encoding_dict as the key 'column':
            encoding_dict['column'] = column
            
            # Loop through each element (named 'column') of the list of columns to analyze,
            # subset_of_features_to_be_encoded
            
            # We could process the whole subset at once, but it could make us lose information
            # about the generated columns
            
            # set a subset of the dataframe X containing 'column' as the only column:
            # it will be equivalent to using .reshape(-1,1) to set a 1D-series
            # or array in the shape for scikit-learn:
            # For doing so, pass a list of columns for column filtering, containing
            # the object column as its single element:
            X  = df[[column]]
            
            #Start the OneHotEncoder object:
            OneHot_enc_obj = OneHotEncoder()
            
            #Fit the object to that column:
            OneHot_enc_obj = OneHot_enc_obj.fit(X)
            # Get the transformed columns as a SciPy sparse matrix: 
            transformed_columns = OneHot_enc_obj.transform(X)
            # Convert the sparse matrix to a NumPy dense array:
            transformed_columns = transformed_columns.toarray()
            
            # Now, let's retrieve the encoding information and save it:
            # Show encoded categories and store this array. 
            # It will give the proper columns' names:
            encoded_columns = OneHot_enc_obj.categories_

            # encoded_columns is an array containing a single element.
            # This element is an array like:
            # array(['cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat6', 'cat7', 'cat8'], dtype=object)
            # Then, this array is the element of index 0 from the list encoded_columns.
            # It is represented as encoded_columns[0]

            # Therefore, we actually want the array which is named as encoded_columns[0]
            # Each element of this array is the name of one of the encoded columns. In the
            # example above, the element 'cat2' would be accessed as encoded_columns[0][1],
            # since it is the element of index [1] (second element) from the array 
            # encoded_columns[0].
            
            new_columns = encoded_columns[0]
            # To identify the column that originated these new columns, we can join the
            # string column to each element from new_columns:
            
            # Update the nested dictionary: store the new_columns as the key 'categories':
            nested_dict['categories'] = new_columns
            # Store the encoder object as the key 'OneHot_enc_obj'
            # Add the encoder object to the dictionary:
            nested_dict['OneHot_enc_obj'] = OneHot_enc_obj
            
            # Store the nested dictionary in the encoding_dict as the key 'OneHot_encoder':
            encoding_dict['OneHot_encoder'] = nested_dict
            # Append the encoding_dict as an element from list encoding_list:
            encoding_list.append(encoding_dict)
            
            # Now we saved all encoding information, let's transform the data:
            
            # Start a support_list to store the concatenated strings:
            support_list = []
            
            for encoded_col in new_columns:
                # Use the str attribute to guarantee that the array stores only strings.
                # Add an underscore "_" to separate the strings and an identifier of the transform:
                new_column = column + "_" + str(encoded_col) + "_OneHotEnc"
                # Append it to the support_list:
                support_list.append(new_column)
                
            # Convert the support list to NumPy array, and make new_columns the support list itself:
            new_columns = np.array(support_list)
            
            # Crete a Pandas dataframe from the array transformed_columns:
            encoded_X_df = pd.DataFrame(transformed_columns)
            
            # Modify the name of the columns to make it equal to new_columns:
            encoded_X_df.columns = new_columns
            
            #Inner join the new dataset with the encoded dataset.
            # Use the index as the key, since indices are necessarily correspondent.
            # To use join on index, we apply pandas .concat method.
            # To join on a specific key, we could use pandas .merge method with the arguments
            # left_on = 'left_key', right_on = 'right_key'; or, if the keys have same name,
            # on = 'key':
            # Check Pandas merge and concat documentation:
            # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.merge.html
            
            new_df = pd.concat([new_df, encoded_X_df], axis = 1, join = "inner")
            # When axis = 0, the .concat operation occurs in the row level, so the rows
            # of the second dataframe are added to the bottom of the first one.
            # It is the SQL union, and creates a dataframe with more rows, and
            # total of columns equals to the total of columns of the first dataframe
            # plus the columns of the second one that were not in the first dataframe.
            # When axis = 1, the operation occurs in the column level: the two
            # dataframes are laterally merged using the index as the key, 
            # preserving all columns from both dataframes. Therefore, the number of
            # rows will be the total of rows of the dataframe with more entries,
            # and the total of columns will be the sum of the total of columns of
            # the first dataframe with the total of columns of the second dataframe.
            
            print(f"Successfully encoded column \'{column}\' and merged the encoded columns to the dataframe.\n")
            print("Check first 5 rows of the encoded table that was merged:\n")
            
            try:
                display(encoded_X_df.head())
            except: # regular mode
                print(encoded_X_df.head())
            
            # The default of the head method, when no parameter is printed, is to show 5 rows; if an
            # integer number Y is passed as argument .head(Y), Pandas shows the first Y-rows.
            print("\n")
            
        print("Finished One-Hot Encoding. Returning the new transformed dataframe; and an encoding list.\n")
        print("Each element from this list is a dictionary with the original column name as key \'column\', and a nested dictionary as the key \'OneHot_encoder\'.\n")
        print("In turns, the nested dictionary shows the different categories as key \'categories\' and the encoder object as the key \'OneHot_enc_obj\'.\n")
        print("Use the encoder object to inverse the One-Hot Encoding in the correspondent function.\n")
        print(f"For each category in the columns \'{subset_of_features_to_be_encoded}\', a new column has value 1, if it is the actual category of that row; or is 0 if not.\n")
        print("Check the first 10 rows of the new dataframe:\n")
        
        try:
            display(new_df.head(10))
        except:
            print(new_df.head(10))

        #return the transformed dataframe and the encoding dictionary:
        return new_df, encoding_list


    def reverse_OneHotEncode (df, encoding_list):

        import pandas as pd
        from sklearn.preprocessing import OneHotEncoder
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html
        
        # df: the whole dataframe to be processed.
        
        # encoding_list: list in the same format of the one generated by OneHotEncode_df function:
        # it must be a list of dictionaries where each dictionary contains two keys:
        # key 'column': string with the original column name (in quotes); 
        # key 'OneHot_encoder': this key must store a nested dictionary.
        # Even though the nested dictionaries generates by the encoding function present
        # two keys:  'categories', storing an array with the different categories;
        # and 'OneHot_enc_obj', storing the encoder object, only the key 'OneHot_enc_obj' is required.
        ## On the other hand, a third key is needed in the nested dictionary:
        ## key 'encoded_columns': this key must store a list or array with the names of the columns
        # obtained from Encoding.
        
        
        # Start a copy of the original dataframe. This copy will be updated to create the new
        # transformed dataframe. Then, we avoid manipulating the original object.
        new_df = df.copy(deep = True)
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display  
        except:
            pass
        
        for encoder_dict in encoding_list:
            
            try:
                # Check if the required arguments are present:
                if ((encoder_dict['column'] is not None) & (encoder_dict['OneHot_encoder']['OneHot_enc_obj'] is not None) & (encoder_dict['OneHot_encoder']['encoded_columns'] is not None)):

                    # Access the column name:
                    col_name = encoder_dict['column']

                    # Access the nested dictionary:
                    nested_dict = encoder_dict['OneHot_encoder']
                    # Access the encoder object on the dictionary
                    OneHot_enc_obj = nested_dict['OneHot_enc_obj']
                    # Access the list of encoded columns:
                    list_of_encoded_cols = list(nested_dict['encoded_columns'])

                    # Get a subset of the encoded columns
                    X = new_df.copy(deep = True)
                    X = X[list_of_encoded_cols]

                    # Reverse the encoding:
                    reversed_array = OneHot_enc_obj.inverse_transform(X)

                    # Add the reversed array as the column col_name on the dataframe:
                    new_df[col_name] = reversed_array
                    
                    print(f"Reversed the encoding for {col_name}. Check the 5 first rows of the re-transformed series:\n")
                    
                    try:
                        display(new_df[[col_name]].head())
                    except:
                        print(new_df[[col_name]].head())
                    
                    print("\n")
                
            except:
                print("Detected dictionary with incorrect keys or format. Unable to reverse encoding. Please, correct it.\n")
        
        print("Finished reversing One-Hot Encoding. Returning the new transformed dataframe.\n")
        print("Check the first 10 rows of the new dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_df.head(10))
                
        except: # regular mode
            print(new_df.head(10))

        #return the transformed dataframe:
        return new_df


    def OrdinalEncode_df (df, subset_of_features_to_be_encoded):

        # Ordinal encoding: let's associate integer sequential numbers to the categorical column
        # to apply the advanced encoding techniques. Even though the one-hot encoding could perform
        # the same task and would, in fact, better, since there may be no ordering relation, the
        # ordinal encoding is simpler and more suitable for this particular task:    
        
        import pandas as pd
        from sklearn.preprocessing import OrdinalEncoder
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OrdinalEncoder.html#sklearn.preprocessing.OrdinalEncoder 
        
        # df: the whole dataframe to be processed.
        
        # subset_of_features_to_be_encoded: list of strings (inside quotes), 
        # containing the names of the columns with the categorical variables that will be 
        # encoded. If a single column will be encoded, declare this parameter as list with
        # only one element e.g.subset_of_features_to_be_encoded = ["column1"] 
        # will analyze the column named as 'column1'; 
        # subset_of_features_to_be_encoded = ["col1", 'col2', 'col3'] will analyze 3 columns
        # with categorical variables: 'col1', 'col2', and 'col3'.
        
        #Start an encoding list empty (it will be a JSON object):
        encoding_list = []
        
        # Start a copy of the original dataframe. This copy will be updated to create the new
        # transformed dataframe. Then, we avoid manipulating the original object.
        new_df = df.copy(deep = True)
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display  
        except:
            pass
    
        #loop through each column of the subset:
        for column in subset_of_features_to_be_encoded:
            
            # Start two empty dictionaries:
            encoding_dict = {}
            nested_dict = {}
            
            # Add the column to encoding_dict as the key 'column':
            encoding_dict['column'] = column
            
            # Loop through each element (named 'column') of the list of columns to analyze,
            # subset_of_features_to_be_encoded
            
            # We could process the whole subset at once, but it could make us lose information
            # about the generated columns
            
            # set a subset of the dataframe X containing 'column' as the only column:
            # it will be equivalent to using .reshape(-1,1) to set a 1D-series
            # or array in the shape for scikit-learn:
            # For doing so, pass a list of columns for column filtering, containing
            # the object column as its single element:
            X  = new_df[[column]]
            
            #Start the OrdinalEncoder object:
            ordinal_enc_obj = OrdinalEncoder()
            
            # Fit the ordinal encoder to the dataframe X:
            ordinal_enc_obj = ordinal_enc_obj.fit(X)
            # Get the transformed dataframe X: 
            transformed_X = ordinal_enc_obj.transform(X)
            # transformed_X is an array of arrays like: [[0.], [0.], ..., [8.]]
            # We want all the values in the first position of the internal arrays:
            transformed_X = transformed_X[:,0]
            # Get the encoded series as a NumPy array:
            encoded_series = np.array(transformed_X)
            
            # Get a name for the new encoded column:
            new_column = column + "_OrdinalEnc"
            # Add this column to the dataframe:
            new_df[new_column] = encoded_series
            
            # Now, let's retrieve the encoding information and save it:
            # Show encoded categories and store this array. 
            # It will give the proper columns' names:
            encoded_categories = ordinal_enc_obj.categories_

            # encoded_categories is an array of strings containing the information of
            # encoded categories and their values.
            
            # Update the nested dictionary: store the categories as the key 'categories':
            nested_dict['categories'] = encoded_categories
            # Store the encoder object as the key 'ordinal_enc_obj'
            # Add the encoder object to the dictionary:
            nested_dict['ordinal_enc_obj'] = ordinal_enc_obj
            
            # Store the nested dictionary in the encoding_dict as the key 'ordinal_encoder':
            encoding_dict['ordinal_encoder'] = nested_dict
            # Append the encoding_dict as an element from list encoding_list:
            encoding_list.append(encoding_dict)
            
            print(f"Successfully encoded column \'{column}\' and added the encoded column to the dataframe.\n")
            print("Check first 5 rows of the encoded series that was merged:\n")
            
            try:
                display(new_df[[new_column]].head())
            except:
                print(new_df[[new_column]].head())
            
            # The default of the head method, when no parameter is printed, is to show 5 rows; if an
            # integer number Y is passed as argument .head(Y), Pandas shows the first Y-rows.
            print("\n")
            
        print("Finished Ordinal Encoding. Returning the new transformed dataframe; and an encoding list.\n")
        print("Each element from this list is a dictionary with the original column name as key \'column\', and a nested dictionary as the key \'ordinal_encoder\'.\n")
        print("In turns, the nested dictionary shows the different categories as key \'categories\' and the encoder object as the key \'ordinal_enc_obj\'.\n")
        print("Use the encoder object to inverse the Ordinal Encoding in the correspondent function.\n")
        print("Check the first 10 rows of the new dataframe:\n")
        
        try:
            display(new_df.head(10))
        except:
            print(new_df.head(10))
        
        #return the transformed dataframe and the encoding dictionary:
        return new_df, encoding_list


    def reverse_OrdinalEncode (df, encoding_list):

        import pandas as pd
        from sklearn.preprocessing import OrdinalEncoder
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OrdinalEncoder.html#sklearn.preprocessing.OrdinalEncoder
        
        # df: the whole dataframe to be processed.
        
        # encoding_list: list in the same format of the one generated by OrdinalEncode_df function:
        # it must be a list of dictionaries where each dictionary contains two keys:
        # key 'column': string with the original column name (in quotes); 
        # key 'ordinal_encoder': this key must store a nested dictionary.
        # Even though the nested dictionaries generates by the encoding function present
        # two keys:  'categories', storing an array with the different categories;
        # and 'ordinal_enc_obj', storing the encoder object, only the key 'ordinal_enc_obj' is required.
        ## On the other hand, a third key is needed in the nested dictionary:
        ## key 'encoded_column': this key must store a string with the name of the column
        # obtained from Encoding.
        
        
        # Start a copy of the original dataframe. This copy will be updated to create the new
        # transformed dataframe. Then, we avoid manipulating the original object.
        new_df = df.copy(deep = True)
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display  
        except:
            pass
    
        for encoder_dict in encoding_list:
            
            try:
                # Check if the required arguments are present:
                if ((encoder_dict['column'] is not None) & (encoder_dict['ordinal_encoder']['ordinal_enc_obj'] is not None) & (encoder_dict['ordinal_encoder']['encoded_column'] is not None)):

                    # Access the column name:
                    col_name = encoder_dict['column']

                    # Access the nested dictionary:
                    nested_dict = encoder_dict['ordinal_encoder']
                    # Access the encoder object on the dictionary
                    ordinal_enc_obj = nested_dict['ordinal_enc_obj']
                    # Access the encoded column and save it as a list:
                    list_of_encoded_cols = [nested_dict['encoded_column']]
                    # In OneHotEncoding we have an array of strings. Applying the list
                    # attribute would convert the array to list. Here, in turns, we have a simple
                    # string, which is also an iterable object. Applying the list attribute to a string
                    # creates a list of characters of that string.
                    # So, here we create a list with the string as its single element.

                    # Get a subset of the encoded column
                    X = new_df.copy(deep = True)
                    X = X[list_of_encoded_cols]

                    # Reverse the encoding:
                    reversed_array = ordinal_enc_obj.inverse_transform(X)

                    # Add the reversed array as the column col_name on the dataframe:
                    new_df[col_name] = reversed_array
                        
                    print(f"Reversed the encoding for {col_name}. Check the 5 first rows of the re-transformed series:\n")
                    
                    try:
                        display(new_df[[col_name]].head())
                    except:
                        print(new_df[[col_name]].head())

                    print("\n")
                    
            except:
                print("Detected dictionary with incorrect keys or format. Unable to reverse encoding. Please, correct it.\n")
        
        
        print("Finished reversing Ordinal Encoding. Returning the new transformed dataframe.\n")
        print("Check the first 10 rows of the new dataframe:\n")
        
        try:
            display(new_df.head(10))
        except:
            print(new_df.head(10))

        #return the transformed dataframe:
        return new_df


    def feature_scaling (df, subset_of_features_to_scale, mode = 'min_max', scale_with_new_params = True, list_of_scaling_params = None, suffix = '_scaled'):
        
        import numpy as np
        import pandas as pd
        from sklearn.preprocessing import StandardScaler
        from sklearn.preprocessing import MinMaxScaler
        # Scikit-learn Preprocessing data guide:
        # https://scikit-learn.org/stable/modules/preprocessing.html#preprocessing-scaler
        # Standard scaler documentation:
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html
        # Min-Max scaler documentation:
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html#sklearn.preprocessing.MinMaxScaler.set_params
        
        ## Machine Learning algorithms are extremely sensitive to scale. 
        
        ## This function provides 4 methods (modes) of scaling:
        ## mode = 'standard': applies the standard scaling, 
        ##  which creates a new variable with mean = 0; and standard deviation = 1.
        ##  Each value Y is transformed as Ytransf = (Y - u)/s, where u is the mean 
        ##  of the training samples, and s is the standard deviation of the training samples.
        
        ## mode = 'min_max': applies min-max normalization, with a resultant feature 
        ## ranging from 0 to 1. each value Y is transformed as 
        ## Ytransf = (Y - Ymin)/(Ymax - Ymin), where Ymin and Ymax are the minimum and 
        ## maximum values of Y, respectively.
        
        ## mode = 'factor': divides the whole series by a numeric value provided as argument. 
        ## For a factor F, the new Y values will be Ytransf = Y/F.
        
        ## mode = 'normalize_by_maximum' is similar to mode = 'factor', but the factor will be selected
        # as the maximum value. This mode is available only for scale_with_new_params = True. If
        # scale_with_new_params = False, you should provide the value of the maximum as a division 'factor'.
        
        # df: the whole dataframe to be processed.
        
        # subset_of_features_to_be_scaled: list of strings (inside quotes), 
        # containing the names of the columns with the categorical variables that will be 
        # encoded. If a single column will be encoded, declare this parameter as list with
        # only one element e.g.subset_of_features_to_be_scaled = ["column1"] 
        # will analyze the column named as 'column1'; 
        # subset_of_features_to_be_scaled = ["col1", 'col2', 'col3'] will analyze 3 columns
        # with categorical variables: 'col1', 'col2', and 'col3'.
        
        # scale_with_new_params = True
        # Alternatively, set scale_with_new_params = True if you want to calculate a new
        # scaler for the data; or set scale_with_new_params = False if you want to apply 
        # parameters previously obtained to the data (i.e., if you want to apply the scaler
        # previously trained to another set of data; or wants to simply apply again the same
        # scaler).
        
        # list_of_scaling_params:
        # This variable has effect only when SCALE_WITH_NEW_PARAMS = False
        ## WARNING: The mode 'factor' demmands the input of the list of factors that will be 
        # used for normalizing each column. Therefore, it can be used only 
        # when scale_with_new_params = False.
        
        # list_of_scaling_params is a list of dictionaries with the same format of the list returned
        # from this function. Each dictionary must correspond to one of the features that will be scaled,
        # but the list do not have to be in the same order of the columns - it will check one of the
        # dictionary keys.
        # The first key of the dictionary must be 'column'. This key must store a string with the exact
        # name of the column that will be scaled.
        # the second key must be 'scaler'. This key must store a dictionary. The dictionary must store
        # one of two keys: 'scaler_obj' - sklearn scaler object to be used; or 'scaler_details' - the
        # numeric parameters for re-calculating the scaler without the object. The key 'scaler_details', 
        # must contain a nested dictionary. For the mode 'min_max', this dictionary should contain 
        # two keys: 'min', with the minimum value of the variable, and 'max', with the maximum value. 
        # For mode 'standard', the keys should be 'mu', with the mean value, and 'sigma', with its 
        # standard deviation. For the mode 'factor', the key should be 'factor', and should contain the 
        # factor for division (the scaling value. e.g 'factor': 2.0 will divide the column by 2.0.).
        # Again, if you want to normalize by the maximum, declare the maximum value as any other factor for
        # division.
        # The key 'scaler_details' will not create an object: the transform will be directly performed 
        # through vectorial operations.
        
        # suffix: string (inside quotes).
        # How the transformed column will be identified in the returned data_transformed_df.
        # If y_label = 'Y' and suffix = '_scaled', the transformed column will be
        # identified as '_scaled'.
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name
        
        if (suffix is None):
            #set as the default
            suffix = '_scaled'
        
        #Start a copy of the original dataframe. This copy will be updated to create the new
        # transformed dataframe. Then, we avoid manipulating the original object.
        new_df = df.copy(deep = True)
        
        #Start an scaling list empty (it will be a JSON object):
        scaling_list = []
        
        for column in subset_of_features_to_scale:
            
            # Create a dataframe X by subsetting only the analyzed column
            # it will be equivalent to using .reshape(-1,1) to set a 1D-series
            # or array in the shape for scikit-learn:
            # For doing so, pass a list of columns for column filtering, containing
            # the object column as its single element:
            X = new_df[[column]]
            
            if (scale_with_new_params == False):
                
                # Use a previously obtained scaler.
                # Loop through each element of the list:
                
                for scaling_dict in list_of_scaling_params:
                    
                    # check if the dictionary is from that column:
                    if (scaling_dict['column'] == column):
                        
                        # We found the correct dictionary. Let's retrieve the information:
                        # retrieve the nested dictionary:
                        nested_dict = scaling_dict['scaler']
                        
                        # try accessing the scaler object:
                        try:
                            scaler = nested_dict['scaler_obj']
                            #calculate the scaled feature, and store it as new array:
                            scaled_feature = scaler.transform(X)
                            
                            # Add the parameters to the nested dictionary:
                            nested_dict['scaling_params'] = scaler.get_params(deep = True)
                            
                            if (mode == 'standard'):
                                
                                nested_dict['scaler_details'] = {
                                    'mu': X[column].mean(),
                                    'sigma': X[column].std()
                                }
                            
                            elif (mode == 'min_max'):
                                
                                nested_dict['scaler_details'] = {
                                    'min': X[column].min(),
                                    'max': X[column].max()
                                }
                        
                        except:
                            
                            try:
                                # As last alternative, let's try accessing the scaler details dict
                                scaler_details = nested_dict['scaler_details']
                                    
                                if (mode == 'standard'):
                                    
                                    nested_dict['scaling_params'] = 'standard_scaler_manually_defined'
                                    mu = scaler_details['mu']
                                    sigma = scaler_details['sigma']
                                        
                                    if (sigma != 0):
                                        scaled_feature = (X - mu)/sigma
                                    else:
                                        scaled_feature = (X - mu)
                                    
                                elif (mode == 'min_max'):
                                        
                                    nested_dict['scaling_params'] = 'min_max_scaler_manually_defined'
                                    minimum = scaler_details['min']
                                    maximum = scaler_details['max']
                                        
                                    if ((maximum - minimum) != 0):
                                        scaled_feature = (X - minimum)/(maximum - minimum)
                                    else:
                                        scaled_feature = X/maximum
                                    
                                elif (mode == 'factor'):
                                    
                                    nested_dict['scaling_params'] = 'normalization_by_factor'
                                    factor = scaler_details['factor']
                                    scaled_feature = X/(factor)
                                    
                                else:
                                    print("Select a valid mode: standard, min_max, or factor.\n")
                                    return "error", "error"
                                
                            except:
                                    
                                print(f"No valid scaling dictionary was input for column {column}.\n")
                                return "error", "error"
                
            elif (mode == 'normalize_by_maximum'):
                
                #Start an scaling dictionary empty:
                scaling_dict = {}

                # add the column to the scaling dictionary:
                scaling_dict['column'] = column

                # Start a nested dictionary:
                nested_dict = {}
                
                factor = X[column].max()
                scaled_feature = X/(factor)
                nested_dict['scaling_params'] = 'normalization_by_factor'
                nested_dict['scaler_details'] = {'factor': factor, 'description': 'division_by_maximum_detected_value'}
        
            else:
                # Create a new scaler:
                
                #Start an scaling dictionary empty:
                scaling_dict = {}

                # add the column to the scaling dictionary:
                scaling_dict['column'] = column
                
                # Start a nested dictionary:
                nested_dict = {}
                    
                #start the scaler object:
                if (mode == 'standard'):
                    
                    scaler = StandardScaler()
                    scaler_details = {'mu': X[column].mean(), 'sigma': X[column].std()}

                elif (mode == 'min_max'):
                    
                    scaler = MinMaxScaler()
                    scaler_details = {'min': X[column].min(), 'max': X[column].max()}
                    
                # fit the scaler to the column
                scaler = scaler.fit(X)
                        
                # calculate the scaled feature, and store it as new array:
                scaled_feature = scaler.transform(X)
                # scaler.inverse_transform(X) would reverse the scaling.
                    
                # Get the scaling parameters for that column:
                scaling_params = scaler.get_params(deep = True)
                        
                # scaling_params is a dictionary containing the scaling parameters.
                # Add the scaling parameters to the nested dictionary:
                nested_dict['scaling_params'] = scaling_params
                    
                # add the scaler object to the nested dictionary:
                nested_dict['scaler_obj'] = scaler
                
                # Add the scaler_details dictionary:
                nested_dict['scaler_details'] = scaler_details
                
                # Now, all steps are the same for all cases, so we can go back to the main
                # for loop:
        
            # Create the new_column name:
            new_column = column + suffix
            # Create the new_column by dividing the previous column by the scaling factor:
                        
            # Set the new column as scaled_feature
            new_df[new_column] = scaled_feature
                    
            # Add the nested dictionary to the scaling_dict:
            scaling_dict['scaler'] = nested_dict
                    
            # Finally, append the scaling_dict to the list scaling_list:
            scaling_list.append(scaling_dict)
                        
            print(f"Successfully scaled column {column}.\n")
                    
        print("Successfully scaled the dataframe. Returning the transformed dataframe and the scaling dictionary.\n")
        print("Check 10 first rows of the new dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_df.head(10))
                
        except: # regular mode
            print(new_df.head(10))
    
        return new_df, scaling_list


    def reverse_feature_scaling (df, subset_of_features_to_scale, list_of_scaling_params, mode = 'min_max', suffix = '_reverseScaling'):
        
        import pandas as pd
        from sklearn.preprocessing import StandardScaler
        from sklearn.preprocessing import MinMaxScaler
        # Scikit-learn Preprocessing data guide:
        # https://scikit-learn.org/stable/modules/preprocessing.html#preprocessing-scaler
        # Standard scaler documentation:
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html
        # Min-Max scaler documentation:
        # https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html#sklearn.preprocessing.MinMaxScaler.set_params
        
        ## mode = 'standard': reverses the standard scaling, 
        ##  which creates a new variable with mean = 0; and standard deviation = 1.
        ##  Each value Y is transformed as Ytransf = (Y - u)/s, where u is the mean 
        ##  of the training samples, and s is the standard deviation of the training samples.
        
        ## mode = 'min_max': reverses min-max normalization, with a resultant feature 
        ## ranging from 0 to 1. each value Y is transformed as 
        ## Ytransf = (Y - Ymin)/(Ymax - Ymin), where Ymin and Ymax are the minimum and 
        ## maximum values of Y, respectively.
        ## mode = 'factor': reverses the division of the whole series by a numeric value 
        # provided as argument. 
        ## For a factor F, the new Y transformed values are Ytransf = Y/F.
        # Notice that if the original mode was 'normalize_by_maximum', then the maximum value used
        # must be declared as any other factor.
        
        # df: the whole dataframe to be processed.
        
        # subset_of_features_to_be_scaled: list of strings (inside quotes), 
        # containing the names of the columns with the categorical variables that will be 
        # encoded. If a single column will be encoded, declare this parameter as list with
        # only one element e.g.subset_of_features_to_be_scaled = ["column1"] 
        # will analyze the column named as 'column1'; 
        # subset_of_features_to_be_scaled = ["col1", 'col2', 'col3'] will analyze 3 columns
        # with categorical variables: 'col1', 'col2', and 'col3'.
        
        # list_of_scaling_params is a list of dictionaries with the same format of the list returned
        # from this function. Each dictionary must correspond to one of the features that will be scaled,
        # but the list do not have to be in the same order of the columns - it will check one of the
        # dictionary keys.
        # The first key of the dictionary must be 'column'. This key must store a string with the exact
        # name of the column that will be scaled.
        # the second key must be 'scaler'. This key must store a dictionary. The dictionary must store
        # one of two keys: 'scaler_obj' - sklearn scaler object to be used; or 'scaler_details' - the
        # numeric parameters for re-calculating the scaler without the object. The key 'scaler_details', 
        # must contain a nested dictionary. For the mode 'min_max', this dictionary should contain 
        # two keys: 'min', with the minimum value of the variable, and 'max', with the maximum value. 
        # For mode 'standard', the keys should be 'mu', with the mean value, and 'sigma', with its 
        # standard deviation. For the mode 'factor', the key should be 'factor', and should contain the 
        # factor for division (the scaling value. e.g 'factor': 2.0 will divide the column by 2.0.).
        # Again, if you want to normalize by the maximum, declare the maximum value as any other factor for
        # division.
        # The key 'scaler_details' will not create an object: the transform will be directly performed 
        # through vectorial operations.
        
        # suffix: string (inside quotes).
        # How the transformed column will be identified in the returned data_transformed_df.
        # If y_label = 'Y' and suffix = '_reverseScaling', the transformed column will be
        # identified as '_reverseScaling'.
        # Alternatively, input inside quotes a string with the desired suffix. Recommendation:
        # start the suffix with "_" to separate it from the original name
        
        if (suffix is None):
            #set as the default
            suffix = '_reverseScaling'
        
        #Start a copy of the original dataframe. This copy will be updated to create the new
        # transformed dataframe. Then, we avoid manipulating the original object.
        new_df = df.copy(deep = True)
        
        #Start an scaling list empty (it will be a JSON object):
        scaling_list = []
        
        # Use a previously obtained scaler:
        
        for column in subset_of_features_to_scale:
            
            # Create a dataframe X by subsetting only the analyzed column
            # it will be equivalent to using .reshape(-1,1) to set a 1D-series
            # or array in the shape for scikit-learn:
            # For doing so, pass a list of columns for column filtering, containing
            # the object column as its single element:
            X = new_df[[column]]

            # Loop through each element of the list:
                
            for scaling_dict in list_of_scaling_params:
                    
                # check if the dictionary is from that column:
                if (scaling_dict['column'] == column):
                        
                    # We found the correct dictionary. Let's retrieve the information:
                    # retrieve the nested dictionary:
                    nested_dict = scaling_dict['scaler']
                        
                    # try accessing the scaler object:
                    try:
                        scaler = nested_dict['scaler_obj']
                        #calculate the reversed scaled feature, and store it as new array:
                        rev_scaled_feature = scaler.inverse_transform(X)
                            
                        # Add the parameters to the nested dictionary:
                        nested_dict['scaling_params'] = scaler.get_params(deep = True)
                            
                        if (mode == 'standard'):
                                
                            nested_dict['scaler_details'] = {
                                    'mu': rev_scaled_feature.mean(),
                                    'sigma': rev_scaled_feature.std()
                                }
                            
                        elif (mode == 'min_max'):
                                
                            nested_dict['scaler_details'] = {
                                    'min': rev_scaled_feature.min(),
                                    'max': rev_scaled_feature.max()
                                }
                        
                    except:
                            
                        try:
                            # As last alternative, let's try accessing the scaler details dict
                            scaler_details = nested_dict['scaler_details']
                                    
                            if (mode == 'standard'):
                                    
                                nested_dict['scaling_params'] = 'standard_scaler_manually_defined'
                                mu = scaler_details['mu']
                                sigma = scaler_details['sigma']
                                        
                                if (sigma != 0):
                                    # scaled_feature = (X - mu)/sigma
                                    rev_scaled_feature = (X * sigma) + mu
                                else:
                                    # scaled_feature = (X - mu)
                                    rev_scaled_feature = (X + mu)
                                    
                            elif (mode == 'min_max'):
                                        
                                nested_dict['scaling_params'] = 'min_max_scaler_manually_defined'
                                minimum = scaler_details['min']
                                maximum = scaler_details['max']
                                        
                                if ((maximum - minimum) != 0):
                                    # scaled_feature = (X - minimum)/(maximum - minimum)
                                    rev_scaled_feature = (X * (maximum - minimum)) + minimum
                                else:
                                    # scaled_feature = X/maximum
                                    rev_scaled_feature = (X * maximum)
                                    
                            elif (mode == 'factor'):
                                    
                                nested_dict['scaling_params'] = 'normalization_by_factor'
                                factor = scaler_details['factor']
                                # scaled_feature = X/(factor)
                                rev_scaled_feature = (X * factor)
                                    
                            else:
                                print("Select a valid mode: standard, min_max, or factor.\n")
                                return "error", "error"
                                
                        except:
                                    
                            print(f"No valid scaling dictionary was input for column {column}.\n")
                            return "error", "error"
            
                    # Create the new_column name:
                    new_column = column + suffix
                    # Create the new_column by dividing the previous column by the scaling factor:

                    # Set the new column as rev_scaled_feature
                    new_df[new_column] = rev_scaled_feature

                    # Add the nested dictionary to the scaling_dict:
                    scaling_dict['scaler'] = nested_dict

                    # Finally, append the scaling_dict to the list scaling_list:
                    scaling_list.append(scaling_dict)

                    print(f"Successfully re-scaled column {column}.\n")
                    
        print("Successfully re-scaled the dataframe.\n")
        print("Check 10 first rows of the new dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(new_df.head(10))
                
        except: # regular mode
            print(new_df.head(10))
                    
        return new_df, scaling_list


    def lag_diagnosis (df, column_to_analyze, number_of_lags = 40, x_axis_rotation = 0, y_axis_rotation = 0, grid = True, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import statsmodels.api as sm
        
        # df: the whole dataframe to be processed.
        
        #column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        #number_of_lags: integer value. e.g. number_of_lags = 50
        # represents how much lags will be tested, and the length of the horizontal axis.
        
        # Set a copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        #Define the series to be analyzed:
        y = DATASET[column_to_analyze]
        
        #Create the figure:
        fig = plt.figure(figsize = (12, 8)) 
        ax1 = fig.add_subplot(211)
        #ax1.set_xlabel("Lags")
        ax1.set_ylabel("Autocorrelation_Function_ACF")
        
        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 0 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)
        
        ax1.grid(grid)
        
        fig = sm.graphics.tsa.plot_acf(y.values.squeeze(), lags = number_of_lags, ax = ax1, color = 'darkblue')
        ax2 = fig.add_subplot(212)
        fig = sm.graphics.tsa.plot_pacf(y, lags = number_of_lags, ax = ax2, color = 'darkblue', method = 'ywm')
        ax2.set_xlabel("Lags")
        ax2.set_ylabel("Partial_Autocorrelation_Function_PACF")
            
        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 0 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)
        
        ax2.grid(grid)
            
        if (export_png == True):
            # Image will be exported
            import os

            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""

            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "lag_diagnosis"

            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330

            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)

            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

        #fig.tight_layout()

        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        
        #Print background and interpretation of the graphic:
        print("\n") #line break
        print("Use this plot to define the parameters (p, q) for testing ARIMA and ARMA models.\n")
        print("p defines the order of the autoregressive part (AR) of the time series.")
        print("p = lags correspondent to the spikes of PACF plot (2nd plot) that are outside the error (blue region).\n")
        print("For instance, if there are spikes in both lag = 1 and lag = 2, then p = 2, or p = 1\n")
        print("q defines the order of the moving average part (MA) of the time series.")
        print("q = lags correspondent to the spikes of ACF plot that are outside blue region.\n")
        print("For instance, if all spikes until lag = 6 are outside the blue region, then q = 1, 2, 3, 4, 5, 6.\n")
        print("WARNING: do not test the ARIMA/ARMA model for p = 0, or q = 0.")
        print("For lag = 0, the correlation and partial correlation coefficients are always equal to 1, because the data is always perfectly correlated to itself.") 
        print("Therefore, ignore the first spikes (lag = 0) from ACF and PACF plots.")


    def test_d_parameters (df, column_to_analyze, number_of_lags = 40, max_tested_d = 2, confidence_level = 0.95, x_axis_rotation = 0, y_axis_rotation = 0, grid = True, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):

        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import statsmodels.api as sm
        from statsmodels.tsa.stattools import adfuller
        
        #df: the whole dataframe to be processed.
        
        #column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        #max_tested_d: differential order (integer value)
        #change the integer if you want to test other cases. By default, max_tested_d = 2, meaning
        # that the values d = 0, 1, and 2 will be tested.
        # If max_tested_d = 1, d = 0 and 1 will be tested.
        # If max_tested_d = 3, d = 0, 1, 2, and 3 will be tested, and so on.
        
        #column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # CONFIDENCE_LEVEL = 0.95 = 95% confidence
        # Set CONFIDENCE_LEVEL = 0.90 to get 0.90 = 90% confidence in the analysis.
        # Notice that, when less trust is needed, we can reduce CONFIDENCE_LEVEL 
        # to get less restrictive results.
        
        #number_of_lags: integer value. e.g. number_of_lags = 50
        # represents how much lags will be tested, and the length of the horizontal axis.
        
        
        # Set a copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        #Define the series to be analyzed:
        time_series = DATASET[column_to_analyze]
        
        # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
        # so that the bars do not completely block other views.
        OPACITY = 0.95
        
        #Create the figure:
        # Original Series
        fig, axes = plt.subplots((max_tested_d + 1), 2, sharex = True, figsize = (12, 8)) 
        # sharex = share axis X
        # number of subplots equals to the total of orders tested (in this case, 2)
        # If max_tested_d = 2, we must have a subplot for d = 0, d = 1 and d = 2, i.e.,
        # wer need 3 subplots = max_tested_d + 1
        axes[0, 0].plot(time_series, color = 'darkblue', alpha = OPACITY); axes[0, 0].set_title('Original Series')
        sm.graphics.tsa.plot_acf(time_series, lags = number_of_lags, ax = axes[0, 1], color = 'darkblue', alpha = 0.30)
        
        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 0 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)
        
        axes[0, 0].grid(grid)
        
        # Create a subplot for each possible 'd'.
        # Notice that d = 0 was already tested.
        for i in range(1, (max_tested_d + 1)):
            # This loop goes from i = 1 to i = (max_tested_d + 1) - 1 = max_tested_d.
            # If max_tested_d = 2, this loop goes from i = 1 to i = 2.
            # If only one value was declared in range(X), then the loop would start from 0.
            
            # Difference the time series:
            time_series = time_series.diff()
            
            #the indexing of the list d goes from zero to len(d) - 1
            # 1st Differencing
            axes[i, 0].plot(time_series, color = 'darkblue', alpha = OPACITY); axes[i, 0].set_title('%d Order Differencing' %(i))
            sm.graphics.tsa.plot_acf(time_series.diff().dropna(), lags = number_of_lags, ax = axes[i, 1], color = 'darkblue', alpha = 0.30)
                    
            #ROTATE X AXIS IN XX DEGREES
            plt.xticks(rotation = x_axis_rotation)
            # XX = 0 DEGREES x_axis (Default)
            #ROTATE Y AXIS IN XX DEGREES:
            plt.yticks(rotation = y_axis_rotation)
            # XX = 0 DEGREES y_axis (Default)

            axes[i, 0].grid(grid)
        
            print('ADF Statistic for %d Order Differencing' %(i))
            result = adfuller(time_series.dropna())
            print('ADF Statistic: %f' % result[0])
            print('p-value: %f' % result[1])
            print("Null-hypothesis: the process is non-stationary. p-value represents this probability.")

            if (result[1] < (1-confidence_level)):
                print("For a %.2f confidence level, the %d Order Difference is stationary." %(confidence_level, i))
                print("You may select d = %d\n" %(i))

            else:
                print("For a %.2f confidence level, the %d Order Difference is non-stationary.\n" %(confidence_level, i))
            
        if (export_png == True):
            # Image will be exported
            import os

            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""

            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "test_d_parameters"

            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330

            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)

            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

        #fig.tight_layout()

        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        
        print("\n")
        print("d = differentiation order for making the process stationary.\n")
        print("If d = N, then we have to make N successive differentiations.")
        print("A differentiation consists on substracting a signal Si from its previous signal Si-1.\n")
        print("Example: 1st-order differentiating consists on taking the differences on the original time series.")
        print("The 2nd-order, in turns, consists in differentiating the 1st-order differentiation series.")


    def best_arima_model (df, column_to_analyze, p_vals, d, q_vals, timestamp_tag_column = None, confidence_level = 0.95, x_axis_rotation = 70, y_axis_rotation = 0, grid = True, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        # https://www.statsmodels.org/stable/generated/statsmodels.tsa.arima.model.ARIMAResults.html#statsmodels.tsa.arima.model.ARIMAResults
        # https://www.statsmodels.org/stable/examples/notebooks/generated/tsa_arma_1.html?highlight=statsmodels%20graphics%20tsaplots%20plot_predict
        
        ## d = 0 corresponds to the ARMA model
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import statsmodels as sm
        from statsmodels.graphics.tsaplots import plot_predict
        #this model is present only in the most recent versions of statsmodels

        from statsmodels.tsa.arima.model import ARIMA, ARIMAResults
        
        #df: the whole dataframe to be processed.
        
        #column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed. 
        # e.g. column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # timestamp_tag_column = None - keep it as None if you do not want to inform the timestamps
        # Alternatively, declare a string (inside quotes), 
        # containing the name of the column containing the time information. 
        # e.g. timestamp_tag_column = "DATE" will take the timestamps from column 'DATE'.
        # If no column is provided, the index in the dataframe will be used.
        
        #p_vals: list of integers correspondent to the lags (spikes) in the PACF plot.
        # From function lag_diagnosis
        #q_vals: list of integers correspondent to the lags (spikes) in ACF plot
        # From function lag_diagnosis
        #d = difference for making the process stationary
        # From function test_d_parameters
        
        ## WARNING: do not test the ARIMA/ARMA model for p = 0, or q = 0.
        ## For lag = 0, the correlation and partial correlation coefficients 
        ## are always equal to 1, because the data is perfectly correlated to itself. 
        ## Therefore, ignore the first spikes (lag = 0) of ACF and PACF plots.
        
        ALPHA = 1 - confidence_level
        # CONFIDENCE_LEVEL = 0.95 = 95% confidence
        # Set CONFIDENCE_LEVEL = 0.90 to get 0.90 = 90% confidence in the analysis.
        # Notice that, when less trust is needed, we can reduce CONFIDENCE_LEVEL 
        # to get less restrictive results.
        
        # Set a copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        #x: timestamp or index series, if no timestamp is provided:
        if (timestamp_tag_column is None):
            #Use the indices of the dataframe
            x = DATASET.index
        
        else:
            #Use the timestamp properly converted to datetime (as in the graphics functions):
            x = (DATASET[timestamp_tag_column]).astype('datetime64[ns]')
            DATASET[timestamp_tag_column] = x
            # To pass the date to ARIMA model, we must specify it as the index, and also pass the
            # start and end dates
            DATASET = DATASET.set_index(timestamp_tag_column)
        
        #y: tested variable series
        y = DATASET[column_to_analyze]
        
        # date to start the plot (the second element from series x - we will ignore the first one):
        start_date = x[1]
        # last date to plot (last element from the series)
        end_date = x[(len(x) - 1)]
        print(f"ARIMA model from date (or measurement) = {start_date}; to date (or measurement) = {end_date}.\n")
        
        #calculate the first aic and bic
        #first argument = time series y
        
        returned_p = p_vals[0]
        returned_q = q_vals[0]
        returned_d = d
        #use the d-value selected in the non-stationary analysis.
        #set the integration in 
        #at this moment, the function simply returns the first elements.
        
        ARIMA_model = ARIMA(y, order = (returned_p, returned_d, returned_q))

        #order = (p, d, q) - these are the parameters of the autoregression (p = 2), 
        #integration (d = parameter selected in previous analysis), and
        #moving average (q = 1, 2, 3, 4, 5, etc)

        ARIMA_Results = ARIMA_model.fit()
        aic_val = ARIMA_Results.aic
        #AIC value for the first combination.
        bic_val = ARIMA_Results.bic
        #BIC value for the first combination, to start the loops.
        
        # Mean absolute error:
        mae = ARIMA_Results.mae
        # log likelihood calculated:
        loglikelihood = ARIMA_Results.llf
        
        returned_ARIMA_Results = ARIMA_Results
        #returned object
        
        for p in p_vals:
            #test each possible value for p (each d, p combination)
            #each p in p_vals list is used.
            for q in q_vals:
                #test each possible value for q (each p, d, q combination)
                #each q in q_vals list is used.
                    
                ARIMA_model = ARIMA(y, order = (p, returned_d, q))
                ARIMA_Results = ARIMA_model.fit()
                aic_tested = ARIMA_Results.aic
                bic_tested = ARIMA_Results.bic
                mae_tested = ARIMA_Results.mae
                loglikelihood_tested = ARIMA_Results.llf
                
                if ((mae_tested < mae) & (abs(loglikelihood_tested) > abs(loglikelihood))):
                    
                    # Check if the absolute error was reduced and the likelihood was increased
                    
                    #if better parameters were found, they should be used
                    #update AIC, BIC; the p, d, q returned values;
                    #and the ARIMA_Results from the ARIMA:
                    aic_val = aic_tested
                    bic_val = bic_tested
                    mae = mae_tested
                    loglikelihood = loglikelihood_tested
                    returned_p = p

                    returned_q = q
                    #return the Statsmodels object:
                    returned_ARIMA_Results = ARIMA_Results
        
        #Create a dictionary containing the best parameters and the metrics AIC and BIC
        arima_summ_dict = {"p": returned_p, "d": returned_d, "q": returned_q,
                    "AIC": returned_ARIMA_Results.aic, "BIC": returned_ARIMA_Results.bic,
                        "MAE": returned_ARIMA_Results.mae, "log_likelihood": returned_ARIMA_Results.llf}
        
        #Show ARIMA results:
        print(returned_ARIMA_Results.summary())
        print("\n")
        #Break the line and show the combination
        print("Best combination found: (p, d, q) = (%d, %d, %d)\n" %(returned_p, returned_d, returned_q))
        #Break the line and print the next indication:
        print(f"Time series, values predicted by the model, and the correspondent {confidence_level * 100}% Confidence interval for the predictions:\n")
        #Break the line and print the ARIMA graphic:
        
        # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
        # so that the bars do not completely block other views.
        OPACITY = 0.95
        
        #Start the figure:
        fig, ax = plt.subplots(figsize = (12, 8))
        # Add line of the actual values:
        ax.plot(x, y, linestyle = '-', marker = '', color = 'darkblue', label = column_to_analyze)
        # Use the name of the analyzed column as the label
        fig = plot_predict(returned_ARIMA_Results, start = start_date, end = end_date,  ax = ax, alpha = ALPHA)
        ## https://www.statsmodels.org/v0.12.2/generated/statsmodels.tsa.arima_model.ARIMAResults.plot_predict.html
        # ax = ax to plot the arima on the original plot of the time series
        # start = x[1]: starts the ARIMA graphic from the second point of the series x
        # if x is the index, then it will be from the second x; if x is a time series, then
        # x[1] will be the second timestamp
        #We defined the start in x[1], instead of index = 0, because the Confidence Interval for the 
        #first point is very larger than the others (there is perfect autocorrelation for lag = 0). 
        #Therefore, it would have resulted in the need for using a very broader y-scale, what
        #would compromise the visualization.
        # We could set another index or even a timestamp to start:
        # start=pd.to_datetime('1998-01-01')
        
        ax.set_alpha(OPACITY)
        
        if not (plot_title is None):
            #graphic's title
            ax.set_title(plot_title)
        
        else:
            #set a default title:
            ax.set_title("ARIMA_model")
        
        if not (horizontal_axis_title is None):
            #X-axis title
            ax.set_xlabel(horizontal_axis_title)
        
        else:
            #set a default title:
            ax.set_xlabel("Time")
        
        if not (vertical_axis_title is None):
            #Y-axis title
            ax.set_ylabel(vertical_axis_title)
        
        else:
            #set a default title:
            ax.set_ylabel(column_to_analyze)
        
        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 70 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)
        
        ax.grid(grid)
        ax.legend(loc = "upper left")
        
        if (export_png == True):
            # Image will be exported
            import os

            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""

            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "arima_model"

            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330

            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)

            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

        #fig.tight_layout()

        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        
        # Get dataframe with the predictions and confidence intervals
        arima_predictions = returned_ARIMA_Results.get_prediction(start = x[0], end = None, dynamic = False, full_results = True, alpha = ALPHA)
        # Here, we started in start = x[0] to obtain a correspondent full dataset, and did not set an end.
        # The start can be an integer representing the index of the data in the dataframe,
        # or a timestamp. Check:
        # https://www.statsmodels.org/devel/generated/statsmodels.tsa.arima.model.ARIMAResults.get_prediction.html#statsmodels.tsa.arima.model.ARIMAResults.get_prediction
        # Again, no matter if x is an index or a time series, x[0] starts from the first element
        # You could set another index or a oparticular timestamp to start, like:
        # start=pd.to_datetime('1998-01-01')
        # The dynamic = False argument ensures that we produce one-step ahead forecasts, 
        # meaning that forecasts at each point are generated using the full history up 
        # to that point.
        predicted_mean_vals = arima_predictions.predicted_mean
        predicted_conf_intervals = arima_predictions.conf_int(alpha = ALPHA)
        # predicted_conf_intervals has two columns: first one is the inferior confidence limit
        # second one is the superior confidence limit
        # each column from this dataframe gets a name derived from the name of the original series.
        # So, let's rename them:
        predicted_conf_intervals.columns = ['lower_cl', 'upper_cl']
        
        # let's create a copy of the dataframe to be returned with the new information:
        # This will avoid manipulating the df:
        arima_df = DATASET.copy(deep = True)
        # This DATASET already contains the timestamp column as the index, so it is adequate for
        # obtaining predictions associated to the correct time values.
        
        #create a column for the predictions:
        arima_df['arima_predictions'] = predicted_mean_vals
        
        #create a column for the inferior (lower) confidence interval.
        # Copy all the rows from the column 0 of predicted_conf_intervals
        arima_df['lower_cl'] = predicted_conf_intervals['lower_cl']
        
        #create a column for the superior (upper) confidence interval.
        # Copy all the rows from the column 1 of predicted_conf_intervals
        arima_df['upper_cl'] = predicted_conf_intervals['upper_cl']
        
        # Let's turn again the timestamps into a column, restart the indices and re-order the columns,
        # so that the last column will be the response, followed by ARIMA predictions
        
        ordered_columns_list = []
        
        if (timestamp_tag_column is not None):
            #Use the indices of the dataframe
            ordered_columns_list.append(timestamp_tag_column)
            # Create a timestamp_tag_column in the dataframe containing the values in the index:
            arima_df[timestamp_tag_column] = np.array(arima_df.index)
            # Reset the indices from the dataframe:
            arima_df = arima_df.reset_index(drop = True)
        
        # create a list of columns with fixed position:
        fixed_columns_list = [column_to_analyze, 'arima_predictions', 'lower_cl', 'upper_cl']
        
        # Now, loop through all the columns:
        for column in list(arima_df.columns):
            
            # If the column is not one from fixed_columns_list, and it is not on ordered_columns_list
            # yet, add it to ordered_columns_list (timestamp_tag_column may be on the list):
            if ((column not in fixed_columns_list) & (column not in ordered_columns_list)):
                ordered_columns_list.append(column)
        
        # Now, concatenate ordered_columns_list to fixed_columns_list. 
        # If a = ['a', 'b'] and b = ['c', 'd'], a + b = ['a', 'b', 'c', 'd'] and b + a = [ 'c', 'd', 'a', 'b']
        ordered_columns_list = ordered_columns_list + fixed_columns_list
        
        # Finally, select the columns from arima_df passing this list as argument:
        arima_df = arima_df[ordered_columns_list]
        print("\n")
        print("Check the dataframe containing the ARIMA predictions:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(arima_df)
                
        except: # regular mode
            print(arima_df)
        
        print("\n") #line break
        print("Notice that the presence of data outside the confidence interval limits of the ARIMA forecast is a strong indicative of outliers or of untrust time series.\n")
        print("For instance: if you observe a very sharp and sudden deviation from the predictive time series, it can be an indicative of incomplete information or outliers presence.\n")
        print("A famous case is the pandemic data: due to lags or latencies on the time needed for consolidating the information in some places, the data could be incomplete in a given day, leading to a sharp decrease that did not actually occurred.")
        
        print("\n")
        print("REMEMBER: q represents the moving average (MA) part of the time series.")
        print(f"Then, it is interesting to group the time series {column_to_analyze} by each q = {returned_q} periods when modelling or analyzing it.")
        print("For that, you can use moving window or rolling functions.\n")
        
        return returned_ARIMA_Results, arima_summ_dict, arima_df


    def arima_forecasting (arima_model_object, df = None, column_to_forecast = None, timestamp_tag_column = None, time_unit = None, number_of_periods_to_forecast = 7, confidence_level = 0.95, plot_predicted_time_series = True, x_axis_rotation = 70, y_axis_rotation = 0, grid = True, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import statsmodels as sm
        from statsmodels.graphics.tsaplots import plot_predict
        #this model is present only in the most recent versions of statsmodels

        from statsmodels.tsa.arima.model import ARIMA, ARIMAResults
        
        # arima_model_object : object containing the ARIMA model previously obtained.
        # e.g. arima_model_object = returned_ARIMA_Results if the model was obtained as returned_ARIMA_Results
        # do not declare in quotes, since it is an object, not a string.
        
        # time_unit: Unit of the new column. If no value is provided, the unit will be considered as nanoseconds. 
        # Alternatively: keep it None, for the results in nanoseconds, or input time_unit = 
        # 'year', 'month', 'day', 'hour', 'minute', or 'second' (keep these inside quotes).
        # It will be the input for the functions CALCULATE_DELAY and ADD_TIMEDELTA.
        
        # number_of_periods_to_forecast = 7
        # integer value representing the total of periods to forecast. The periods will be in the
        # unit (dimension) of the original dataset. If 1 period = 1 day, 7 periods will represent
        # seven days.
        
        # Keep plot_predicted_time_series = True to see the graphic of the predicted values.
        # Alternatively, set plot_predicted_time_series = True not to show the plot.
        
        # df = None, column_to_analyze = None - keep it as None if you do not want to show
        # the ARIMA predictions combined to the original data; or if you do not want to append
        # the ARIMA predictions to the original dataframe.
        # Alternatively, set:
        # df: the whole dataframe to be processed.
        
        # column_to_forecast: string (inside quotes), 
        # containing the name of the column that will be analyzed.
        # Keep it as None if the graphic of the predictions will not be shown with the
        # responses or if the combined dataset will not be returned.
        # e.g. column_to_forecast = "column1" will analyze and forecast values for 
        # the column named as 'column1'.
        
        # timestamp_tag_column: string (inside quotes), 
        # containing the name of the column containing timestamps.
        # Keep it as None if the graphic of the predictions will not be shown with the
        # responses or if the combined dataset will not be returned.
        # e.g. timestamp_tag_column = "column1" will analyze the column named as 'column1'.
        
        ## ARIMA predictions:
        ## The .forecast and .predict methods only produce point predictions:
        ## y_forecast = ARIMA_Results.forecast(7) results in a group of values (7 predictions)
        ## without the confidence intervals
        ## On the other hand, the .get_forecast and .get_prediction methods 
        ## produce full results including prediction intervals.

        ## In our example, we can do:
        ## forecast = ARIMA_Results.get_forecast(123)
        ## yhat = forecast.predicted_mean
        ## yhat_conf_int = forecast.conf_int(alpha=0.05)
        ## If your data is a Pandas Series, then yhat_conf_int will be a DataFrame with 
        ## two columns, lower <name> and upper <name>, where <name> is the name of the Pandas 
        ## Series.
        ## If your data is a numpy array (or Python list), then yhat_conf_int will be an 
        ## (n_forecasts, 2) array, where the first column is the lower part of the interval 
        ## and the second column is the upper part.
    
        numeric_data_types = [np.float16, np.float32, np.float64, np.int16, np.int32, np.int64]
        
        ALPHA = 1 - confidence_level
        # CONFIDENCE_LEVEL = 0.95 = 95% confidence
        # Set CONFIDENCE_LEVEL = 0.90 to get 0.90 = 90% confidence in the analysis.
        # Notice that, when less trust is needed, we can reduce CONFIDENCE_LEVEL 
        # to get less restrictive results.
        
        # Calculate the predictions:
        # It does not depend on the presence of a dataframe df
        # https://www.statsmodels.org/devel/generated/statsmodels.tsa.arima.model.ARIMAResults.get_forecast.html#statsmodels.tsa.arima.model.ARIMAResults.get_forecast
        
        arima_forecasts = arima_model_object.get_forecast(number_of_periods_to_forecast, dynamic = False, full_results = True, alpha = ALPHA)
        
        forecast_mean_vals = arima_forecasts.predicted_mean
        forecast_conf_intervals = arima_forecasts.conf_int(alpha = ALPHA)
        # forecast_conf_intervals has two columns: first one is the inferior confidence limit
        # second one is the superior confidence limit
        # each column from this dataframe gets a name derived from the name of the original series.
        # So, let's rename them:
        forecast_conf_intervals.columns = ['lower_cl', 'upper_cl']
        
        #create a series for the inferior confidence interval.
        lower_cl = forecast_conf_intervals['lower_cl'].copy(deep = True)
        #create a series for the superior confidence interval.
        upper_cl = forecast_conf_intervals['upper_cl'].copy(deep = True)
        
        # If there is no df, we can already obtain a X series, that will be a series of indices
        # starting from period 1 (the first forecast. Period zero corresponds to the last actual value):
        if (df is None):
            
            x_forecast = []
            
            for j in range (1, (number_of_periods_to_forecast + 1)):
                #Goes from j = 1, the first forecast period; to j = (number_of_periods_to_forecast+1)-1
                # = number_of_periods_to_forecast, which must be the last forecast.
                x_forecast.append(j)
            
            # Now, create the dictionary of forecasts:
            forecast_dict = {
                
                "x": x_forecast,
                "forecast_mean_vals": forecast_mean_vals,
                "lower_cl": lower_cl,
                "upper_cl": upper_cl,
                'source': 'forecast'
            }
            
            # Convert it to a dataframe:
            forecast_df = pd.DataFrame(data = forecast_dict)
            x_forecast_series = forecast_df['x']
            y_forecast_series = forecast_df['forecast_mean_vals']
            lcl_series = forecast_df['lower_cl']
            ucl_series = forecast_df['upper_cl']
        
        
        # If there is a dataframe df, we must combine the original data from df
        # with the new predictions:
        else:
            
            # Start a dataset copy to manipulate:
            DATASET = df.copy(deep = True)
            
            # Create a column with a label indicating that the data in DATASET (before concatenation 
            # with predictions) is from the original dataframe:
            DATASET['source'] = 'input_dataframe'
            # In turns, The source column in the dataframe from the forecasts will 
            # be labelled with the string 'forecast'
            
            # Check if a column_to_forecast was indicated in the input dataframe:
            if not (column_to_forecast is None):
                
                # If there is a response column indicated, then the forecast column of the 
                # generated predictions must be stored in a column with the exact same name, so that
                # the columns can be correctly appended
                y_forecast_label = column_to_forecast
                # Also, create a separate series for the original data, that will be used for
                # differentiating between data and forecasts on the plot:
                y_original_series = DATASET[column_to_forecast].copy(deep = True)
            
            # If no column was indicated, set a default column name for the forecasts:
            else:
                # Set the default name:
                y_forecast_label = "y_forecast"
            
            
            # Check if a timestamp_tag_column was input. If not, use the indices themselves as times.
            # Create a new standard name for the column in forecasts:
            if (timestamp_tag_column is None):
                
                # Let's set an index series as the index of the dataframe:
                DATASET['index_series'] = DATASET.index
                # Check if this series contains an object. If it has, then, user set the timestamps
                # as the indices
                index_series_type = DATASET['index_series'].dtype
                
                # If it is an object, the user may be trying to pass the date as index. 
                # So, let's try to convert it to datetime:
                if ((index_series_type not in numeric_data_types) | (index_series_type == 'O') | (index_series_type == 'object')):
                    
                    try:
                        DATASET['index_series'] = (DATASET['index_series']).astype('datetime64[ns]')
                        
                        # Rename column 'index_series':
                        # https://www.statology.org/pandas-rename-columns/
                        DATASET.rename(columns = {'index_series': 'timestamp'}, inplace = True)
                        # Set 'timestamp' as the timestamp_tag_column:
                        timestamp_tag_column = 'timestamp'
                        
                    except:
                        # A variable that is not neither numeric nor date was passed. Reset the index:
                        DATASET = DATASET.reset_index(drop = True)
                        # Update the index series
                        DATASET['index_series'] = DATASET.index
                        
                # Now, try to manipulate the 'index_series'. An exception will be raised in case
                # the name was changed because the index contains a timestamp
                try:
                    
                    # convert the 'index_series' to the default name for column X:
                    x_forecast_label  = "x_forecast"
                    DATASET.rename(columns = {'index_series': x_forecast_label}, inplace = True)
                    x_original_series = DATASET[x_forecast_label].copy(deep = True)
                
                    # Let's create the series for forecasted X. Simply add more values to x
                    # until reaching the number_of_periods_to_forecast:
                
                    # Get the value of the last X of the dataframe
                    # index start from zero, so the last one is length - 1
                    last_x = x_original_series[(len(x_original_series) - 1)]

                    #Start the list
                    x_forecast = []

                    #Append its first value: last X plus 1 period
                    x_forecast.append(last_x + 1)
                
                    j = 1
                    while (j < number_of_periods_to_forecast):
                        #Last cycle occurs when j == number_of_periods_to_forecast - 1
                        # Since indexing starts from zero, there will be number_of_periods_to_forecast
                        # elements in the list.
                        # Also, we already added the first period, so we are starting from the 2nd
                        # forecast period, j = 1.
                        x_forecast.append((x_forecast[(j - 1)] + 1))

                        #Go to next iteration:
                        j = j + 1
                    
                    # Now, x_forecast stores the next indices for the situation where no timestamp
                    # was provided initially.
                
                except:
                    #simply pass
                    pass
            
            
            # Again, check if timestamp_tag_column is None. It may have changed, since we created a value
            # for the case where a timestamp is in the index. So, we will not use else: the else would
            # ignore the modification in the first if:
            
            if not (timestamp_tag_column is None):
                # Use the timestamp properly converted to datetime (as in the graphics functions).
                # The labels must be the same for when the dataframes are merged.
                x_forecast_label  = timestamp_tag_column
                
                # Check if it is an object or is not a numeric variable (e.g. if it is a timestamp):
                if ((DATASET[timestamp_tag_column].dtype not in numeric_data_types) | (DATASET[timestamp_tag_column].dtype == 'O') | (DATASET[timestamp_tag_column].dtype == 'object')):
                    
                    # Try to convert it to np.datetime64
                    try:

                        DATASET[timestamp_tag_column] = (DATASET[timestamp_tag_column]).astype('datetime64[ns]')
                    
                    except:
                        pass
                    
                # Now, timestamp is either a numeric column or a datetime column.
                # Again, create a separate series for the original data, that will be used for
                # differentiating between data and forecasts on the plot:
                x_original_series = DATASET[timestamp_tag_column].copy(deep = True)
                
                # Get the last X of the dataframe
                # index start from zero, so the last one is length - 1
                last_x = x_original_series[(len(x_original_series) - 1)]
                
                # Check the case where it is a timestamp:
                if (type(x_original_series[0]) == np.datetime64):
                    
                    # Let's obtain the mean value of the timedeltas between each measurement:
                    TIMESTAMP_TAG_COLUMN = timestamp_tag_column
                    NEW_TIMEDELTA_COLUMN_NAME = None
                    RETURNED_TIMEDELTA_UNIT = time_unit
                    # If it is none, the value will be returned in nanoseconds.
                    # keep it None, for the results in nanoseconds
                    RETURN_AVG_DELAY = True
                    _, avg_delay = etl.CALCULATE_DELAY (df = DATASET, timestamp_tag_column = TIMESTAMP_TAG_COLUMN, new_timedelta_column_name  = NEW_TIMEDELTA_COLUMN_NAME, returned_timedelta_unit = RETURNED_TIMEDELTA_UNIT, return_avg_delay = RETURN_AVG_DELAY)
                    # The underscore indicates that we will not keep the returned dataframe
                    # only the average time delay in nanoseconds.
                    print("\n")
                    print(f"Average delay on the original time series, used for obtaining times of predicted values = {avg_delay}.\n")

                    # Now, avg_delay stores the mean time difference between successive measurements from the
                    # original dataset.
                
                    # Now, let's create the prediction timestamps, by adding the avg_delay to
                    # the last X.
                    # Firstly, convert last_x to a Pandas timestamp, so that we can add a pandas
                    # timedelta:
                    last_x = pd.Timestamp(last_x, unit = 'ns')
                
                    # Now, let's create a pandas timedelta object correspondent to avg_delay
                    # 1. Check units:
                    if (time_unit is None):
                        time_unit = 'ns'
                
                    # Notice that CALCULATE_DELAY and ADD_TIMEDELTA update the unit for us, but
                    # such functions deal with a dataframe, not with a single value. So, we are
                    # using a small piece from ADD_TIMEDELTA function to operate with a single
                    # timestamp.

                    #2. Create the pandas timedelta object:
                    timedelta = pd.Timedelta(avg_delay, time_unit)
                
                    #3. Let's create the values of timestamps correspondent to the forecast
                    x_forecast = []
                    # The number of elements of this list is number_of_periods_to_forecast
                    # if number_of_periods_to_forecast, we will forecast a single period further,
                    # then we need to sum timedelta once. If number_of_periods_to_forecast = 3,
                    # we will sum timedelta 3 times, and so on.
                
                    # Append the first element (last element + timedelta) - first value forecast
                    x_forecast.append((last_x + timedelta))
                
                    j = 1
                    while (j < number_of_periods_to_forecast):
                        #Last cycle occurs when j == number_of_periods_to_forecast - 1
                        # Since indexing starts from zero, there will be number_of_periods_to_forecast
                        # elements in the list.
                        # Also, we already added the first period, so we are starting from the 2nd
                        # forecast period, j = 1.

                        # append the previous element + timedelta.
                        # If j = 1, (j - 1) = 0, the first element
                        x_forecast.append((x_forecast[(j - 1)] + timedelta))
                    
                        #Go to next iteration:
                        j = j + 1
            
                    # Now, x_forecast stores the values of timestamps correspondent to
                    # the forecasts.
                    # Convert x_forecast to Pandas Series, so that it will be possible to perform vectorial
                    # operations:
                    x_forecast = pd.Series(x_forecast)
                    # Convert it to datetime64:
                    x_forecast = (x_forecast).astype('datetime64[ns]')
                
                else:
                    # We have a numerical variable used as time. We have to calculate the average 'delay' between
                    # Successive values. For that, we can again use Pandas.diff method, which may be applied to
                    # Series or DataFrames
                    # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.diff.html
                    # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.diff.html#pandas.Series.diff
                    
                    x_diff_series = x_original_series.copy(deep = True)
                    x_diff_series = x_diff_series.diff(periods = 1)
                    # periods - int, default 1 - Periods to shift for calculating difference, accepts negative values.
                    
                    # Now, get the average delay as the mean value from series x_diff_series:
                    avg_delay = x_diff_series.mean()
                    print("\n")
                    print(f"Average delay on the original time series, used for obtaining times of predicted values = {avg_delay}.\n")
                    
                    # Let's create the values of times correspondent to the forecast
                    x_forecast = []
                    # The number of elements of this list is number_of_periods_to_forecast
                    # if number_of_periods_to_forecast, we will forecast a single period further,
                    # then we need to sum timedelta once. If number_of_periods_to_forecast = 3,
                    # we will sum timedelta 3 times, and so on.
                
                    # Append the first element (last element + avg_delay) - first value forecast
                    x_forecast.append((last_x + avg_delay))
                
                    j = 1
                    while (j < number_of_periods_to_forecast):
                        #Last cycle occurs when j == number_of_periods_to_forecast - 1
                        # Since indexing starts from zero, there will be number_of_periods_to_forecast
                        # elements in the list.
                        # Also, we already added the first period, so we are starting from the 2nd
                        # forecast period, j = 1.

                        # append the previous element + timedelta.
                        # If j = 1, (j - 1) = 0, the first element
                        x_forecast.append((x_forecast[(j - 1)] + avg_delay))
                    
                        #Go to next iteration:
                        j = j + 1
                    
            # Notice that all these steps are possible only when the timestamps were
            # given, and so they should not be executed if the values are not provided.
            
            
            # Now, steps are the same for all cases where there is a dataframe (Main Else node that we are evaluating): 
            # create the dictionary of forecasts:
            forecast_dict = {

                x_forecast_label: x_forecast,
                y_forecast_label: forecast_mean_vals,
                "lower_cl": lower_cl,
                "upper_cl": upper_cl,
                'source': 'forecast'
            }

            # Convert it to a dataframe:
            forecast_df = pd.DataFrame(data = forecast_dict)
            x_forecast_series = forecast_df[x_forecast_label]
            y_forecast_series = forecast_df[y_forecast_label]
            lcl_series = forecast_df['lower_cl']
            ucl_series = forecast_df['upper_cl']

            # Now, let's concatenate the new dataframe to the old one.
            # The use of the variables 'source', x_ and y_forecast_label guarantees that the new
            # columns have the same name as the ones of the original dataframe, so that
            # the concatenation is performed correctly.

            # Now, merge the dataframes:
            LIST_OF_DATAFRAMES = [DATASET, forecast_df]
            IGNORE_INDEX_ON_UNION = True
            SORT_VALUES_ON_UNION = True
            UNION_JOIN_TYPE = None
            forecast_df = etl.UNION_DATAFRAMES (list_of_dataframes = LIST_OF_DATAFRAMES, ignore_index_on_union = IGNORE_INDEX_ON_UNION, sort_values_on_union = SORT_VALUES_ON_UNION, union_join_type = UNION_JOIN_TYPE)
            
            # Full series, with input data and forecasts:
            x = forecast_df[x_forecast_label]
            y = forecast_df[y_forecast_label]
            
            # Now, let's re-order the dataframe, putting the x_forecast_label as the first column
            # and y_forecast_label: forecast_mean_vals, "lower_cl", "upper_cl", and 'source'
            # as the last ones.
            
            # Set a list of the last columns (fixed positions):
            fixed_columns_list = [y_forecast_label, 'lower_cl', 'upper_cl', 'source']
            
            # Start the list with only x_forecast_label:
            ordered_columns_list = [x_forecast_label]
            
            # Now, loop through all the columns:
            for column in list(forecast_df.columns):

                # If the column is not one from fixed_columns_list, and it is not on ordered_columns_list
                # yet, add it to ordered_columns_list (timestamp_tag_column may be on the list):
                if ((column not in fixed_columns_list) & (column not in ordered_columns_list)):
                    ordered_columns_list.append(column)
            
            # Now, concatenate ordered_columns_list to fixed_columns_list. 
            # If a = ['a', 'b'] and b = ['c', 'd'], a + b = ['a', 'b', 'c', 'd'] and b + a = [ 'c', 'd', 'a', 'b']
            ordered_columns_list = ordered_columns_list + fixed_columns_list
            
            # Finally, pass ordered_columns_list as argument for column filtering and re-order:
            forecast_df = forecast_df[ordered_columns_list]

            
        # We are finally in the general case, after obtaining the dataframe through all possible ways:
        print(f"Finished the obtention of the forecast dataset. Check the 10 last rows of the forecast dataset:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(forecast_df.tail(10))
                
        except: # regular mode
            print(forecast_df.tail(10))
            
        
        # Now, let's create the graphics
        if (plot_predicted_time_series == True):
            
            LINE_STYLE = '-'
            MARKER = ''
            
            if (plot_title is None):
                # Set graphic title
                plot_title = f"ARIMA_forecasts"

            if (horizontal_axis_title is None):
                # Set horizontal axis title
                if (timestamp_tag_column is None):
                    horizontal_axis_title = "timestamp"
                else:
                    horizontal_axis_title = timestamp_tag_column

            if (vertical_axis_title is None):
                if (column_to_forecast is None):
                    vertical_axis_title = "time_series"
                else:
                    vertical_axis_title = column_to_forecast
            
            # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
            # so that the bars do not completely block other views.
            OPACITY = 0.95
            
            #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
            fig = plt.figure(figsize = (12, 8))
            ax = fig.add_subplot()
            
            if ((df is not None) & ((column_to_forecast is not None))):
                
                # Plot the original data series:
                ax.plot(x, y, linestyle = LINE_STYLE, marker = MARKER, color = 'darkblue', alpha = OPACITY, label = 'input_dataframe')
            
            # Plot the predictions (completely opaque, so that the input data series will not be
            # visible)
            ax.plot(x_forecast_series, y_forecast_series, linestyle = LINE_STYLE, marker = MARKER, color = 'red', alpha = 1.0, label = 'forecast')
            
            # Plot the confidence limits:
            ax.plot(x_forecast_series, lcl_series, linestyle = 'dashed', marker = MARKER, color = 'magenta', alpha = 0.70, label = 'lower_confidence_limit')
            ax.plot(x_forecast_series, ucl_series, linestyle = 'dashed', marker = MARKER, color = 'magenta', alpha = 0.70, label = 'upper_confidence_limit')    
            # Now we finished plotting all of the series, we can set the general configuration:
            
            #ROTATE X AXIS IN XX DEGREES
            plt.xticks(rotation = x_axis_rotation)
            # XX = 0 DEGREES x_axis (Default)
            #ROTATE Y AXIS IN XX DEGREES:
            plt.yticks(rotation = y_axis_rotation)
            # XX = 0 DEGREES y_axis (Default)

            ax.set_title(plot_title)
            ax.set_xlabel(horizontal_axis_title)
            ax.set_ylabel(vertical_axis_title)

            ax.grid(grid) # show grid or not
            ax.legend()
            # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
            # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
            # https://www.statology.org/matplotlib-legend-position/

            if (export_png == True):
                # Image will be exported
                import os

                #check if the user defined a directory path. If not, set as the default root path:
                if (directory_to_save is None):
                    #set as the default
                    directory_to_save = ""

                #check if the user defined a file name. If not, set as the default name for this
                # function.
                if (file_name is None):
                    #set as the default
                    file_name = "arima_forecast"

                #check if the user defined an image resolution. If not, set as the default 110 dpi
                # resolution.
                if (png_resolution_dpi is None):
                    #set as 330 dpi
                    png_resolution_dpi = 330

                #Get the new_file_path
                new_file_path = os.path.join(directory_to_save, file_name)

                #Export the file to this new path:
                # The extension will be automatically added by the savefig method:
                plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                #quality could be set from 1 to 100, where 100 is the best quality
                #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                #transparent = True or False
                # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

            #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
            #plt.figure(figsize = (12, 8))
            #fig.tight_layout()

            ## Show an image read from an image file:
            ## import matplotlib.image as pltimg
            ## img=pltimg.imread('mydecisiontree.png')
            ## imgplot = plt.imshow(img)
            ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
            ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
            ##  '03_05_END.ipynb'
            plt.show()
        
        print("\nARIMA Forecasting completed.\n")
        
        return forecast_df


    def df_rolling_window_stats (df, window_size = 2, window_statistics = 'mean', min_periods_required = None, window_center = False, window_type = None, window_on = None, row_accross = 'rows', how_to_close_window = None, drop_missing_values = True):
        
        import numpy as np
        import pandas as pd
        
        # Check Pandas rolling statistics documentation:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.rolling.html
        # https://pandas.pydata.org/pandas-docs/stable/user_guide/window.html#window-generic
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.diff.html
        
        ## df: the whole dataframe to be processed.
        
        # window_statistics = 'mean', 'std', 'sum', or 'difference'
        # 'difference' will perform the first discrete difference of element: Yn - Yn-1
        
        # window_size: integer value or offset time.
        # Manipulate parameter window of the .rolling method; or the parameter
        # periods of the .diff method.
        # window = window_size; or periods = window_size.
        # Size of the moving window. If an integer, the fixed number of observations 
        # used for each window. If an offset, the time period of each window. 
        # Each window will be a variable sized based on the observations included in the 
        # time-period. This is only valid for datetime-like indexes. 
        
        # min_periods_required = None. Alternatively, set as an integer value.
        # Manipulate parameter min_periods of .rolling method
        # min_periods = min_periods_required
        # Minimum number of observations in window required to have a value; otherwise, 
        # result is np.nan. For a window that is specified by an offset, min_periods will 
        # default to 1. For a window that is specified by an integer, min_periods will default 
        # to the size of the window.
        
        # window_center = False.
        # Manipulate parameter center of .rolling method
        # center = window_center
        # If False, set the window labels as the right edge of the window index.
        # If True, set the window labels as the center of the window index.
        
        # window_type = None
        # Manipulate parameter win_type of .rolling method
        # win_type = window_type
        # If None, all points are evenly weighted. If a string, it must be a valid 
        # scipy.signal window function:
        # https://docs.scipy.org/doc/scipy/reference/signal.windows.html#module-scipy.signal.windows
        # Certain Scipy window types require additional parameters to be passed in the 
        # aggregation function. The additional parameters must match the keywords specified 
        # in the Scipy window type method signature.
        
        # window_on = None
        # Manipulate parameter on of .rolling method
        # on = window_on
        # string; For a DataFrame, a column label or Index level on which to calculate 
        # the rolling window, rather than the DataFrame’s index. Provided integer column is 
        # ignored and excluded from result since an integer index is not used to calculate 
        # the rolling window.
        
        # row_accross = 'rows'. Alternatively, row_accross = 'columns'
        # manipulate the parameter axis of .rolling method:
        # if row_accross = 'rows', axis = 0; if row_accross = 'columns', axis = 1.
        # If axis = 0 or 'index', roll across the rows.
        # If 1 or 'columns', roll across the columns.
        
        # how_to_close_window = None
        # Manipulate parameter closed of .rolling method
        # closed = how_to_close_window
        # String: If 'right', the first point in the window is excluded from calculations.
        # If 'left', the last point in the window is excluded from calculations.
        # If 'both', the no points in the window are excluded from calculations.
        # If 'neither', the first and last points in the window are excluded from calculations.
        # Default None ('right').
        
        # drop_missing_values = True will remove all missing values created by the methods (all
        # rows containing missing values). 
        # If drop_missing_values = False, the positions containing NAs will be kept.
        
        DATASET = df.copy(deep = True)
        WINDOW = window_size
        MIN_PERIODS = min_periods_required
        CENTER = window_center
        WIN_TYPE = window_type
        ON = window_on
        
        numeric_data_types = [np.float16, np.float32, np.float64, np.int16, np.int32, np.int64]
        
        # Variable to map if the timestamp was correctly parsed. It will be set of True
        # only when it happens:
        date_parser_marker = False
        
        try:
            if (type(DATASET[ON][0]) not in numeric_data_types):
                # Try to Parse the date:
                try:
                    
                    DATASET[ON] = (DATASET[ON]).astype('datetime64[ns]')
                    # Change the value of the marker to map that the date was correctly parsed:
                    date_parser_marker = True
                    print(f"Column {ON} successfully converted to numpy.datetime64[ns].\n")     
                
                except:
                    pass
        except:
            pass
        
        
        if (row_accross == 'columns'):
            
            AXIS = 1
        
        else:
            # 'rows' or an invalid value was set, so set to 'rows' (Axis = 0)
            AXIS = 0
        
        CLOSED = how_to_close_window
        
        # Now all the parameters for the rolling method are set. Calculate the dataframe
        # for the selected statistic:
        
        if (window_statistics == 'mean'):
            
            rolling_window_df = DATASET.rolling(window = WINDOW, min_periods = MIN_PERIODS, center = CENTER, win_type = WIN_TYPE, on = ON, axis = AXIS, closed = CLOSED).mean()
            print(f"Calculated rolling mean for a window size of {WINDOW}. Returning the rolling \'mean\' dataframe.\n")
        
        elif (window_statistics == 'std'):
            
            rolling_window_df = DATASET.rolling(window = WINDOW, min_periods = MIN_PERIODS, center = CENTER, win_type = WIN_TYPE, on = ON, axis = AXIS, closed = CLOSED).std()
            print(f"Calculated rolling standard deviation for a window size of {WINDOW}. Returning the rolling \'std\' dataframe.\n")
        
        elif (window_statistics == 'sum'):
            
            rolling_window_df = DATASET.rolling(window = WINDOW, min_periods = MIN_PERIODS, center = CENTER, win_type = WIN_TYPE, on = ON, axis = AXIS, closed = CLOSED).sum()
            print(f"Calculated rolling sum for a window size of {WINDOW}. Returning the rolling \'sum\' dataframe.\n")
        
        elif (window_statistics == 'difference'):
            
            # Create a list of the columns that can be differentiated and of those that cannot be.
            diff_columns = []
            excluded_columns = []
            for column in list(DATASET.columns):
                
                if (type(DATASET[column][0]) in numeric_data_types):
                    diff_columns.append(column)
                
                else:
                    
                    if ((column == ON) & (date_parser_marker == True)):
                        # This is the column we converted to date. Set it as the index.
                        # It will allow us to calculate the differences without losing this
                        # information
                        DATASET = DATASET.set_index(column)
                    
                    else:
                        excluded_columns.append(column)
            
            # Select only the columns in diff_columns:
            DATASET= DATASET[diff_columns]
            
            if (len(excluded_columns) > 0):
                print(f"It is not possible to calculate the differences for columns in {excluded_columns}, so they were removed.\n")
            
            rolling_window_df = DATASET.diff(periods = WINDOW, axis = AXIS)
            print(f"Calculated discrete differences ({WINDOW} periods). Returning the differentiated dataframe.\n")
        
        else:
            print("Please, select a valid rolling window function: \'mean\', \'std\', \'sum\', or \'difference\'.")
            return "error"
        
        # drop missing values generated:
        if (drop_missing_values):
            # Run of it is True. Do not reset index for mode 'difference'.
            # In this mode, the index was set as the date.
            rolling_window_df.dropna(axis = 0, how = 'any', inplace = True)
            
            if (window_statistics != 'difference'):
                rolling_window_df.reset_index(drop = True, inplace = True)
        
        print("Check the rolling dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(rolling_window_df)
                
        except: # regular mode
            print(rolling_window_df)
        
        print('\n')
        print("ATTENTION: depending on the window size, the windowed dataset may be considerable smaller than the original dataframe, with several missing values indicated by NA.\n")
        print("For understanding it, consider a dataframe containing daily new cases of an illness, where we want to obtain the 7-day rolling average.")
        print("Here, we will obtain 6 rows containing only missing values. The reason is that it is not possible to calculate the 7-periods average for the first 6 rows.")
        print("In the first row, we have only 1 data; in the second row, we have only two, the day and the day before; ..., and so on.")
        print("We can only calculate a 7-period average from the 7th day, when we have that day and the 6 days before it.")
        print("Once it is not possible to obtain the rolling statistic for some rows, missing values are generated.")
        print("So, even if the rolling statistic was calculated for only 2 consecutive periods, there would be a row with missing values, since it is not possible to calculate the window statistic for a single entry.\n")
        print(f"Naturally, this examples suppose that the user set how_to_close_window = {'right'}, when the first point in the window is excluded from calculations.")
        print(f"If how_to_close_window = {'left'}, then the last point in the window would be excluded from calculations, so the missing values would appear at the end of the dataset.")
        print("Even though it is not so intuitive, in this case we would take an entry and the next ones for calculating the statistic. For instance, the 7-day rolling average would be calculated as the average between a day and the next 6 days.")
        print(f"Finally, if how_to_close_window = {'both'}, we would have a centralized window, where the some of the values come from the times before; and some come from the times after.")
        print("In this last case, the 7-day rolling average would be calculated as the average between a day; the 3 days before; and the 3 next days.")
        print("So, missing values would appear in both the beginning and the end of the dataframe.\n")
        
        print("For this function, the default is how_to_close_window = {'right'}, i.e., statistics are calculated from the row and the values before it.\n")
        
        return rolling_window_df


    def seasonal_decomposition (df, response_column_to_analyze, column_with_timestamps = None, decomposition_mode = "additive", maximum_number_of_cycles_or_periods_to_test = 100, x_axis_rotation = 70, y_axis_rotation = 0, grid = True, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        import statsmodels as sm
        from statsmodels.tsa.seasonal import DecomposeResult
        from statsmodels.tsa.seasonal import seasonal_decompose
        
        #Check seasonal_decompose and DecomposeResult documentations:
        # https://www.statsmodels.org/dev/generated/statsmodels.tsa.seasonal.seasonal_decompose.html
        # https://www.statsmodels.org/dev/generated/statsmodels.tsa.seasonal.DecomposeResult.html#statsmodels.tsa.seasonal.DecomposeResult
        # seasonal_decompose results in an object from class DecomposeResult.
        # Check the documentation of the .plot method for DecomposeResult objects:
        # https://www.statsmodels.org/dev/generated/statsmodels.tsa.seasonal.DecomposeResult.plot.html#statsmodels.tsa.seasonal.DecomposeResult.plot
        
        #number_of_periods_to_forecast = 7
        # integer value representing the total of periods to forecast. The periods will be in the
        # unit (dimension) of the original dataset. If 1 period = 1 day, 7 periods will represent
        # seven days.
        
        # df: the whole dataframe to be processed.
        
        # response_column_to_analyze: string (inside quotes), 
        # containing the name of the column that will be analyzed.
        # e.g. response_column_to_analyze = "column1" will analyze the column named as 'column1'.
        # WARNING: This must be the response variable
        
        # column_with_timestamps: string (inside quotes), 
        # containing the name of the column containing timestamps.
        # Keep it as None if you want to set the index as the time.
        # e.g. response_column_to_analyze = "column1" will analyze the column named as 'column1'.
        
        # MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST = integer (minimum value is 2) representing
        # the total of cycles or periods that may be present on time series. The function will loop through
        # 2 to MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST to find the number that minimizes the sum of
        # modules (absolute values) of the residues.
        # e.g. MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST = 4 will test 2, 3 and 4 cycles on the time series.

        # decomposition_mode = "additive" - manipulate the parameter 'model' from seasonal_decompose.
        # model = decomposition_mode
        # Alternatively, set decomposition_mode = "multiplicative" for decomposing as a multiplicative time series.
        
        ## 'additive' model: An additive model suggests that the components are added together as: 
        ## y(t) = Level + Trend + Seasonality + Noise
        ## An additive model is linear where changes over time are consistently made by the same amount. A linear trend is 
        ## a straight line. A linear seasonality has the same frequency (width of cycles) and amplitude (height of cycles).
        
        ## 'multiplicative' model: A multiplicative model suggests that the components are multiplied together as:
        ## y(t) = Level * Trend * Seasonality * Noise
        ## A multiplicative model is nonlinear, such as quadratic or exponential. Changes increase or decrease over time.
        ## A nonlinear trend is a curved line. A non-linear seasonality has an increasing or decreasing frequency 
        ## and/or amplitude over time.
        # https://machinelearningmastery.com/decompose-time-series-data-trend-seasonality/#:~:text=The%20statsmodels%20library%20provides%20an%20implementation%20of%20the,careful%20to%20be%20critical%20when%20interpreting%20the%20result.
        
        
        # Set a local copy of the dataframe to manipulate?
        DATASET = df.copy(deep = True)
        
        #Check if there is a column with the timestamps:
        if not (column_with_timestamps is None):
            
            DATASET = DATASET.sort_values(by = column_with_timestamps)
            
            x = DATASET[column_with_timestamps].copy()
            
            # try to convert it to datetime:
            try:
                x = x.astype(np.datetime64)
            
            except:
                pass
            
            # Set it as index to include it in the seasonal decomposition model:
            DATASET = DATASET.set_index(column_with_timestamps)
        
        else:
            #the index will be used to plot the charts:
            x = DATASET.index
            
        # Extract the time series from the dataframe:
        Y = DATASET[response_column_to_analyze]
        
        # Set the parameters for modelling:
        MODEL = decomposition_mode
        MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST = maximum_number_of_cycles_or_periods_to_test
        
        # Check if the arguments are valid:
        if MODEL not in ["additive", "multiplicative"]:
            # set model as 'additive'
            MODEL = "additive"
        
        print(f"Testing {MODEL} model for seasonal decomposition.\n")
        
        try:
            MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST = int(MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST)
            # If it is lower than 2, make it equals to 2:
            if (MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST < 2):
                MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST = 2
            
            print(f"Testing the presence of until {MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST} periods or cycles in the time series.\n")
        
        except:
            print("Input a valid MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST. It must be an integer higher or equal to 2, representing the maximum possible number of cycles or periods present in time series.\n")
            return "error"
        
        # Now, let's loop through the possible number of cycles and periods:
        # Start a dictionary to store the number of cycles and the correspondent sum of 
        # absolute values of the residues:
        residues_dict = {}
        
        for TOTAL_OF_CYCLES in range (2, (MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST + 1)):
            
            # TOTAL_OF_CYCLES is an integer looping from TOTAL_OF_CYCLES = 2 to
            # TOTAL_OF_CYCLES = (MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST + 1) - 1 = (MAXIMUM_NUMBER_OF_CYCLES_OR_PERIODS_TO_TEST)
            
            try:
            
                # Start an instance (object) from class DecomposeResult
                # Set this object as the resultant from seasonal_decompose
                decompose_res_obj = seasonal_decompose(Y, model = MODEL, period = TOTAL_OF_CYCLES, two_sided = True)
                # decompose_res_obj is an instance (object) from class DecomposeResult

                # Get the array of the residues. Convert it to NumPy array to guarantee the vectorial operations:
                residues_array = np.array(decompose_res_obj.resid)
                # Convert the values in the array to the absolute values:
                residues_array = abs(residues_array)
                # Get the sum of the absolute residues:
                sum_of_abs_residues = np.sum(residues_array)

                # Store it in the dictionary (value correspondent to the key TOTAL_OF_CYCLES):
                residues_dict[TOTAL_OF_CYCLES] = sum_of_abs_residues
            
            except:
                # There are no sufficient measurements to test this total of cycles
                pass
        
        # get the list of dictionary's values:
        dict_vals = list(residues_dict.values())
        # Get the minimum value on the list:
        minimum_residue = min(dict_vals)
        # Get the index of minimum_residue on the list:
        minimum_residue_index = dict_vals.index(minimum_residue)
        
        # Now, retrieve OPTIMAL_TOTAL_CYCLES. It will be the value with index minimum_residue_index
        # in the list of keys:
        list_of_keys = list(residues_dict.keys())
        OPTIMAL_TOTAL_CYCLES = list_of_keys[minimum_residue_index]
        
        print(f"Number of total cycles or periods in time series: {OPTIMAL_TOTAL_CYCLES}.\n")
        
        # Start an instance (object) from class DecomposeResult
        # Set this object as the resultant from seasonal_decompose
        decompose_res_obj = seasonal_decompose(Y, model = MODEL, period = OPTIMAL_TOTAL_CYCLES, two_sided = True)
        # decompose_res_obj is an instance (object) from class DecomposeResult
        
        # Create a dictionary with the resultants from the seasonal decompose:
        # These resultants are obtained as attributes of the decompose_res_obj
        
        number_of_observations_used = decompose_res_obj.nobs
        print(f"Seasonal decomposition concluded using {number_of_observations_used} observations.\n")
        
        decompose_dict = {
            
            'timestamp': x,
            "observed_data": decompose_res_obj.observed,
            "seasonal_component": decompose_res_obj.seasonal,
            "trend_component": decompose_res_obj.trend,
            "residuals": decompose_res_obj.resid
        }
        
        # Convert it into a returned dataframe:
        seasonal_decompose_df = pd.DataFrame(data = decompose_dict)
        
        print("Check the first 10 rows of the seasonal decompose dataframe obtained:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(seasonal_decompose_df.head(10))
                
        except: # regular mode
            print(seasonal_decompose_df.head(10))
        
        print("\n") # line break
        print(f"Check the time series decomposition graphics for the {MODEL} model:\n")
        
        # Plot parameters:
        x = decompose_dict['timestamp']
        y1 = decompose_dict['observed_data']
        lab1 = "observed_data"
        y2 = decompose_dict['seasonal_component']
        lab2 = 'seasonal_component'
        y3 = decompose_dict['trend_component']
        lab3 = 'trend_component'
        y4 = decompose_dict['residuals']
        lab4 = 'residuals'
        
        plot_title = "seasonal_decomposition_for_" + response_column_to_analyze
        
        # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
        # so that the bars do not completely block other views.
        OPACITY = 0.95
        
        # Now, let's obtain the graphic:
        # Create the figure:
        fig, ax = plt.subplots(4, 1, sharex = True, figsize = (12, 8)) 
        # sharex = share axis X
        # number of subplots equals to the total of series to plot (in this case, 4)
        
        ax[0].plot(x, y1, linestyle = '-', marker = '', color = 'darkblue', alpha = OPACITY, label = lab1)
        # Set title only for this subplot:
        ax[0].set_title(plot_title)
        ax[0].grid(grid)
        ax[0].legend(loc = 'upper right')
        # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
        # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
        # https://www.statology.org/matplotlib-legend-position/
        
        ax[1].plot(x, y2, linestyle = '-', marker = '', color = 'crimson', alpha = OPACITY, label = lab2)
        # Add the y-title only for this subplot:
        ax[1].set_ylabel(response_column_to_analyze)
        ax[1].grid(grid)
        ax[1].legend(loc = 'upper right')
        
        ax[2].plot(x, y3, linestyle = '-', marker = '', color = 'darkgreen', alpha = OPACITY, label = lab3)
        ax[2].grid(grid)
        ax[2].legend(loc = 'upper right')
        
        ax[3].plot(x, y4, linestyle = '', marker = 'o', color = 'red', alpha = OPACITY, label = lab4)
        # Add an horizontal line in y = zero:
        ax[3].axhline(0, color = 'black', linestyle = 'dashed', alpha = OPACITY)
        # Set the x label only for this subplot
        ax[3].set_xlabel('timestamp')
        ax[3].grid(grid)
        ax[3].legend(loc = 'upper right')
        
        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 0 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)
        
        if (export_png == True):
            # Image will be exported
            import os

            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""

            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "seasonal_decomposition"

            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330

            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)

            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

            #fig.tight_layout()

        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        
        #Finally, return the full dataframe:
        print("The full dataframe obtained from the decomposition was returned as seasonal_decompose_df.")
        
        return seasonal_decompose_df


    def COLUMN_GENERAL_STATISTICS (df, column_to_analyze):
        
        import numpy as np
        import pandas as pd
        
        #df: dataframe to be analyzed.
        
        #column_to_analyze: name of the new column. e.g. column_to_analyze = 'col1'
        # will analyze column named as 'col1'
        analyzed_series = df[column_to_analyze].copy()
        #Drop missing values. Dropping them with describe method is deprecated and may raise errors.
        analyzed_series = analyzed_series.dropna()
        
        general_stats = analyzed_series.describe()
        print(f"General descriptive statistics from variable {column_to_analyze}, ignoring missing values:\n") 
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(general_stats)
                
        except: # regular mode
            print(general_stats)
        
        interpretation_df = pd.DataFrame(
        
            data = {
                'statistic': ["count", "mean", "std", "min", "25% = 0.25",
                            "50% = 0.50", "75% = 0.75", "max"],
                'interpretation': ["total of values evaluated (number of entries)",
                                "mean value of the series", "standard deviation of the series",
                                "minimum value observed",
                                "1st-quartile: 25% of data <= this value",
                                "2nd-quartile: 50% of data <= this value", 
                                "3rd-quartile: 75% of data <= this value",
                                "maximum value observed"
                                ]}
        )
        interpretation_df.set_index('statistic', inplace = True)
        
        print("\n") #line break
        print("Interpretation (missing values ignored):")
        
        try:
            display(interpretation_df)        
        except:
            print(interpretation_df)
        
        print("\n")
        print("ATTENTION: This function shows the general statistics only for numerical variables.")
        print("The results were returned as the dataframe general_stats.\n")
        
        # Return only the dataframe
        return general_stats


    def GET_QUANTILES_FOR_COLUMN (df, column_to_analyze):
        
        import numpy as np
        import pandas as pd
        
        #df: dataframe to be analyzed.
        
        #column_to_analyze: name of the new column. e.g. column_to_analyze = 'col1'
        # will analyze column named as 'col1'
        analyzed_series = df[column_to_analyze].copy()
        #Drop missing values. Dropping them with describe method is deprecated and may raise errors.
        analyzed_series = analyzed_series.dropna()
        
        list_of_quantiles = []
        list_of_pcts = []
        list_of_values = []
        list_of_interpretation = []
        
        #First element: minimum
        list_of_quantiles.append(0.0)
        list_of_pcts.append(0)
        list_of_values.append(analyzed_series.min())
        list_of_interpretation.append(f"minimum {column_to_analyze}")
        
        i = 5
        #Start from the 5% quantile
        while (i < 100):
            
            list_of_quantiles.append(i/100)
            list_of_pcts.append(i)
            list_of_values.append(analyzed_series.quantile(i/100))
            list_of_interpretation.append(f"{i}% of data <= this value")
            
            i = i + 5
        
        # Last element: maximum value
        list_of_quantiles.append(1.0)
        list_of_pcts.append(100)
        list_of_values.append(analyzed_series.max())
        list_of_interpretation.append(f"maximum {column_to_analyze}")
        
        # Summarize the lists as a dataframe:
        
        quantiles_summ_df = pd.DataFrame(data = {"quantile": list_of_quantiles, 
                                                "%": list_of_pcts,
                                                column_to_analyze: list_of_values,
                                                "interpretation": list_of_interpretation})
        quantiles_summ_df.set_index(['quantile', "%", column_to_analyze], inplace = True)
        
        print("Quantiles returned as dataframe quantiles_summ_df. Check it below:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(quantiles_summ_df)
                
        except: # regular mode
            print(quantiles_summ_df)
        
        return quantiles_summ_df


    def GET_P_PERCENT_QUANTILE_LIM_FOR_COLUMN (df, column_to_analyze, p_percent = 100):
        
        import numpy as np
        import pandas as pd
        
        #df: dataframe to be analyzed.
        
        #column_to_analyze: name of the new column. e.g. column_to_analyze = 'col1'
        # will analyze column named as 'col1'
        
        # p_percent: float value from 0 to 100 representing the percent of the quantile
        # if p_percent = 31.2, then 31.2% of the data will fall below the returned value
        # if p_percent = 75, then 75% of the data will fall below the returned value
        # if p_percent = 0, the minimum value is returned.
        # if p_percent = 100, the maximum value is returned.
        
        analyzed_series = df[column_to_analyze].copy()
        #Drop missing values. Dropping them with describe method is deprecated and may raise errors.
        analyzed_series = analyzed_series.dropna()
        
        #convert the quantile to fraction
        quantile_fraction = p_percent/100.0 #.0 to guarantee a float result
        
        if (quantile_fraction < 0):
            print("Invalid percent value - it cannot be lower than zero.")
            return "error"
        
        elif (quantile_fraction == 0):
            #get the minimum value
            quantile_lim = analyzed_series.min()
            print(f"Minimum value of {column_to_analyze} =")
            print("%.4f" %(quantile_lim))
        
        elif (quantile_fraction == 1):
            #get the maximum value
            quantile_lim = analyzed_series.max()
            print(f"Maximum value of {column_to_analyze} =")
            print("%.4f" %(quantile_lim))
            
        else:
            #get the quantile
            quantile_lim = analyzed_series.quantile(quantile_fraction)
            print(f"{quantile_fraction}-quantile: {p_percent}% of data <=")
            print("%.4f" %(quantile_lim))
        
        return quantile_lim


    def LABEL_DATAFRAME_SUBSETS (df, list_of_labels = [{'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}, {'filter': None, 'value_to_apply': None, 'new_column_name': None}]):
        
        import numpy as np
        import pandas as pd
        
        print("Attention: this function selects subsets from the dataframe and label them, allowing the seggregation of the data.\n")
        print("If you want to filter the dataframe to eliminate non-selected rows, use the function APPLY_ROW_FILTERS_LIST\n")
        
        ## This function selects subsets of the dataframe by applying a list
        ## of row filters, and then it labels each one of the filtered subsets.
        
        # df: dataframe to be analyzed.
        
        # list_of_labels = [{'filter': None, 'value_to_apply': None, 'new_column_name': None}]
        # list_of_labels is as a list of dictionaries. It must be declared as a list, in brackets,
        # even if there is a single dictionary.
        # Use always the same keys: 'filter' for one of the boolean filters that will be applied; 
        # 'value_to_apply' the value that will be used for labelling that specific subset selected
        # from the boolean filter (it may be either a string or a value); and
        # 'new_column_name': a string or variable to be the name of the new column created. If None,
        # a standard name will be applied.
        
        ## ATTENTION: If you want the labels to be applied to a same column, declare the exact same value
        # for the key 'new_column_name'. Also, if you want the value to be applied to an existing column,
        # declare the existing column's name in 'new_column_name'.
        
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to plot more series.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'filter': filter, 'value_to_apply': value, 'new_column_name': name}, where 
        # filter, value, and name represent the boolean filter, the value for labelling, and the new
        # column name (you can pass 'value_to_apply': None, 'new_column_name': None, but if 
        # 'filter' is None, the new dictionary will be ignored).
        
        ## define the filters and only them define the filters list
        # EXAMPLES OF BOOLEAN FILTERS TO COMPOSE THE LIST
        # boolean_filter1 = ((None) & (None)) 
        # (condition1 AND (&) condition2)
        # boolean_filter2 = ((None) | (None)) 
        # condition1 OR (|) condition2
        
        # boolean filters result into boolean values True or False.

        ## Examples of filters:
        ## filter1 = (condition 1) & (condition 2)
        ## filter1 = (df['column1'] > = 0) & (df['column2']) < 0)
        ## filter2 = (condition)
        ## filter2 = (df['column3'] <= 2.5)
        ## filter3 = (df['column4'] > 10.7)
        ## filter3 = (condition 1) | (condition 2)
        ## filter3 = (df['column5'] != 'string1') | (df['column5'] == 'string2')

        ## comparative operators: > (higher); >= (higher or equal); < (lower); 
        ## <= (lower or equal); == (equal); != (different)

        ## concatenation operators: & (and): the filter is True only if the 
        ## two conditions concatenated through & are True
        ## | (or): the filter is True if at least one of the two conditions concatenated
        ## through | are True.
        ## ~ (not): inverts the boolean, i.e., True becomes False, and False becomes True. 

        ## separate conditions with parentheses. Use parentheses to define a order
        ## of definition of the conditions:
        ## filter = ((condition1) & (condition2)) | (condition3)
        ## Here, firstly ((condition1) & (condition2)) = subfilter is evaluated. 
        ## Then, the resultant (subfilter) | (condition3) is evaluated.

        ## Pandas .isin method: you can also use this method to filter rows belonging to
        ## a given subset (the row that is in the subset is selected). The syntax is:
        ## is_black_or_brown = dogs["color"].isin(["Black", "Brown"])
        ## or: filter = (dataframe_column_series).isin([value1, value2, ...])
        # The negative of this condition may be acessed with ~ operator:
        ##  filter = ~(dataframe_column_series).isin([value1, value2, ...])
        ## Also, you may use isna() method as filter for missing values:
        ## filter = (dataframe_column_series).isna()
        ## or, for not missing: ~(dataframe_column_series).isna()
        
        # Warning: the sequence of filtering dictionaries must be correspondent to the sequence of labels. 
        # Rows selected from the first filter are labelled with the first item from the labels
        # list; rows selected by the 2nd filter are labelled with the 2nd element, and so on.
        
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]
        
        # List of new columns:
        new_cols_list = []
        
        # Loop through all dictionaries in list_of_labels:
        for dictionary in list_of_labels:
            
            # Check if the 'filter' key is not None:
            boolean_filter = dictionary['filter']
            
            if (boolean_filter is not None):
                
                if (dictionary['value_to_apply'] is None):
                    label = np.nan
                
                else:
                    label = dictionary['value_to_apply']
                
                if (dictionary['new_column_name'] is None):
                    
                    new_column = "labelled_column_" + str(list_of_labels.index(dictionary))
                
                else:
                    new_column = str(dictionary['new_column_name'])
                
                #Apply the filter to select a group of rows, and apply the correspondent label
                # to the selected rows

                # syntax: dataset.loc[dataset['column_filtered'] <= 0.87, 'labelled_column'] = 1
                # which is equivalent to dataset.loc[(filter), 'labelled_column'] = label
                DATASET.loc[(boolean_filter), new_column] = label
                
                # If the new_column is not in the list of new_Columns, append it:
                if (new_column not in new_cols_list):
                    new_cols_list.append(new_column)
                
        # If new_column in numeric types, try to convert it to strings.
        # It is important for the box and violin plots:
        
        for new_column in new_cols_list:
            
            if (DATASET[new_column].dtypes in numeric_dtypes):
                try:
                    DATASET[new_column] = (DATASET[new_column]).astype(str)
                    
                except: #simply pass
                    pass
        
        # Reset index:
        DATASET = DATASET.reset_index(drop = True)
        
        print("Successfully labelled the dataframe. Check its 10 first rows:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(DATASET.head(10))
                
        except: # regular mode
            print(DATASET.head(10))
        
        return DATASET


    def anova_box_violin_plot (plot_type = 'box', confidence_level_pct = 95, orientation = 'vertical', reference_value = None, data_in_same_column = False, df = None, column_with_labels_or_groups = None, variable_to_analyze = None, list_of_dictionaries_with_series_to_analyze = [{'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}, {'values_to_analyze': None, 'label': None}], obtain_boxplot_with_filled_boxes = True, obtain_boxplot_with_notched_boxes = False, x_axis_rotation = 0, y_axis_rotation = 0, grid = True, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
            
        print ("If an error message is shown, update statsmodels. Declare and run a cell as:")
        print ("!pip install statsmodels --upgrade\n")
        import random
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from statsmodels.stats.oneway import anova_oneway
            
        # plot_type = 'box' to plot a boxplot.
        # plot_type = 'violin' to plot a violinplot.
        # If plot_type = None, or plot_type = 'only_anova', only the anova analysis will be performed.
        # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.boxplot.html
        # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.violinplot.html
            
        # confidence_level_pct = 95 = 95% confidence
        # It is the percent of confidence for the analysis.
        # Set confidence_level_pct = 90 to get 0.90 = 90% confidence in the analysis.
        # Notice that, when less trust is needed, we can reduce confidence_level_pct
        # to get less restrictive results.
        alpha_anova = 1 - (confidence_level_pct/100)
        
        # orientation = 'vertical' for vertical plots; 
        # or orientation = 'horizontal', for horizontal plots.
        # Manipulate the parameter vert (boolean).
            
        # reference_value: keep it as None or add a float value.
        # This reference value will be shown as a red constant line to be compared
        # with the plots. e.g. reference_value = 1.0 will plot a red line passing through
        # VARIABLE_TO_ANALYZE = 1.0
        
        # data_in_same_column: set as True if all the values to plot are in a same column.
        # If data_in_same_column = True, you must specify the dataframe containing the data as df;
        # the column containing the label or group indication as column_with_labels_or_groups; and the column 
        # containing the variable to analyze as variable_to_analyze.
        # If column_with_labels_or_groups is None, the ANOVA analysis will not be performed and 
        # the plot will be obtained for the whole series.
        
        # df is an object, so do not declare it in quotes. The other three arguments (columns' names) 
        # are strings, so declare in quotes. 
        
        # Example: suppose you have a dataframe saved as dataset, and two groups A and B to compare. 
        # All the results for both groups are in a column named 'results'. If the result is for
        # an entry from group A, then a column named 'group' has the value 'A'. If it is for group B,
        # column 'group' shows the value 'B'. In this example:
        # data_in_same_column = True,
        # df = dataset,
        # column_with_labels_or_groups = 'group',
        # variable_to_analyze = 'results'.
        # If you want to declare a list of dictionaries, keep data_in_same_column = False and keep
        # df = None (the other arguments may be set as None, but it is not mandatory: 
        # column_with_labels_or_groups = None, variable_to_analyze = None).
        

        # Parameter to input when DATA_IN_SAME_COLUMN = False:
        # list_of_dictionaries_with_series_to_analyze {'values_to_analyze': None, 'label': None}:
        # if data is already converted to series, lists or arrays, provide them as a list of dictionaries. 
        # It must be declared as a list, in brackets, even if there is a single dictionary.
        # Use always the same keys: 'values_to_analyze' for values that will be analyzed, and 'label' for
        # the label or group correspondent to the series (may be a number or a string). 
        # If you do not want to declare a series, simply keep as None, but do not remove or rename a 
        # key (ALWAYS USE THE KEYS SHOWN AS MODEL).
        # If you want, you can remove elements (dictionaries) from the list to declare fewer elements;
        # and you can also add more elements (dictionaries) to the lists, if you need to plot more series.
        # Simply put a comma after the last element from the list and declare a new dictionary, keeping the
        # same keys: {'values_to_analyze': y, 'label': 'series_y'}, where y represents the values
        # to analyze, and 'series_y' is the label 
        # (you can pass 'label': None, but if values_to_analyze' is None, the new 
        # dictionary will be ignored).
        
        # Examples:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'values_to_analyze': y, 'label': 0}]
        # will plot a single variable. In turns:
        # list_of_dictionaries_with_series_to_analyze = 
        # [{'values_to_analyze': DATASET['Y1'], 'label': 'label1'}, 
        # {'values_to_analyze': DATASET['Y2'], 'label': 'label2'}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}, {'x': None, 'y': None, 'lab': None}]
        # will plot two series, Y1 and Y2.
        # Notice that all dictionaries where 'values_to_analyze' is None are automatically ignored.
        # If None is provided to 'label', an automatic label will be generated.
        
        ## Parameters with effect only for boxplots (plot_type = 'box'):
        # obtain_boxplot_with_filled_boxes = True
        # Manipulate parameter patch_artist (boolean, default: False)
        # If obtain_boxplot_with_filled_boxes = True, the boxes are created filled. 
        # If obtain_boxplot_with_filled_boxes = False, only the contour of the boxes are shown
        # (obtain void white boxes).
        
        # obtain_boxplot_with_notched_boxes = False
        # Manipulate parameter notch (boolean, default: False) from the boxplot object
        # Whether to draw a notched boxplot (obtain_boxplot_with_notched_boxes = True), 
        # or a rectangular boxplot (obtain_boxplot_with_notched_boxes = False). 
        # The notches represent the confidence interval (CI) around the median. 
        
        # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.boxplot.html
        
        
        if (data_in_same_column == True):
            
            print("Data to be plotted in a same column.\n")
            
            if (df is None):
                
                print("Please, input a valid dataframe as df.\n")
                list_of_dictionaries_with_series_to_analyze = []
                # The code will check the size of this list on the next block.
                # If it is zero, code is simply interrupted.
                # Instead of returning an error, we use this code structure that can be applied
                # on other graphic functions that do not return a summary (and so we should not
                # return a value like 'error' to interrupt the function).
            
            elif (variable_to_analyze is None):
                
                print("Please, input a valid column name as variable_to_analyze.\n")
                list_of_dictionaries_with_series_to_analyze = []
            
            else:
                
                # set a local copy of the dataframe:
                DATASET = df.copy(deep = True)
                
                if (column_with_labels_or_groups is None):
                
                    print("Using the whole series (column).\n")
                    column_with_labels_or_groups = 'whole_series_' + variable_to_analyze
                    DATASET[column_with_labels_or_groups] = column_with_labels_or_groups
                
                # sort DATASET; by column_with_labels_or_groups; and by variable_to_analyze,
                # all in Ascending order
                # Since we sort by label (group), it is easier to separate the groups.
                DATASET = DATASET.sort_values(by = [column_with_labels_or_groups, variable_to_analyze], ascending = [True, True])
                
                # Drop rows containing missing values in column_with_labels_or_groups or variable_to_analyze:
                DATASET = DATASET.dropna(how = 'any', subset = [column_with_labels_or_groups, variable_to_analyze])
                
                # Reset indices:
                DATASET = DATASET.reset_index(drop = True)
                
                # Get a series of unique values of the labels, and save it as a list using the
                # list attribute:
                unique_labels = list(DATASET[column_with_labels_or_groups].unique())
                print(f"{len(unique_labels)} different labels detected: {unique_labels}.\n")
                
                # Start a list to store the dictionaries containing the keys:
                # 'values_to_analyze' and 'label'
                list_of_dictionaries_with_series_to_analyze = []
                
                # Loop through each possible label:
                for lab in unique_labels:
                    # loop through each element from the list unique_labels, referred as lab
                    
                    # Set a filter for the dataset, to select only rows correspondent to that
                    # label:
                    boolean_filter = (DATASET[column_with_labels_or_groups] == lab)
                    
                    # Create a copy of the dataset, with entries selected by that filter:
                    ds_copy = (DATASET[boolean_filter]).copy(deep = True)
                    # Sort again by X and Y, to guarantee the results are in order:
                    ds_copy = ds_copy.sort_values(by = [column_with_labels_or_groups, variable_to_analyze], ascending = [True, True])
                    # Restart the index of the copy:
                    ds_copy = ds_copy.reset_index(drop = True)
                    
                    # Re-extract the analyzed series and convert it to NumPy array: 
                    # (these arrays will be important later in the function):
                    y = np.array(ds_copy[variable_to_analyze])
                
                    # Then, create the dictionary:
                    dict_of_values = {'values_to_analyze': y, 'label': lab}
                    
                    # Now, append dict_of_values to list_of_dictionaries_with_series_to_analyze:
                    list_of_dictionaries_with_series_to_analyze.append(dict_of_values)
                    
                # Now, we have a list of dictionaries with the same format of the input list.
                
        else:
            
            # The user input a list_of_dictionaries_with_series_to_analyze
            # Create a support list:
            support_list = []
            
            # Loop through each element on the list list_of_dictionaries_with_series_to_analyze:
            
            for i in range (0, len(list_of_dictionaries_with_series_to_analyze)):
                # from i = 0 to i = len(list_of_dictionaries_with_series_to_analyze) - 1, index of the
                # last element from the list
                
                # pick the i-th dictionary from the list:
                dictionary = list_of_dictionaries_with_series_to_analyze[i]
                
                # access 'values_to_analyze' and 'label' keys from the dictionary:
                values_to_analyze = dictionary['values_to_analyze']
                lab = dictionary['label']
                # Remember that all this variables are series from a dataframe, so we can apply
                # the astype function:
                # https://www.askpython.com/python/built-in-methods/python-astype?msclkid=8f3de8afd0d411ec86a9c1a1e290f37c
                
                # check if at least values_to_analyze is not None:
                if (values_to_analyze is not None):
                
                    # Possibly, series is a not ordered Pandas series, and may contain missing values.
                    # Let's order it and clean it, if it is a Pandas object:
                    try:
                        # Create a local copy to manipulate
                        series = values_to_analyze.copy(deep = True)
                        series = series.sort_values(ascending = True)
                        series = series.dropna()
                    
                    except:
                        # It is not a Pandas object. Simply copy to use the same variable name:
                        series = values_to_analyze
                    
                    # Re-extract Y series and convert it to NumPy array 
                    # (these arrays will be important later in the function):
                    y = np.array(series)
                    
                    # check if label is None:
                    if (lab is None):
                        # input a default label.
                        # Use the str attribute to convert the integer to string, allowing it
                        # to be concatenated
                        lab = "series_" + str(i)
                        
                    # Then, create the dictionary:
                    dict_of_values = {'values_to_analyze': y, 'label': lab}
                    
                    # Now, append dict_of_values to support list:
                    support_list.append(dict_of_values)
                
            # Now, support_list contains only the dictionaries with valid entries, as well
            # as labels for each collection of data. The values are independent from their origin,
            # and now they are ordered and in the same format of the data extracted directly from
            # the dataframe.
            # So, make the list_of_dictionaries_with_series_to_analyze the support_list itself:
            list_of_dictionaries_with_series_to_analyze = support_list
            print(f"{len(list_of_dictionaries_with_series_to_analyze)} valid series input.\n")

            
        # Now that both methods of input resulted in the same format of list, we can process both
        # with the same code.
        
        # Each dictionary in list_of_dictionaries_with_series_to_analyze represents a series to
        # plot. So, the total of series to plot is:
        total_of_series = len(list_of_dictionaries_with_series_to_analyze)
        
        if (total_of_series <= 0):
            
            print("No valid series to plot. Please, provide valid arguments.\n")
        
        else:
            
            # For performing the ANOVA, we must store all series into an array of arrays,
            # and must create a correspondent array of labels: each label will correspond to one
            # series (one element from the array of arrays).
            # Create the lists (d represents a dictionary):
            list_of_arrays = [list(d['values_to_analyze']) for d in list_of_dictionaries_with_series_to_analyze]
            list_of_labels = [d['label'] for d in list_of_dictionaries_with_series_to_analyze]
            
            # Store the total of valid series:
            total_series = len(list_of_labels)
            # Let's pad the series by eliminating random elements until all series have the
            # same lengths:
            
            # If there are 2 or more series to analyze, perform the ANOVA:
            if (total_series <= 1):
                print("There is a single valid series, so the analysis of variance (ANOVA) will not be performed.\n")
                # An empty ANOVA dictionary will be returned:
                anova_summary_dict = {}
            
            else:
                print(f"Analysis of variance (ANOVA) for the {total_series} detected series:\n")
                
        
                # Let's get the minimum length for the lists above:
                min_length = min(len(list_of_values) for list_of_values in list_of_arrays)

                # start a supporting list:
                support_list = []
                
                for list_of_values in list_of_arrays:
                    
                    while (min_length < len(list_of_values)):
                        
                        # Let's randomly select len_difference indices to delete:
                        list_indices = range(0, len(list_of_values))
                        
                        # Randomly select len_difference elements:
                        # Function random.choices(input_sequence, k = number_of_samples):
                        # randomly select k elements from the sequence input_sequence. This function is
                        # newer than random.sample
                        
                        # Select an index to delete
                        deleted_index = random.choices(list_indices, k = 1)
                        # Now, delete this element:
                        # The pop function is safer and returns the deleted elements, but cannot do
                        # it in a single call.
                        # Even though del can process a list, we cannot pass a list comprehension like
                        # del [list_of_values[i] for i in deleted_indices]
                        # deleted_index is a 1-element list. Pick only the first
                        deleted_element = list_of_values.pop(deleted_index[0])
                    
                    # Now, convert it to NumPy array:
                    list_of_values = np.array(list_of_values)
                    # Append it to the support list:
                    support_list.append(list_of_values)
                
                # Make the support_list the list_of_arrays itself:
                list_of_arrays = support_list
                # Now all lists are padded and are represented by padded arrays.
                
                #Now, we can pass the arrays as arguments for the one-way Anova:
                anova_one_way_summary = anova_oneway(list_of_arrays)
                # https://www.statsmodels.org/stable/generated/statsmodels.stats.oneway.anova_oneway.html

                # The information is stored in a tuple (f_statistic, p-value)
                # f_statistic: Test statistic for k-sample mean comparison which is approximately 
                # F-distributed.
                # p-value: If use_var="bf", then the p-value is based on corrected degrees of freedom following Mehrotra 1997.
                f_statistic = anova_one_way_summary[0]
                p_value = anova_one_way_summary[1]
                
                print(f"Total of samples in each group used for ANOVA (after padding): {min_length}\n")
                print(f"Probability that the means of the groups are the same = {100*p_value:.2f}% (p-value = {p_value:e})\n")
                print(f"Calculated F-statistic for the variances = {f_statistic:e}\n")
                # :e indicates the scientific notation

                if (p_value <= alpha_anova):
                    print(f"For a confidence level of {confidence_level_pct:.2f}%, we can reject the null hypothesis.")
                    print(f"The means are different for a {confidence_level_pct:.2f}% confidence level.")

                else:
                    print(f"For a confidence level of {confidence_level_pct}%, we can accept the null hypothesis.")
                    print(f"The means are equal for a {confidence_level_pct}% confidence level.")

                anova_summary_dict = {'groups_length':min_length, 'F_statistic': f_statistic, 'p_value': p_value}

            if ((plot_type is not None) & (plot_type != 'only_anova')):

                # Now, let's obtain the plots:
                # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
                # so that the bars do not completely block other views.
                OPACITY = 0.95

                # Manipulate the parameter vert (boolean, default: True)
                # If True, draws vertical boxes. If False, draw horizontal boxes.
                if (orientation == 'horizontal'):
                    VERT = False

                    if (horizontal_axis_title is None):
                        # Set horizontal axis title
                        horizontal_axis_title = "analyzed_series"

                    if (vertical_axis_title is None):
                        # Set vertical axis title
                        vertical_axis_title = "group_or_label"

                else:
                    VERT = True

                    if (horizontal_axis_title is None):
                        # Set horizontal axis title
                        horizontal_axis_title = "group_or_label"

                    if (vertical_axis_title is None):
                        # Set vertical axis title
                        vertical_axis_title = "analyzed_series"

                if (obtain_boxplot_with_filled_boxes is None):
                    obtain_boxplot_with_filled_boxes = True

                if (obtain_boxplot_with_notched_boxes is None):
                    obtain_boxplot_with_notched_boxes = False

                if (plot_title is None):
                    # Set graphic title
                    plot_title = f"{plot_type}_plot"

                # Now, let's obtain the boxplot
                fig, ax = plt.subplots(figsize = (12, 8))

                if (plot_type == 'box'):
                    # rectangular box plot
                    # The arrays of each group are the elements of the list humongous_list
                    plot_returned_dict = ax.boxplot(list_of_arrays, labels = list_of_labels, notch = obtain_boxplot_with_notched_boxes, vert = VERT, patch_artist = obtain_boxplot_with_filled_boxes)

                    # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.boxplot.html

                    # plot_returned_dict: A dictionary mapping each component of the boxplot to 
                    # a list of the Line2D instances created. That dictionary has the following keys 
                    # (assuming vertical boxplots):
                    # boxes: the main body of the boxplot showing the quartiles and the median's 
                    # confidence intervals if enabled.
                    # medians: horizontal lines at the median of each box.
                    # whiskers: the vertical lines extending to the most extreme, non-outlier data 
                    # points.
                    # caps: the horizontal lines at the ends of the whiskers.
                    # fliers: points representing data that extend beyond the whiskers (fliers).
                    # means: points or lines representing the means.

                    # boxplot contains only lists (iterable collections) of objects
                    # (matplotlib.lines.Line2D objects):
                    # Each object on the list corresponds to one series being plot. For setting
                    # different colors, the parameters must be different for each object from one list.

                    for whisker in plot_returned_dict['whiskers']:
                        whisker.set_color('crimson')
                        whisker.set_alpha(OPACITY)

                    for cap in plot_returned_dict['caps']:
                        cap.set_color('crimson')
                        cap.set_alpha(OPACITY)

                    for flier in plot_returned_dict['fliers']:
                        flier.set_color('crimson')
                        flier.set_alpha(OPACITY)

                    for mean in plot_returned_dict['means']:
                        mean.set_color('crimson')
                        mean.set_alpha(OPACITY)

                    for median in plot_returned_dict['medians']:
                        median.set_color('crimson')
                        median.set_alpha(OPACITY)

                    # Set the boxes configuration for each case, where it should be filled or not:
                    if (obtain_boxplot_with_filled_boxes):
                        for box in plot_returned_dict['boxes']:
                            box.set_color('lightgrey')
                            box.set_alpha(0.5)
                    else: # only the contour of the box
                        for box in plot_returned_dict['boxes']:
                            box.set_color('black')
                            box.set_alpha(1.0)

                if (plot_type == 'violin'):
                    # violin plot, estimate of the statistical distribution
                    # The arrays of each group are the elements of the list humongous_list
                    plot_returned_dict = ax.violinplot(list_of_arrays, vert = VERT, showmeans = True, showextrema = True, showmedians = True)

                    # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.violinplot.html

                    # plot_returned_dict: A dictionary mapping each component of the violinplot to a list of 
                    # the corresponding collection instances created. The dictionary has the following keys:
                    # bodies: A list of the PolyCollection instances containing the filled area of each 
                    # violin.
                    # cmeans: A LineCollection instance that marks the mean values of each of the violin's 
                    # distribution.
                    # cmins: A LineCollection instance that marks the bottom of each violin's distribution.
                    # cmaxes: A LineCollection instance that marks the top of each violin's distribution.
                    # cbars: A LineCollection instance that marks the centers of each violin's distribution.
                    # cmedians: A LineCollection instance that marks the median values of each of the violin's distribution.
                    # cquantiles: A LineCollection instance created to identify the quantile values of each 
                    # of the violin's distribution.

                    # Here, the labels must be defined manually:
                    if (orientation == 'horizontal'):
                        ax.set_yticks(np.arange(1, len(list_of_labels) + 1), labels = list_of_labels)

                    else:
                        ax.set_xticks(np.arange(1, len(list_of_labels) + 1), labels = list_of_labels)
                        # np.arange(1, len(list_of_labels) + 1) is the same list of numbers that the violinplot
                        # associates to each sequence.

                    # https://matplotlib.org/stable/gallery/statistics/customized_violin.html#sphx-glr-gallery-statistics-customized-violin-py

                    # Violinplot contains line objects and lists (iterable collections) of objects
                    # matplotlib.collections.LineCollection objects: not iterable
                    # These are specific from violin plots:
                    plot_returned_dict['cmeans'].set_facecolor('crimson')
                    plot_returned_dict['cmeans'].set_edgecolor('crimson')
                    plot_returned_dict['cmeans'].set_alpha(OPACITY)

                    plot_returned_dict['cmedians'].set_facecolor('crimson')
                    plot_returned_dict['cmedians'].set_edgecolor('crimson')
                    plot_returned_dict['cmedians'].set_alpha(OPACITY)

                    plot_returned_dict['cmaxes'].set_facecolor('crimson')
                    plot_returned_dict['cmaxes'].set_edgecolor('crimson')
                    plot_returned_dict['cmaxes'].set_alpha(OPACITY)

                    plot_returned_dict['cmins'].set_facecolor('crimson')
                    plot_returned_dict['cmins'].set_edgecolor('crimson')
                    plot_returned_dict['cmins'].set_alpha(OPACITY)

                    plot_returned_dict['cbars'].set_facecolor('crimson')
                    plot_returned_dict['cbars'].set_edgecolor('crimson')
                    plot_returned_dict['cbars'].set_alpha(OPACITY)

                    # 'bodies': list of matplotlib.collections.PolyCollection objects (iterable)
                    # Each object on the list corresponds to one series being plot. For setting
                    # different colors, the parameters must be different for each object from one list.
                    for body in plot_returned_dict['bodies']:
                        body.set_facecolor('lightgrey')
                        body.set_edgecolor('black')
                        body.set_alpha(0.5)

                ax.set_title(plot_title)
                ax.set_xlabel(horizontal_axis_title)
                ax.set_ylabel(vertical_axis_title)
                ax.grid(grid)

                if (orientation == 'vertical'):
                    # generate vertically-oriented plot

                    if not (reference_value is None):
                        # Add an horizontal reference_line to compare against the boxes:
                        # If the boxplot was horizontally-oriented, this line should be vertical instead.
                        ax.axhline(reference_value, color = 'black', linestyle = 'dashed', label = 'reference', alpha = OPACITY)
                        # axhline generates an horizontal (h) line on ax

                else:

                    if not (reference_value is None):
                        # Add an horizontal reference_line to compare against the boxes:
                        # If the boxplot was horizontally-oriented, this line should be vertical instead.
                        ax.axvline(reference_value, color = 'black', linestyle = 'dashed', label = 'reference', alpha = OPACITY)
                        # axvline generates a vertical (v) line on ax

                #ROTATE X AXIS IN XX DEGREES
                plt.xticks(rotation = x_axis_rotation)
                # XX = 70 DEGREES x_axis (Default)
                #ROTATE Y AXIS IN XX DEGREES:
                plt.yticks(rotation = y_axis_rotation)
                # XX = 0 DEGREES y_axis (Default)

                if (export_png == True):
                    # Image will be exported
                    import os

                    #check if the user defined a directory path. If not, set as the default root path:
                    if (directory_to_save is None):
                        #set as the default
                        directory_to_save = ""

                    #check if the user defined a file name. If not, set as the default name for this
                    # function.
                    if (file_name is None):
                        #set as the default
                        file_name = f"{plot_type}_plot"

                    #check if the user defined an image resolution. If not, set as the default 110 dpi
                    # resolution.
                    if (png_resolution_dpi is None):
                        #set as 330 dpi
                        png_resolution_dpi = 330

                    #Get the new_file_path
                    new_file_path = os.path.join(directory_to_save, file_name)

                    #Export the file to this new path:
                    # The extension will be automatically added by the savefig method:
                    plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
                    #quality could be set from 1 to 100, where 100 is the best quality
                    #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
                    #transparent = True or False
                    # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
                    print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

                #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
                #plt.figure(figsize = (12, 8))
                #fig.tight_layout()

                ## Show an image read from an image file:
                ## import matplotlib.image as pltimg
                ## img=pltimg.imread('mydecisiontree.png')
                ## imgplot = plt.imshow(img)
                ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
                ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
                ##  '03_05_END.ipynb'

                plt.show()
            
            else:
                print(f"Plot type set as {plot_type}. So, no plot was obtained.\n")
            
            print("\n") #line break
            print("Successfully returned 2 dictionaries: anova_summary_dict (dictionary storing ANOVA F-test and p-value); and plot_returned_dict (dictionary mapping each component of the plot).\n")
            
            if (plot_type == 'box'):
            
                print("Boxplot interpretation:\n")
                print("Boxplot presents the following key visual components:\n")
                
                print("Main box")
                print("The main box represents the Interquartile Range (IQR).")
                print("It represents the data that is from quartile Q1 to quartile Q3.\n")
                
                print("Q1 = 1st quartile of the dataset")
                print("25% of values lie below this level (i.e., it is the 0.25-quantile or percentile).\n")
                
                print("Q2 = 2nd quartile of the dataset")
                print("50% of values lie above and below this level (i.e., it is the 0.50-quantile or percentile).\n")
                
                print("Q3 = 3rd quartile of the dataset")
                print("75% of values lie below and 25% lie above this level (i.e., it is the 0.75-quantile or percentile).\n")
                
                print("Median line")
                print("Boxplot main box (the IQR) is divided by an horizontal line if it is vertically-oriented; or by a vertical line if it is horizontally-oriented.")
                print("This line represents the median: it is the midpoint of the dataset.\n")
                
                print("Limit lines")      
                print("There are lines extending beyond the main boxes limits.")
                print("These lines end in horizontal limits, if the boxplot is vertically oriented; or in vertical limits, for an horizontal plot.\n")
                
                print("Minimum limit")
                print("The minimum limit of the boxplot is defined as:")
                print("Q1 - (1.5) x (IQR width) = Q1 - 1.5*(Q3-Q1)\n")
                
                print("Maximum limit")
                print("The maximum limit of the boxplot is defined as:")
                print("Q3 + (1.5) x (IQR width) = Q3 + 1.5*(Q3-Q1)\n")
                
                print("Outliers")
                print("Finally, there are isolated points (circles) on the plot.")
                print("These points lie below the minimum bar, or above the maximum bar line.")
                print("They are defined as outliers.\n")
                
                print("Application of the plot")        
                print("Like violin plots, box plots are used to represent comparison of a variable distribution (or sample distribution) across different 'categories'.") 
                print("Examples: temperature distribution compared between day and night; or distribution of car prices compared across different car makers.\n")
                # https://nickmccullum.com/python-visualization/boxplot/
            
            elif (plot_type == 'violin'):
                
                print("Violin plot interpretation:\n")
                
                print("A violin plot is similar to a box plot, with the addition of a rotated kernel density plot on each side of the violin.\n")
                print("So, this plot also shows the probability density of the data at different values, usually smoothed by a kernel density estimator.\n")
                print("Typically a violin plot will include all the data that is in a box plot.\n")
                print("It includes a filled area extending to represent the entire data range; with lines at the mean, the median, the minimum, and the maximum.\n")
                
                print("So, let's firstly check the box plot components.")
                print("Notice that the interquartile range represented by the main box will not be present.")
                print("The violin plot replaces this box region by the density distribution itself.\n")
                
                print("Main box")
                print("The main box represents the Interquartile Range (IQR).")
                print("It represents the data that is from quartile Q1 to quartile Q3.\n")
                
                print("Q1 = 1st quartile of the dataset")
                print("25% of values lie below this level (i.e., it is the 0.25-quantile or percentile).\n")
                
                print("Q2 = 2nd quartile of the dataset")
                print("50% of values lie above and below this level (i.e., it is the 0.50-quantile or percentile).\n")
                
                print("Q3 = 3rd quartile of the dataset")
                print("75% of values lie below and 25% lie above this level (i.e., it is the 0.75-quantile or percentile).\n")
                
                print("Median line")
                print("Boxplot main box (the IQR) is divided by an horizontal line if it is vertically-oriented; or by a vertical line if it is horizontally-oriented.")
                print("This line represents the median: it is the midpoint of the dataset.\n")
                
                print("Limit lines")      
                print("There are lines extending beyond the main boxes limits.")
                print("These lines end in horizontal limits, if the boxplot is vertically oriented; or in vertical limits, for an horizontal plot.\n")
                
                print("Minimum limit")
                print("The minimum limit of the boxplot is defined as:")
                print("Q1 - (1.5) x (IQR width) = Q1 - 1.5*(Q3-Q1)\n")
                
                print("Maximum limit")
                print("The maximum limit of the boxplot is defined as:")
                print("Q3 + (1.5) x (IQR width) = Q3 + 1.5*(Q3-Q1)\n")
                
                print("Outliers")
                print("Finally, there are isolated points (circles) on the box plot.")
                print("These points lie below the minimum bar, or above the maximum bar line.")
                print("They are defined as outliers.\n")
                
                print("ATTENTION:")
                print("Since the probability density is shown, these isolated outlier points are not represented in the violin plot.\n")
                
                print("Presence on multiple peaks in the violin plot")
                print("A violin plot is more informative than a plain box plot.")
                print("While a box plot only shows summary statistics such as mean/median and interquartile ranges, the violin plot shows the full distribution of the data.")
                print("This difference is particularly useful when the data distribution is multimodal (more than one peak).")
                print("In this case, a violin plot shows the presence of different peaks, their positions and relative amplitudes.\n")
                
                print("Application of the plot")
                print("Like box plots, violin plots are used to represent comparison of a variable distribution (or sample distribution) across different 'categories'.")
                print("Examples: temperature distribution compared between day and night; or distribution of car prices compared across different car makers.\n")
                
                print("Presence of multiple layers")
                print("A violin plot can have multiple layers. For instance, the outer shape represents all possible results.")
                print("The next layer inside might represent the values that occur 95% of the time.")
                print("The next layer (if it exists) inside might represent the values that occur 50% of the time.\n")
                
                print("Alternative to this plot")
                print("Although more informative than box plots, they are less popular.")
                print("Because of their unpopularity, they may be harder to understand for readers not familiar with them.")
                print("In this case, a more accessible alternative is to plot a series of stacked histograms or kernel density distributions (KDE plots).\n")
                # https://en.wikipedia.org/wiki/Violin_plot#:~:text=A%20violin%20plot%20is%20a%20method%20of%20plotting,values%2C%20usually%20smoothed%20by%20a%20kernel%20density%20estimator.
                
            return anova_summary_dict, plot_returned_dict


    def statistical_process_control_chart (df, column_with_variable_to_be_analyzed, timestamp_tag_column = None, column_with_labels_or_subgroups = None, column_with_event_frame_indication = None, specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': None}, reference_value = None, use_spc_chart_assistant = False, chart_to_use = 'std_error', consider_skewed_dist_when_estimating_with_std = False, rare_event_indication = None, rare_event_timedelta_unit = 'day', x_axis_rotation = 70, y_axis_rotation = 0, grid = True, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from scipy import stats
        
        # matplotlib.colors documentation:
        # https://matplotlib.org/3.5.0/api/colors_api.html?msclkid=94286fa9d12f11ec94660321f39bf47f
        
        # Matplotlib list of colors:
        # https://matplotlib.org/stable/gallery/color/named_colors.html?msclkid=0bb86abbd12e11ecbeb0a2439e5b0d23
        # Matplotlib colors tutorial:
        # https://matplotlib.org/stable/tutorials/colors/colors.html
        # Matplotlib example of Python code using matplotlib.colors:
        # https://matplotlib.org/stable/_downloads/0843ee646a32fc214e9f09328c0cd008/colors.py
        # Same example as Jupyter Notebook:
        # https://matplotlib.org/stable/_downloads/2a7b13c059456984288f5b84b4b73f45/colors.ipynb
        
        # df: dataframe to be analyzed.
        
        # TIMESTAMP_TAG_COLUMN: name (header) of the column containing the timestamps or the numeric scale
        # used to represent time (column with floats or integers). The column name may be a string or a number.
        # e.g. TIMESTAMP_TAG_COLUMN = 'date' will use the values from column 'date'
        # Keep TIMESTAMP_TAG_COLUMN = None if the dataframe do not contain timestamps, so the index will be
        # used.

        # COLUMN_WITH_VARIABLE_TO_BE_ANALYZED: name (header) of the column containing the variable
        # which stability will be analyzed by the control chart. The column name may be a string or a number.
        # Example: COLUMN_WITH_VARIABLE_TO_BE_ANALYZED = 'analyzed_column' will analyze a column named
        # "analyzed_column", whereas COLUMN_WITH_VARIABLE_TO_BE_ANALYZED = 'col1' will evaluate column 'col1'.

        # COLUMN_WITH_LABELS_OR_SUBGROUPS: name (header) of the column containing the variable
        # indicating the subgroups or label indication which will be used for grouping the samples. 
        # Example: Suppose you want to analyze the means for 4 different subgroups: 1, 2, 3, 4. For each subgroup,
        # 4 or 5 samples of data (the values in COLUMN_WITH_VARIABLE_TO_BE_ANALYZED) are collected, and you
        # want the average and standard deviation within the subgroups. Them, you create a column named
        # 'label' and store the values: 1 for samples correspondent to subgroup 1; 2 for samples from
        # subgroup 2,... etc. In this case, COLUMN_WITH_LABELS_OR_SUBGROUPS = 'label'
        # Notice that the samples do not need to be collected in order. The function will automatically separate
        # the entries according to the subgroups. So, the entries in the dataset may be in an arbitrary order
        # like: 1, 1, 2, 1, 4, 3, etc.
        # The values in the COLUMN_WITH_LABELS_OR_SUBGROUPS may be strings (text) or numeric values (like
        # integers), but different values will be interpreted as different subgroups.
        # As an example of text, you could have a column named 'col1' with group identifications as: 
        # 'A', 'B', 'C', 'D', and COLUMN_WITH_LABELS_OR_SUBGROUPS = 'col1'.
        # Notice the difference between COLUMN_WITH_LABELS_OR_SUBGROUPS and COLUMN_WITH_VARIABLE_TO_BE_ANALYZED:
        # COLUMN_WITH_VARIABLE_TO_BE_ANALYZED accepts only numeric values, so the binary variables must be
        # converted to integers 0 and 1 before the analysis. The COLUMN_WITH_LABELS_OR_SUBGROUPS, in turns,
        # accept both numeric and text (string) values.
        
        # COLUMN_WITH_EVENT_FRAME_INDICATION: name (header) of the column containing the variable
        # indicating the stages, time windows, or event frames. The central line and the limits of natural
        # variation will be independently calculated for each event frame. The indication of an event frame
        # may be textual (string) or numeric. 
        # For example: suppose you have a column named 'event_frames'. For half of the entries, event_frame = 
        # 'A'; and for the second half, event_frame = 'B'. If COLUMN_WITH_EVENT_FRAME_INDICATION = 'event_frame',
        # the dataframe will be divided into two subsets: 'A', and 'B'. For each subset, the central lines
        # and the limits of natural variation will be calculated independently. So, you can check if there is
        # modification of the average value and of the dispersion when the time window is modified. It could
        # reflect, for example, the use of different operational parameters on each event frame.
        # Other possibilities of event indications: 0, 1, 2, 3, ... (sequence of integers); 'A', 'B', 'C', etc;
        # 'stage1', 'stage2', ..., 'treatment1', 'treatment2',....; 'frame0', 'frame1', 'frame2', etc.
        # ATTENTION: Do not identify different frames with the same value. For example, if
        # COLUMN_WITH_EVENT_FRAME_INDICATION has missing values for the first values, then a sequence of rows
        # is identified as number 0; followed by a sequence of missing values. In this case, the two windows
        # with missing values would be merged as a single window, and the mean and variation would be
        # obtained for this merged subset. Then, always specify different windows with different values.
        # Other example: COLUMN_WITH_EVENT_FRAME_INDICATION = 'col1' will search for event frame indications
        # in column 'col1'.
        
        # specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': None}
        # If there are specification limits, input them in this dictionary. Do not modify the keys,
        # simply substitute None by the lower and/or the upper specification.
        # e.g. Suppose you have a tank that cannot have more than 10 L. So:
        # specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': 10}, there is only
        # an upper specification equals to 10 (do not add units);
        # Suppose a temperature cannot be lower than 10 ºC, but there is no upper specification. So,
        # specification_limits = {'lower_spec_lim': 10, 'upper_spec_lim': None}. Finally, suppose
        # a liquid which pH must be between 6.8 and 7.2:
        # specification_limits = {'lower_spec_lim': 6.8, 'upper_spec_lim': 7.2}
        
        # REFERENCE_VALUE: keep it as None or add a float value.
        # This reference value will be shown as a red constant line to be compared
        # with the plots. e.g. REFERENCE_VALUE = 1.0 will plot a line passing through
        # VARIABLE_TO_ANALYZE = 1.0
        
        # USE_SPC_CHART_ASSISTANT = False. Set USE_SPC_CHART_ASSISTANT = True to open the 
        # visual flow chart assistant that will help you select the appropriate parameters; 
        # as well as passing the data in the correct format. If the assistant is open, many of the 
        # arguments of the function will be filled when using it.

        # chart_to_use = '3s_as_natural_variation', 'std_error', 'i_mr', 'xbar_s', 'np', 'p', 
        # 'u', 'c', 'g', 't'
        # The type of chart that will be obtained, as well as the methodology used for estimating the
        # natural variation of the process. Notice that it may be strongly dependent on the statistical
        # distribution. So, if you are not sure about the distribution, or simply want to apply a more
        # general (less restrictive) methodology, set:

        # CHART_TO_USE = '3s_as_natural_variation' to estimate the natural variation as 3 times the
        # standard deviation (s); or
        # CHART_TO_USE = 'std_error' for estimating it as 3 times the standard error, where 
        # standard error = s/(n**0.5) = s/sqrt(n), n = number of samples (that may be the number of 
        # individual data samples collected, or the number of subgroups or labels); sqrt is the square root.
        # https://en.wikipedia.org/wiki/Standard_error
        # CHART_TO_USE = '3s_as_natural_variation' and CHART_TO_USE = 'std_error' are the only ones available
        # for both individual and grouped data.
        ## ATTENTION: Do not group the variables before using the function. It will perform the automatic
        # grouping in accordance to the values specified as COLUMN_WITH_LABELS_OR_SUBGROUPS.
        # Other values allowed for CHART_TO_USE:
        # CHART_TO_USE = 'i_mr', for individual values of a numeric (continuous) variable which follows the
        # NORMAL distribution.
        # CHART_TO_USE = 'xbar_s', for grouped values (by mean) of a numeric variable, where the mean values
        # of labels or subgroups follow a NORMAL distribution.
        # CHART_TO_USE = 'np', for grouped binary variables (allowed values: 0 or 1). This is the
        # control chart for proportion of defectives. - Original data must follow the BINOMIAL distribution.
        # CHART_TO_USE = 'p', for grouped binary variables (allowed values: 0 or 1). This is the
        # control chart for count of defectives. - Original data must follow the BINOMIAL distribution.
        # Attention: an error will be raised if CHART_TO_USE = 'np' or 'p' and the variable was not converted
        # to a numeric binary, with values 0 or 1. This function do not perform the automatic ordinal or
        # One-Hot encoding of the categorical features.
        # CHART_TO_USE = 'u', for counts of occurrences per unit. - Original data must follow the POISSON
        # distribution (special case of the gamma distribution).
        # CHART_TO_USE = 'c', for average occurrence per unit. - Original data must follow the POISSON
        # distribution (special case of the gamma distribution).
        # CHARTS FOR ANALYZING RARE EVENTS
        # CHART_TO_USE = 'g', for analyzing count of events between successive rare events occurrences 
        # (data follow the GEOMETRIC distribution).
        # CHART_TO_USE = 't', for analyzing time interval between successive rare events occurrences.
        
        # consider_skewed_dist_when_estimating_with_std.
        # Whether the distribution of data to be analyzed present high skewness or kurtosis.
        # If CONSIDER_SKEWED_DISTRIBUTION_WHEN_ESTIMATING_STD = False, the central lines will be estimated
        # as the mean values of the analyzed variable. 
        # If CONSIDER_SKEWED_DISTRIBUTION_WHEN_ESTIMATING_STD = True, the central lines will be estimated 
        # as the median of the analyzed variable, which is a better alternative for skewed data such as the 
        # ones that follow geometric or lognormal distributions (median = mean × 0.693).
        # Notice that this argument has effect only when CHART_TO_USE = '3s_as_natural_variation' or 
        # CHART_TO_USE = 'std_error'.

        # RARE_EVENT_INDICATION = None. String (in quotes), float or integer. If you want to analyze a
        # rare event through 'g' or 't' control charts, this parameter is obbligatory. Also, notice that:
        # COLUMN_WITH_VARIABLE_TO_BE_ANALYZED must be the column which contains an indication of the rare
        # event occurrence, and the RARE_EVENT_INDICATION is the value of the column COLUMN_WITH_VARIABLE_TO_BE_ANALYZED
        # when a rare event takes place.
        # For instance, suppose RARE_EVENT_INDICATION = 'shutdown'. It means that column COLUMN_WITH_VARIABLE_TO_BE_ANALYZED
        # has the value 'shutdown' when the rare event occurs, i.e., for timestamps when the system
        # stopped. Other possibilities are RARE_EVENT_INDICATION = 0, or RARE_EVENT_INDICATION = -1,
        # indicating that when COLUMN_WITH_VARIABLE_TO_BE_ANALYZED = 0 (or -1), we know that
        # a rare event occurred. The most important thing here is that the value given to the rare event
        # should be assigned ONLY to the rare events.

        # You do not need to assign values for the other timestamps when no rare event took place. But it is
        # important to keep all timestamps in the dataframe. That is because the rare events charts will
        # compare the rare event occurrence against all other events and timestamps.
        # If you are not analyzing rare events with 'g' or 't' charts, keep RARE_EVENT_INDICATION = None.
        
        # RARE_EVENT_TIMEDELTA_UNIT: 'day', 'second', 'nanosecond', 'minute', 'hour',
        # 'month', 'year' - This is the unit of time that will be used to plot the time interval
        # (timedelta) between each successive rare events. If None or invalid value used, timedelta
        # will be given in days.
        # Notice that this parameter is referrent only to the rare events analysis with 'g' or 't' charts.
        # Also, it is valid only when the timetag column effectively stores a timestamp. If the timestamp
        # column stores a float or an integer (numeric) value, then the final dataframe and plot will be
        # obtained in the same numeric scale of the original data, not in the unit indicated as
        # RARE_EVENT_TIMEDELTA_UNIT.
        
        # List the possible numeric data types for a Pandas dataframe column:
        numeric_dtypes = [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]

        ## CONTROL CHARTS CALCULATION
        
        # References: 
        # Douglas C. Montgomery. Introduction to Statistical Quality Control. 6th Edition. 
        # John Wiley and Sons, 2009.
        # Jacob Anhoej. Control Charts with qicharts for R. 2021-04-20. In: https://cran.r-project.org/web/packages/qicharts/vignettes/controlcharts.html

        # CENTER LINE = m (mean)
        # s = std deviation
        # General equation of the control limits:
        # UCL = upper control limit = m + L*s
        # LCL = lower control limit = m - L*s
        # where L is a measurement of distance from central line.
        
        # for a subgroup of data collected for a continuous variable X:
        # x_bar_1 = mean value for subgroup 1,..., x_bar_n = mean for subgroup n.
        # x_bar_bar = (x_bar_1 + ... + x_bar_n)/n
        
        # On the other hand, the range R between two collected values is defined as: R = x_max - x_min
        # If R1, ... Rm are the ranges for m subgroups (which may be of size 2),
        # we have R_bar = (R1 + R2 + ... + Rm)/m
        
        # Analogously, if s_i is the standard deviation for a subgroup i and there are m subgroups:
        # s_bar = (s_1 +... +s_m)/m
        
        ## INDIVIDUAL MEASUREMENTS (Montgomery, pg.259, section 6.4)
        
        # For individual measurements, we consider subgroups formed by two consecutive measurements, so
        # m = 2, and R = abs(x_i - x_(i-1)), where abs function calculates the absolute value of the
        # difference between two successive subgroups.
        # UCL =  x_bar_bar + 3*(1/d2)*R_bar
        # LCL =  x_bar_bar - 3*(1/d2)*R_bar
        # Center line = x_bar_bar
            
        # CONTROL CHARTS FOR SUBGROUPS 
                
        # X-bar-S (continuous variables) - (Montgomery, pg.253, section 6.3.1):
        # For subgroups with m elements:
        # UCL = x_bar_bar + A3*s_bar
        # Center line = x_bar_bar
        # LCL = x_bar_bar - A3*s_bar
        
        # CONTROL CHARTS FOR CATEGORICAL VARIABLES
        
        # P-chart - (Montgomery, pg.291, section 7.2.1):
        # The sample fraction nonconforming is defined as the ratio of the number of nonconforming units 
        # in the sample D to the sample size n:
        # p = D/n. For a subgroup i containing n elements: p_i = D_i/n, i = 1,... m subgroups
        # From the binomial distribution, the mean should be estimated as p, and the variance s² 
        # as p(1-p)/n. The average of p is:
        # p_bar = (p_1 + ... + p_m)/m
        
        # UCL = p_bar + 3((p_bar(1-p_bar)/n)**(0.5))
        # Center line = p_bar
        # LCL = p_bar - 3((p_bar(1-p_bar)/n)**(0.5)), whereas n is the size of a given subgroup
        # Therefore, the width of control limits here vary with the size of the subgroups (it is
        # a function of the size of each subgroup)
        
        # np-chart - (Montgomery, pg.300, section 7.2.1):
        # UCL = np + 3((np(1-p))**(0.5))
        # Center line = np_bar
        # LCL = np - 3((np(1-p))**(0.5))
        # where np is the total sum of nonconforming, considering all subgroups, and p is the
        # propoertion for each individual subgroup, as previously defined.
        # Then, again the control limits depend on the subgroup (label) size.
        
        # C-chart - (Montgomery, pg.309, section 7.3.1):
        # In Poisson distribution, mean and variance are both equal to c.
        # UCL = c_bar + 3(c_bar**(0.5))
        # Center line = c_bar
        # LCL = c_bar + 3(c_bar**(0.5))
        # where c_bar is the mean c among all subgroups (labels)
        
        # U-chart - (Montgomery, pg.314, section 7.3.1):
        # If we find x total nonconformities in a sample of n inspection units, then the average 
        # number of nonconformities per inspection unit is: u = x/n
        # UCL = u_bar + 3((u_bar/n)**(0.5))
        # Center line = u_bar
        # LCL = u_bar + 3((u_bar/n)**(0.5))
        # where u_bar is the mean u among all subgroups (labels), but n is the individual subgroup size,
        # making the chart limit not-constant depending on the subgroups' sizes

        # RARE EVENTS
        # ATTENTION: Due not group data in this case. Since events are rare, they are likely to be 
        # eliminated during aggregation.
        # Usually, similarly to what is observed for the log-normal, data that follow the geometric
        # distribution is highly skewed, so the mean is not a good estimator for the central line.
        # Usually, it is better to apply the median = 0.693 x mean.
        
        # G-Chart: counts of total occurences between successive rare events occurences.
        # https://www.spcforexcel.com/knowledge/attribute-control-charts/g-control-chart
        # (Montgomery, pg.317, section 7.3.1)
        
        # e.g. total of patients between patient with ulcer.
        # The probability model that they use for the geometric distribution is p(x) = p(1-p)**(x-a)
        # where a is the known minimum possible number of events.
        # The number of units between defectives is modelled by the geometric distribution.
        # So, the G-control chart plots counting of occurrence by number, time unit, or timestamp.
        # Geometric distribution is highly skewed, thus the median is a better representation of the 
        # process centre to be used with the runs analysis.

        # Let y = total count of events between successive occurences of the rare event.
        
        # g_bar = median(y) = 0.693 x mean(y)
        # One can estimate the control limits as g_bar + 3(((g_bar)*(g_bar+1))**(0.5)) and
        # g_bar - 3(((g_bar)*(g_bar+1))**(0.5)), and central line = g_bar.
        # A better approach takes into account the probabilities associated to the geometric (g)
        # distribution.
        
        # In the probability approach, we start by calculating the value of p, which is an event
        # probability:
        # p = (1/(g_bar + 1))*((N-1)/N), where N is the total of individual values of counting between
        # successive rare events. So, if we have 3 rare events, A, B, and C, we have two values of 
        # counting between rare events, from A to B, and from B to C. In this case, since we have two
        # values that follow the G distribution, N = 2.
        # Let alpha_UCL a constant dependent on the number of sigma limits for the control limits. For
        # the usual 3 sigma limits, the value of alpha_UCL = 0.00135. With this constant, we can
        # calculate the control limits and central line
        
        # UCL = ((ln(0.00135))/(ln(1 - p))) - 1
        # Center line = ((ln(0.5))/(ln(1 - p))) - 1
        # LCL = max(0, (((ln(1 - 0.00135))/(ln(1 - p))) - 1))
        # where max represent the maximum among the two values in parentheses; and ln is the natural
        # logarithm (base e), sometimes referred as log (inverse of the exponential): ln(exp(x)) = x
        # = log(exp(x)) = x
        
        # t-chart: timedelta between rare events.
        # (Montgomery, pg.324, section 7.3.5):
        # https://www.qimacros.com/control-chart-formulas/t-chart-formula/
        
        # But instead of displaying the number of cases between events (defectives) it displays 
        # the time between events.
        
        # Nelson (1994) has suggested solving this problem by transforming the exponential random
        # variable to a Weibull random variable such that the resulting Weibull distribution is well
        # approximated by the normal distribution. 
        # If y represents the original exponential random variable (timedelta between successive rare
        # event occurence), the appropriate transformation is:
        # x = y**(1/3.6)= y**(0.2777)
        # where x is treated as a normal distributed variable, and the control chart for x is the I-MR.
        # So: 1. transform y into x;
        # 2. Calculate the control limits for x as if it was a chart for individuals;
        # reconvert the values to the original scale doing: y = x**(3.6)
        
        # We calculate for the transformed values x the moving ranges R and the mean of the 
        # mean values calculated for the 2-elements consecutive subgroups x_bar_bar, in the exact same way
        # we would do for any I-MR chart
        # UCL_transf =  x_bar_bar + 3*(1/d2)*R_bar
        # LCL_transf =  x_bar_bar - 3*(1/d2)*R_bar
        # Center line_transf = x_bar_bar
        
        # These values are in the transformed scale. So, we must reconvert them to the original scale for
        # obtaining the control limits and central line:
        # UCL = (UCL_transf)**(3.6)
        # LCL = (LCL_transf)**(3.6)
        # Center line = (Center line_transf)**(3.6)
        
        # Notice that this procedure naturally corrects the deviations caused by the skewness of the 
        # distribution. Actually, log and exponential transforms tend to reduce the skewness and to 
        # normalize the data.
        
        
        if (use_spc_chart_assistant == True):
            
            # Run if it is True. Requires TensorFlow to load. Load the extra library only
            # if necessary:
            # To show the Python class attributes, use the __dict__ method:
            # http://www.learningaboutelectronics.com/Articles/How-to-display-all-attributes-of-a-class-or-instance-of-a-class-in-Python.php#:~:text=So%20the%20__dict__%20method%20is%20a%20very%20useful,other%20data%20type%20such%20as%20a%20class%20itself.

            # instantiate the object
            assistant = spc_chart_assistant()
            # Download the images:
            assistant = assistant.download_assistant_imgs()

            # Run the assistant:
            while (assistant.keep_assistant_on == True):

                # Run the wrapped function until the user tells you to stop:
                # Notice that both variables are True for starting the first loop:
                assistant = assistant.open_chart_assistant_screen()

            # Delete the images
            assistant.delete_assistant_imgs()
            # Select the chart and the parameters:
            chart_to_use, column_with_labels_or_subgroups, consider_skewed_dist_when_estimating_with_std, column_with_variable_to_be_analyzed, timestamp_tag_column, column_with_event_frame_indication, rare_event_timedelta_unit, rare_event_indication = assistant.chart_selection()

        # Back to the main code, independently on the use of the assistant:    
        # set a local copy of the dataframe:
        DATASET = df.copy(deep = True)

        # Start a list unique_labels containing only element 0:
        unique_labels = [0]
        # If there is a column of labels or subgroups, this list will be updated. So we can control
        # the chart selection using the length of this list: if it is higher than 1, we have subgroups,
        # not individual values.

        if (timestamp_tag_column is None):

            # use the index itself:
            timestamp_tag_column = 'index'
            DATASET[timestamp_tag_column] = DATASET.index

        elif (DATASET[timestamp_tag_column].dtype not in numeric_dtypes):

            # The timestamp_tag_column was read as an object, indicating that it is probably a timestamp.
            # Try to convert it to datetime64:
            try:
                DATASET[timestamp_tag_column] = (DATASET[timestamp_tag_column]).astype(np.datetime64)
                print(f"Variable {timestamp_tag_column} successfully converted to datetime64[ns].\n")

            except:
                # Simply ignore it
                pass
            
            
        # Check if there are subgroups or if values are individuals. Also, check if there is no selected
        # chart:
        if ((column_with_labels_or_subgroups is None) & (chart_to_use is None)):
                
            print("There are only individual observations and no valid chart selected, so using standard error as natural variation (control limits).\n")
            # values are individual, so set it as so:
            chart_to_use = 'std_error'
            
        elif ((chart_to_use is None) | (chart_to_use not in ['std_error', '3s_as_natural_variation', 'i_mr', 'xbar_s', 'np', 'p', 'u', 'c', 'g', 't'])):
                
            print("No valid chart selected, so using standard error as natural variation (control limits).\n")
            # values are individual, so set it as so:
            chart_to_use = 'std_error'
            
        elif (chart_to_use in ['g', 't']):
                
            if (rare_event_timedelta_unit is None):
                print("No valid timedelta unit provided, so selecting \'day\' for analysis of rare events.\n")
                rare_event_timedelta_unit = 'day'
            elif (rare_event_timedelta_unit not in ['day', 'second', 'nanosecond', 'milisecond', 'hour', 'week', 'month', 'year']):
                print("No valid timedelta unit provided, so selecting \'day\' for analysis of rare events.\n")
                rare_event_timedelta_unit = 'day'
                
            if (rare_event_indication is None):
                print("No rare event indication provided, so changing the plot to \'std_error\'.\n")
                chart_to_use = 'std_error'
                    
        # Now, a valid chart_to_use was selected and is saved as chart_to_use.
        # We can sort the dataframe according to the columns present:
            
        if ((column_with_labels_or_subgroups is not None) & (column_with_event_frame_indication is not None)):
            # There are time windows to consider and labels.
            # Update the list of unique labels:
            unique_labels = list(DATASET[column_with_labels_or_subgroups].unique())
            
            # sort DATASET by timestamp_tag_column, column_with_labels_or_subgroups, 
            # column_with_event_frame_indication, and column_with_variable_to_be_analyzed,
            # all in Ascending order:
            DATASET = DATASET.sort_values(by = [timestamp_tag_column, column_with_labels_or_subgroups, column_with_event_frame_indication, column_with_variable_to_be_analyzed], ascending = [True, True, True, True])
            
        elif (column_with_event_frame_indication is not None):
            # We already tested the simultaneous presence of both. So, to reach this condition, 
            # there is no column_with_labels_or_subgroups, but there is column_with_event_frame_indication
                
            # sort DATASET by timestamp_tag_column, column_with_event_frame_indication, 
            # and column_with_variable_to_be_analyzed, all in Ascending order:
            DATASET = DATASET.sort_values(by = [timestamp_tag_column, column_with_event_frame_indication, column_with_variable_to_be_analyzed], ascending = [True, True, True])
            
        elif (column_with_labels_or_subgroups is not None):
            # We already tested the simultaneous presence of both. So, to reach this condition, 
            # there is no column_with_event_frame_indication, but there is column_with_labels_or_subgroups
            # Update the list of unique labels:
            unique_labels = list(DATASET[column_with_labels_or_subgroups].unique())
            
            # sort DATASET by timestamp_tag_column, column_with_labels_or_subgroups, 
            # and column_with_variable_to_be_analyzed, all in Ascending order:
            DATASET = DATASET.sort_values(by = [timestamp_tag_column, column_with_labels_or_subgroups, column_with_variable_to_be_analyzed], ascending = [True, True, True])
            
        else:
            # There is neither column_with_labels_or_subgroups, nor column_with_event_frame_indication
            # sort DATASET by timestamp_tag_column, and column_with_variable_to_be_analyzed, 
            # both in Ascending order:
            DATASET = DATASET.sort_values(by = [timestamp_tag_column, column_with_variable_to_be_analyzed], ascending = [True, True])
            
        # Finally, reset indices:
        DATASET = DATASET.reset_index(drop = True)
        
        # Start a list of dictionaries to store the dataframes and subdataframes that will be analyzed:
        list_of_dictionaries_with_dfs = []
        # By now, dictionaries will contain a key 'event_frame' storing an integer identifier for the
        # event frame, starting from 0 (the index of the event in the list of unique event frames), or
        # zero, for the cases where there is a single event; a key 'df' storing the dataframe object, 
        # a key 'mean', storing the mean value of column_with_variable_to_be_analyzed; keys 'std' and
        # 'var', storing the standard deviation and the variance for column_with_variable_to_be_analyzed;
        # and column 'count', storing the counting of rows. 
        # After obtaining the control limits, they will also get a key 'list_of_points_out_of_cl'. 
        # This key will store a list of dictionaries (nested JSON), where each dictionary 
        # will have two keys, 'x', and 'y', with the coordinates of the point outside control 
        # limits as the values.
        
        if ((column_with_event_frame_indication is not None) & (chart_to_use not in ['g', 't'])):
            
            # Get a series of unique values of the event frame indications, and save it as a list 
            # using the list attribute:
            unique_event_indication = list(DATASET[column_with_event_frame_indication].unique())
            
            print(f"{len(unique_event_indication)} different values of event frame indication detected: {unique_event_indication}.\n")
            
            if (len(unique_event_indication) > 0):
                
                # There are more than one event frame to consider. Let's loop through the list of
                # event frame indication, where each element is referred as 'event_frame':
                for event_frame in unique_event_indication:
                    
                    # Define a boolean filter to select only the rows correspondent to event_frame:
                    boolean_filter = (DATASET[column_with_event_frame_indication] == event_frame)
                    
                    # Create a copy of the dataset, with entries selected by that filter:
                    ds_copy = (DATASET[boolean_filter]).copy(deep = True)
                    # Sort again by timestamp_tag_column and column_with_variable_to_be_analyzed, 
                    # to guarantee the results are in order:
                    ds_copy = ds_copy.sort_values(by = [timestamp_tag_column, column_with_variable_to_be_analyzed], ascending = [True, True])
                    # Restart the index of the copy:
                    ds_copy = ds_copy.reset_index(drop = True)
                    
                    # Store ds_copy in a dictionary of values. Put the index of event_frame
                    # in the list unique_event_indication as the value in key 'event_frame', to numerically
                    # identify the dictionary according to the event frame.
                    # Also, store the 'mean', 'sum,' 'std', 'var', and 'count' aggregates for the column
                    # column_with_variable_to_be_analyzed from ds_copy:
                    
                    dict_of_values = {
                                        'event_frame': unique_event_indication.index(event_frame),
                                        'df': ds_copy, 
                                        'center': ds_copy[column_with_variable_to_be_analyzed].mean(),
                                        'sum': ds_copy[column_with_variable_to_be_analyzed].sum(),
                                        'std': ds_copy[column_with_variable_to_be_analyzed].std(),
                                        'var': ds_copy[column_with_variable_to_be_analyzed].var(),
                                        'count': ds_copy[column_with_variable_to_be_analyzed].count()
                                    }
                    
                    # append dict_of_values to the list:
                    list_of_dictionaries_with_dfs.append(dict_of_values)
                    
                    # Now, the loop will pick the next event frame.
            
            else:
                
                # There is actually a single time window. The only dataframe 
                # stored in the dictionary is DATASET itself, which is stored with 
                # the aggregate statistics calculated for the whole dataframe. Since
                # there is a single dataframe, the single value in 'event_frame' is 0.
                dict_of_values = {
                                    'event_frame': 0,
                                    'df': DATASET, 
                                    'center': DATASET[column_with_variable_to_be_analyzed].mean(),
                                    'sum': DATASET[column_with_variable_to_be_analyzed].sum(),
                                    'std': DATASET[column_with_variable_to_be_analyzed].std(),
                                    'var': DATASET[column_with_variable_to_be_analyzed].var(),
                                    'count': DATASET[column_with_variable_to_be_analyzed].count()
                                }
                
                # append dict_of_values to the list:
                list_of_dictionaries_with_dfs.append(dict_of_values)
        
        else:
            # The other case where there is a single time window or 'g' or 't' charts. 
            # So, the only dataframe stored in the dictionary is DATASET itself, which is stored with 
            # the aggregate statistics calculated for the whole dataframe. Since
            # there is a single dataframe, the single value in 'event_frame' is 0.
            dict_of_values = {
                                'event_frame': 0,
                                'df': DATASET, 
                                'center': DATASET[column_with_variable_to_be_analyzed].mean(),
                                'sum': DATASET[column_with_variable_to_be_analyzed].sum(),
                                'std': DATASET[column_with_variable_to_be_analyzed].std(),
                                'var': DATASET[column_with_variable_to_be_analyzed].var(),
                                'count': DATASET[column_with_variable_to_be_analyzed].count()
                            }
                
            # append dict_of_values to the list:
            list_of_dictionaries_with_dfs.append(dict_of_values)
        
        # Now, data is sorted, timestamps were converted to datetime objects, and values collected
        # for different timestamps were separated into dictionaries (elements) from the list
        # list_of_dictionaries_with_dfs. Each dictionary contains a key 'df' used to access the
        # dataframe, as well as keys storing the aggregate statistics: 'mean', 'std', 'var', and
        # 'count'.
        
        # Now, we can process the different control limits calculations.
        
        # Start a support list:
        support_list = []
        
        ## INDIVIDUAL VALUES
        # Use the unique_labels list to guarantee that there have to be more than 1 subgroup
        # to not treat data as individual values. If there is a single subgroup, unique_labels
        # will have a single element.
        if ((column_with_labels_or_subgroups is None) | (len(unique_labels) <= 1)):
            
            if (chart_to_use == 'i_mr'):
                
                print("WARNING: the I-MR control limits are based on the strong hypothesis that data follows a normal distribution. If it is not the case, do not use this chart.")
                print("If you are not confident about the statistical distribution, select chart_to_use = \'3s_as_natural_variation\' to use 3 times the standard deviation as estimator for the natural variation (the control limits); or chart_to_use = \'std_error\' to use 3 times the standard error as control limits.\n")
                
                for dictionary in list_of_dictionaries_with_dfs:
                    
                    # Create an instance (object) from spc_plot class:
                    plot = spc_plot (dictionary = dictionary, column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed, timestamp_tag_column = timestamp_tag_column, chart_to_use = chart_to_use, column_with_labels_or_subgroups = column_with_labels_or_subgroups, consider_skewed_dist_when_estimating_with_std = consider_skewed_dist_when_estimating_with_std, rare_event_indication = rare_event_indication, rare_event_timedelta_unit = rare_event_timedelta_unit)
                    # Apply the method
                    plot = plot.chart_i_mr()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                
                # Now that we finished looping through dictionaries, make list_of_dictionaries_with_dfs
                # the support_list itself:
                list_of_dictionaries_with_dfs = support_list
            
            elif (chart_to_use in ['g', 't']):
                
                print(f"Analyzing occurence of rare event {rare_event_indication} through chart {chart_to_use}.\n")
                
                for dictionary in list_of_dictionaries_with_dfs:
                    
                    # Create an instance (object) from spc_plot class:
                    plot = spc_plot (dictionary = dictionary, column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed, timestamp_tag_column = timestamp_tag_column, chart_to_use = chart_to_use, column_with_labels_or_subgroups = column_with_labels_or_subgroups, consider_skewed_dist_when_estimating_with_std = consider_skewed_dist_when_estimating_with_std, rare_event_indication = rare_event_indication, rare_event_timedelta_unit = rare_event_timedelta_unit)
                    # Apply the method
                    plot = plot.rare_events_chart()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                
                # Now that we finished looping through dictionaries, make list_of_dictionaries_with_dfs
                # the support_list itself:
                list_of_dictionaries_with_dfs = support_list
                
            elif (chart_to_use == '3s_as_natural_variation'):
                
                print("Using 3s (3 times the standard deviation) as estimator of the natural variation (control limits).\n")
                
                for dictionary in list_of_dictionaries_with_dfs:
                    
                    # Create an instance (object) from spc_plot class:
                    plot = spc_plot (dictionary = dictionary, column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed, timestamp_tag_column = timestamp_tag_column, chart_to_use = chart_to_use, column_with_labels_or_subgroups = column_with_labels_or_subgroups, consider_skewed_dist_when_estimating_with_std = consider_skewed_dist_when_estimating_with_std, rare_event_indication = rare_event_indication, rare_event_timedelta_unit = rare_event_timedelta_unit)
                    # Apply the method
                    plot = plot.chart_3s()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                
                # Now that we finished looping through dictionaries, make list_of_dictionaries_with_dfs
                # the support_list itself:
                list_of_dictionaries_with_dfs = support_list
            
            else:
                
                print("Using 3 times the standard error as estimator of the natural variation (control limits).\n")
                
                for dictionary in list_of_dictionaries_with_dfs:
                    
                    # Create an instance (object) from spc_plot class:
                    plot = spc_plot (dictionary = dictionary, column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed, timestamp_tag_column = timestamp_tag_column, chart_to_use = chart_to_use, column_with_labels_or_subgroups = column_with_labels_or_subgroups, consider_skewed_dist_when_estimating_with_std = consider_skewed_dist_when_estimating_with_std, rare_event_indication = rare_event_indication, rare_event_timedelta_unit = rare_event_timedelta_unit)
                    # Apply the method
                    plot = plot.chart_std_error()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                
                # Now that we finished looping through dictionaries, make list_of_dictionaries_with_dfs
                # the support_list itself:
                list_of_dictionaries_with_dfs = support_list
        
        ## DATA IN SUBGROUPS
        else:
            
            # Loop through each dataframe:
            for dictionary in list_of_dictionaries_with_dfs:
                
                # Create an instance (object) from spc_plot class:
                plot = spc_plot (dictionary = dictionary, column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed, timestamp_tag_column = timestamp_tag_column, chart_to_use = chart_to_use, column_with_labels_or_subgroups = column_with_labels_or_subgroups, consider_skewed_dist_when_estimating_with_std = consider_skewed_dist_when_estimating_with_std, rare_event_indication = rare_event_indication, rare_event_timedelta_unit = rare_event_timedelta_unit)
                # Apply the method
                plot = plot.create_grouped_df()
                    
                # Now, dataframe is ready for the calculation of control limits.
                # Let's select the appropriate chart:
            
                if (chart_to_use == '3s_as_natural_variation'):

                    plot = plot.chart_3s()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                
                elif (chart_to_use == 'std_error'):

                    plot = plot.chart_std_error()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                    
                elif (chart_to_use == 'xbar_s'):
                    
                    plot = plot.chart_x_bar_s()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                
                elif (chart_to_use == 'p'):
                    
                    plot = plot.chart_p()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)

                elif (chart_to_use == 'np'):
                
                    plot = plot.chart_np()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)

                elif (chart_to_use == 'c'):
                    
                    plot = plot.chart_c()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)
                    
                elif (chart_to_use == 'u'):
                    
                    plot = plot.chart_u()
                    # Append the updated dictionary to the support_list (access the dictionary attribute):
                    support_list.append(plot.dictionary)

                else:

                    print(f"Select a valid control chart: {['3s_as_natural_variation', 'std_error', 'i_mr', 'xbar_s', 'np', 'p', 'u', 'c', 'g', 't']}.\n")
                    return "error", "error" # Two, since two dataframes are returned
                
                # Go to the next element (dictionary) from the list list_of_dictionaries_with_dfs
            
            # Now that we finished looping through dictionaries, make list_of_dictionaries_with_dfs
            # the support_list itself:
            list_of_dictionaries_with_dfs = support_list
            
            # Now we finished looping, we can print the warnings
            if (chart_to_use == '3s_as_natural_variation'):
        
                print("Using 3s (3 times the standard deviation) as estimator of the natural variation (control limits). Remember that we are taking the standard deviation from the subgroup (label) aggregates.\n")
            
            elif (chart_to_use == 'xbar_s'):
                
                print("WARNING: the X-bar-S control limits are based on the strong hypothesis that the mean values from the subgroups follow a normal distribution. If it is not the case, do not use this chart.")
                print("If you are not confident about the statistical distribution, select chart_to_use = \'3s_as_natural_variation\' to use 3 times the standard deviation as estimator for the natural variation (the control limits).\n")
                print("Use this chart for analyzing mean values from multiple data collected together in groups (or specific labels), usually in close moments.\n")
            
            elif (chart_to_use == 'np'):
                print("WARNING: the U control limits are based on the strong hypothesis that the counting of values from the subgroups follow a Poisson distribution (Poisson is a case from the Gamma distribution). If it is not the case, do not use this chart.")
                
            
            elif (chart_to_use == 'p'):
                print("WARNING: the U control limits are based on the strong hypothesis that the counting of values from the subgroups follow a Poisson distribution (Poisson is a case from the Gamma distribution). If it is not the case, do not use this chart.")
                
            
            elif (chart_to_use == 'u'):
                
                print("WARNING: the U control limits are based on the strong hypothesis that the counting of values from the subgroups follow a Poisson distribution (Poisson is a case from the Gamma distribution). If it is not the case, do not use this chart.")
                print("If you are not confident about the statistical distribution, select chart_to_use = \'3s_as_natural_variation\' to use 3 times the standard deviation as estimator for the natural variation (the control limits).\n")
            
            else:
                # chart_to_use == 'c'
                print("WARNING: the C control limits are based on the strong hypothesis that the counting of values from the subgroups follow a Poisson distribution (Poisson is a case from the Gamma distribution). If it is not the case, do not use this chart.")
                print("If you are not confident about the statistical distribution, select chart_to_use = \'3s_as_natural_variation\' to use 3 times the standard deviation as estimator for the natural variation (the control limits).\n")
            
            
        # Now, we have all the control limits calculated, and data aggregated when it is the case.
        # Let's merge (append - SQL UNION) all the dataframes stored in the dictionaries (elements)
        # from the list list_of_dictionaries_with_dfs. Pick the first dictionary, i.e., element of
        # index 0 from the list:
        
        dictionary = list_of_dictionaries_with_dfs[0]
        # access the dataframe:
        df = dictionary['df']
        
        # Also, start a list for storing the timestamps where the event frames start:
        time_of_event_frame_start = []
        # Let's pick the last element from df[timestamp_tag_column]: it is the time
        # when the time window (event frame) is changing. We can convert it to a list
        # and slice the list from its last element (index -1; the immediately before is -2,
        # etc).
        timestamp_tag_value = list(df[timestamp_tag_column])[-1:]
        # Now concatenate this 1-element list with time_of_event_frame_start:
        time_of_event_frame_start = time_of_event_frame_start + timestamp_tag_value
        # When concatenating lists, the elements from the right list are sequentially added to the
        # first one: list1 = ['a', 'b'], list2 = ['c', 'd'], list1 + list2 = ['a', 'b', 'c', 'd'] 
        
        # Now, let's loop through each one of the other dictionaries from the list:
        
        for i in range (1, len(list_of_dictionaries_with_dfs)):
            
            # start from i = 1, index of the second element (we already picked the first one),
            # and goes to the index of the last dictionary, len(list_of_dictionaries_with_dfs) - 1:
            
            # access element (dictionary) i from the list:
            dictionary_i = list_of_dictionaries_with_dfs[i]
            # access the dataframe i:
            df_i = dictionary_i['df']
            
            # Again, pick the last timestamp from this window:
            timestamp_tag_value = list(df[timestamp_tag_column])[-1:]
            # Now concatenate this 1-element list with time_of_event_frame_start:
            time_of_event_frame_start = time_of_event_frame_start + timestamp_tag_value
            
            # Append df_i to df (SQL UNION - concatenate or append rows):
            # Save in df itself:
            df = pd.concat([df, df_i], axis = 0, ignore_index = True, sort = True, join = 'inner')
            # axis = 0 to append rows; 'inner' join removes missing values
        
        # Now, reset the index of the final concatenated dataframe, containing all the event frames
        # separately processed, and containing the control limits and mean values for each event:
        df = df.reset_index(drop = True)
        
        # Now, add a column to inform if each point is in or out of the control limits.
        # apply a filter to select each situation:
        # (syntax: dataset.loc[dataset['column_filtered'] <= 0.87, 'labelled_column'] = 1)
        # Start the new column as 'in_control_lim' (default case):
        df['control_limits_check'] = 'in_control_limits'
        
        # Get the control limits:
        lower_cl = df['lower_cl']
        upper_cl = df['upper_cl']
        
        # Now modify only points which are out of the control ranges:
        df.loc[(df[column_with_variable_to_be_analyzed] < lower_cl), 'control_limits_check'] = 'below_lower_control_limit'
        df.loc[(df[column_with_variable_to_be_analyzed] > upper_cl), 'control_limits_check'] = 'above_upper_control_limit'
                    
        # Let's also create the 'red_df' containing only the values outside of the control limits.
        # This dataframe will be used to highlight the values outside the control limits
        
        # copy the dataframe:
        red_df = df.copy(deep = True)
        # Filter red_df to contain only the values outside the control limits:
        boolean_filter = ((red_df[column_with_variable_to_be_analyzed] < lower_cl) | (red_df[column_with_variable_to_be_analyzed] > upper_cl))
        red_df = red_df[boolean_filter]
        # Reset the index of this dataframe:
        red_df = red_df.reset_index(drop = True)
        
        if (len(red_df) > 0):
            
            # There is at least one row outside the control limits.
            print("Attention! Point outside of natural variation (control limits).")
            print("Check the red_df dataframe returned for details on values outside the control limits.")
            print("They occur at the following time values:\n")
            
            try:
                # only works in Jupyter Notebook:
                from IPython.display import display
                display(red_df)

            except: # regular mode
                print(list(red_df[timestamp_tag_column]))
        
            print("\n")
        
        # specification_limits = {'lower_spec_lim': value1, 'upper_spec_lim': value2}
        
        # Check if there is a lower specification limit:
        if (specification_limits['lower_spec_lim'] is not None):
            
            lower_spec_lim = specification_limits['lower_spec_lim']
            
            # Now, add a column to inform if each point is in or out of the specification limits.
            # apply a filter to select each situation:

            # Start the new column as 'in_spec_lim' (default case):
            df['spec_limits_check'] = 'in_spec_lim'
            # Now modify only points which are below the specification limit:
            df.loc[(df[column_with_variable_to_be_analyzed] < lower_spec_lim), 'spec_limits_check'] = 'below_lower_spec_lim'
        
            if (len(df[(df[column_with_variable_to_be_analyzed] < lower_spec_lim)]) > 0):
                
                print("Attention! Point below lower specification limit.")
                print("Check the returned dataframe df to obtain more details.\n")
            
            # Check if there is an upper specification limit too:
            if (specification_limits['upper_spec_lim'] is not None):

                upper_spec_lim = specification_limits['upper_spec_lim']

                # Now modify only points which are above the specification limit:
                df.loc[(df[column_with_variable_to_be_analyzed] > upper_spec_lim), 'spec_limits_check'] = 'above_upper_spec_lim'

                if (len(df[(df[column_with_variable_to_be_analyzed] > upper_spec_lim)]) > 0):

                    print("Attention! Point above upper specification limit.")
                    print("Check the returned dataframe df to obtain more details.\n")
        
        # Check the case where there is no lower, but there is a specification limit:
        elif (specification_limits['upper_spec_lim'] is not None):
            
            upper_spec_lim = specification_limits['upper_spec_lim']
            
            # Start the new column as 'in_spec_lim' (default case):
            df['spec_limits_check'] = 'in_spec_lim'
            
            # Now modify only points which are above the specification limit:
            df.loc[(df[column_with_variable_to_be_analyzed] > upper_spec_lim), 'spec_limits_check'] = 'above_upper_spec_lim'

            if (len(df[(df[column_with_variable_to_be_analyzed] > upper_spec_lim)]) > 0):

                print("Attention! Point above upper specification limit.")
                print("Check the returned dataframe df to obtain more details.\n")
        
        
        # Now, we have the final dataframe containing analysis regarding the control and specification
        # limits, and the red_df, that will be used to plot red points for values outside the control
        # range. Then, we can plot the graph.
        
        # Now, we can plot the figure.
        # we set alpha = 0.95 (opacity) to give a degree of transparency (5%), 
        # so that one series do not completely block the visualization of the other.
            
        # Matplotlib linestyle:
        # https://matplotlib.org/stable/gallery/lines_bars_and_markers/linestyles.html?msclkid=68737f24d16011eca9e9c4b41313f1ad
            
        if (plot_title is None):
            
            if (chart_to_use == 'g'):
                plot_title = f"{chart_to_use}_for_count_of_events_between_rare_occurence"
            
            elif (chart_to_use == 't'):
                plot_title = f"{chart_to_use}_for_timedelta_between_rare_occurence"
            
            else:
                plot_title = f"{chart_to_use}_for_{column_with_variable_to_be_analyzed}"
        
        if ((column_with_labels_or_subgroups is None) | (len(unique_labels) <= 1)):
            
            # 'i_mr' or, 'std_error', or '3s_as_natural_variation' (individual measurements)
            
            LABEL = column_with_variable_to_be_analyzed
            
            if (chart_to_use == 'i_mr'):
                LABEL_MEAN = 'mean'
                
            elif ((chart_to_use == '3s_as_natural_variation')|(chart_to_use == 'std_error')):
                if(consider_skewed_dist_when_estimating_with_std):
                    LABEL_MEAN = 'median'

                else:
                    LABEL_MEAN = 'mean'
            
            else:
                
                if (chart_to_use == 'g'):
                    LABEL = 'count_between\nrare_events'
                    LABEL_MEAN = 'median'
                
                elif (chart_to_use == 't'):
                    LABEL = 'timedelta_between\nrare_events'
                    LABEL_MEAN = 'central_line'
                    
        else:
            if ((chart_to_use == '3s_as_natural_variation')|(chart_to_use == 'std_error')):
                
                LABEL = "mean_value\nby_label"
                
                if(consider_skewed_dist_when_estimating_with_std):
                    LABEL_MEAN = 'median'
                else:
                    LABEL_MEAN = 'mean'
            
            elif (chart_to_use == 'xbar_s'):
                
                LABEL = "mean_value\nby_label"
                LABEL_MEAN = 'mean'
                
            elif (chart_to_use == 'np'):
                
                LABEL = "total_occurences\nby_label"
                LABEL_MEAN = 'sum_of_ocurrences'
                
            elif (chart_to_use == 'p'):
                
                LABEL = "mean_value\ngrouped_by_label"
                LABEL_MEAN = 'mean'
            
            elif (chart_to_use == 'u'):
                
                LABEL = "mean_value\ngrouped_by_label"
                LABEL_MEAN = 'mean'
                
            else:
                # chart_to_use == 'c'
                LABEL = "total_occurences\nby_label"
                LABEL_MEAN = 'average_sum\nof_ocurrences'
        
        x = df[timestamp_tag_column]
        
        y = df[column_with_variable_to_be_analyzed]  
        upper_control_lim = df['upper_cl']
        lower_control_lim = df['lower_cl']
        mean_line = df['center']
        
        if (specification_limits['lower_spec_lim'] is not None):
            
            lower_spec_lim = specification_limits['lower_spec_lim']
        
        else:
            lower_spec_lim = None
        
        if (specification_limits['upper_spec_lim'] is not None):
            
            upper_spec_lim = specification_limits['upper_spec_lim']
        
        else:
            upper_spec_lim = None
        
        if (len(red_df) > 0):
            
            red_x = red_df[timestamp_tag_column]
            red_y = red_df[column_with_variable_to_be_analyzed]
            
        # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
        # so that the bars do not completely block other views.
        OPACITY = 0.95
            
        #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
        fig = plt.figure(figsize = (12, 8))
        ax = fig.add_subplot()
        
        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 0 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)
        
        # Set graphic title
        ax.set_title(plot_title) 

        if not (horizontal_axis_title is None):
            # Set horizontal axis title
            ax.set_xlabel(horizontal_axis_title)

        if not (vertical_axis_title is None):
            # Set vertical axis title
            ax.set_ylabel(vertical_axis_title)

        # Scatter plot of time series:
        ax.plot(x, y, linestyle = "-", marker = '', color = 'darkblue', alpha = OPACITY, label = LABEL)
        # Axes.plot documentation:
        # https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.plot.html?msclkid=42bc92c1d13511eca8634a2c93ab89b5
                
        # x and y are positional arguments: they are specified by their position in function
        # call, not by an argument name like 'marker'.
                
        # Matplotlib markers:
        # https://matplotlib.org/stable/api/markers_api.html?msclkid=36c5eec5d16011ec9583a5777dc39d1f
        
        # Plot the mean line as a step function (values connected by straight splines, forming steps):
        ax.step(x, y = mean_line, color = 'fuchsia', linestyle = 'dashed', label = LABEL_MEAN, alpha = OPACITY)
        
        # Plot the control limits as step functions too:
        ax.step(x, y = upper_control_lim, color = 'crimson', linestyle = 'dashed', alpha = OPACITY, label = 'control\nlimit')
        ax.step(x, y = lower_control_lim, color = 'crimson', linestyle = 'dashed', alpha = OPACITY)
        
        # If there are specifications or reference values, plot as horizontal constant lines (axhlines):
        if (lower_spec_lim is not None):
            
            ax.axhline(lower_spec_lim, color = 'black', linestyle = 'dashed', label = 'specification\nlimit', alpha = OPACITY)
        
        if (upper_spec_lim is not None):
            
            ax.axhline(upper_spec_lim, color = 'black', linestyle = 'dashed', label = 'specification\nlimit', alpha = OPACITY)
        
        if (reference_value is not None):
            
            ax.axhline(reference_value, color = 'darkgreen', linestyle = 'dashed', label = 'reference\nvalue', alpha = OPACITY)   
        
        # If there are red points outside of control limits to highlight, plot them above the graph
        # (plot as scatter plot, with no spline, and 100% opacity = 1.0):
        if (len(red_df) > 0):
            
            ax.plot(red_x, red_y, linestyle = '', marker = 'o', color = 'firebrick', alpha = 1.0)
        
        # If the length of list time_of_event_frame_start is higher than zero,
        # loop through each element on the list and add a vertical constant line for the timestamp
        # correspondent to the beginning of an event frame:
        
        if (len(time_of_event_frame_start) > 0):
            
            for timestamp in time_of_event_frame_start:
                # add timestamp as a vertical line (axvline):
                ax.axvline(timestamp, color = 'aqua', linestyle = 'dashed', label = 'event_frame\nchange', alpha = OPACITY)
                
        # Now we finished plotting all of the series, we can set the general configuration:
        ax.grid(grid) # show grid or not
        ax.legend()

        if (export_png == True):
            # Image will be exported
            import os

            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""

            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = f"control_chart_{chart_to_use}"

            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330

            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)

            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

        #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
        #plt.figure(figsize = (12, 8))
        #fig.tight_layout()

        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()
        
        return df, red_df


    def process_capability (df, column_with_variable_to_be_analyzed, specification_limits = {'lower_spec_lim': None, 'upper_spec_lim': None}, reference_value = None, x_axis_rotation = 0, y_axis_rotation = 0, grid = True, horizontal_axis_title = None, vertical_axis_title = None, plot_title = None, export_png = False, directory_to_save = None, file_name = None, png_resolution_dpi = 330):
        
        import numpy as np
        import pandas as pd
        import matplotlib.pyplot as plt
        from scipy import stats
        
        # COLUMN_WITH_VARIABLE_TO_BE_ANALYZED: name (header) of the column containing the variable
        # which stability will be analyzed by the control chart. The column name may be a string or a number.
        # Example: COLUMN_WITH_VARIABLE_TO_BE_ANALYZED = 'analyzed_column' will analyze a column named
        # "analyzed_column", whereas COLUMN_WITH_VARIABLE_TO_BE_ANALYZED = 'col1' will evaluate column 'col1'.

        # SPECIFICATION_LIMITS = {'lower_spec_lim': None, 'upper_spec_lim': None}
        # If there are specification limits, input them in this dictionary. Do not modify the keys,
        # simply substitute None by the lower and/or the upper specification.
        # e.g. Suppose you have a tank that cannot have more than 10 L. So:
        # SPECIFICATION_LIMITS = {'lower_spec_lim': None, 'upper_spec_lim': 10}, there is only
        # an upper specification equals to 10 (do not add units);
        # Suppose a temperature cannot be lower than 10 ºC, but there is no upper specification. So,
        # SPECIFICATION_LIMITS = {'lower_spec_lim': 10, 'upper_spec_lim': None}. Finally, suppose
        # a liquid which pH must be between 6.8 and 7.2:
        # SPECIFICATION_LIMITS = {'lower_spec_lim': 6.8, 'upper_spec_lim': 7.2}

        # REFERENCE_VALUE: keep it as None or add a float value.
        # This reference value will be shown as a red constant line to be compared
        # with the plots. e.g. REFERENCE_VALUE = 1.0 will plot a line passing through
        # VARIABLE_TO_ANALYZE = 1.0
        
        # Set a local copy of the dataframe to manipulate:
        DATASET = df.copy(deep = True)
        
        # Sort by the column to analyze (ascending order) and reset the index:
        DATASET = DATASET.sort_values(by = column_with_variable_to_be_analyzed, ascending = True)
        
        DATASET = DATASET.reset_index(drop = True)
        
        # Create an instance (object) from class capability_analysis:
        capability_obj = capability_analysis(df = DATASET, column_with_variable_to_be_analyzed = column_with_variable_to_be_analyzed, specification_limits = specification_limits)
        
        # Check data normality:
        capability_obj = capability_obj.check_data_normality()
        # Attribute .normality_dict: dictionary with results from normality tests
        
        # Get histogram array:
        capability_obj = capability_obj.get_histogram_array()
        # Attribute .histogram_dict: dictionary with keys 'list_of_bins' and 'list_of_counts'.
        
        # Get desired normal:
        capability_obj = capability_obj.get_desired_normal()
        # Now the .specification_limits attribute contains the nested dict desired_normal = {'x': x_of_normal, 'y': y_normal}
        # in key 'desired_normal'.
        
        # Get the actual probability density function (PDF):
        capability_obj = capability_obj.get_actual_pdf()
        # Now the dictionary in the attribute .specification_limits has the nested dict actual_pdf = {'x': array_to_analyze, 'y': array_of_probs}
        # in key 'actual_pdf'.
        
        # Get capability indicators:
        capability_obj = capability_obj.get_capability_indicators()
        # Now the attribute capability_dict has a dictionary with the indicators, already in the format
        # for pandas DataFrame constructor.
        
        # Retrieve general statistics:
        stats_dict = {
            
            'sample_size': capability_obj.sample_size,
            'mu': capability_obj.mu,
            'median': capability_obj.median,
            'sigma': capability_obj.sigma,
            'lowest': capability_obj.lowest,
            'highest': capability_obj.highest
        }
        
        # Retrieve the normality summary:
        normality_dict = capability_obj.normality_dict
        # Nest this dictionary in stats_dict:
        stats_dict['normality_dict'] = normality_dict
        
        # Retrieve the histogram dict:
        histogram_dict = capability_obj.histogram_dict
        # Nest this dictionary in stats_dict:
        stats_dict['histogram_dict'] = histogram_dict
        
        # Retrieve the specification limits dictionary updated:
        specification_limits = capability_obj.specification_limits
        # Retrieve the desired normal and actual PDFs dictionaries:
        desired_normal = specification_limits['desired_normal']
        # Nest this dictionary in stats_dict:
        stats_dict['desired_normal'] = desired_normal
        
        actual_pdf = specification_limits['actual_pdf']
        # Nest this dictionary in stats_dict:
        stats_dict['actual_pdf'] = actual_pdf
        
        # Retrieve the capability dictionary:
        capability_dict = capability_obj.capability_dict
        # Create a capability dataframe:
        capability_df = pd.DataFrame(data = capability_dict)
        # Set column 'indicator' as index:
        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.set_index.html
        capability_df.set_index('indicator', inplace = True)
        # Nest this dataframe in stats_dict:
        stats_dict['capability_df'] = capability_df
        
        print("\n")
        print("Check the capability summary dataframe:\n")
        
        try:
            # only works in Jupyter Notebook:
            from IPython.display import display
            display(capability_df)
                
        except: # regular mode
            print(capability_df)
        
        # Print the indicators' interpretation:
        capability_obj.capability_interpretation()
        
        string_for_title = " - $\mu = %.2f$, $\sigma = %.2f$" %(stats_dict['mu'], stats_dict['sigma'])
        
        if (plot_title is not None):
            plot_title = plot_title + string_for_title
            
        else:
            # Set graphic title
            plot_title = f"Process Capability" + string_for_title

        if (horizontal_axis_title is None):
            # Set horizontal axis title
            horizontal_axis_title = column_with_variable_to_be_analyzed

        if (vertical_axis_title is None):
            # Set vertical axis title
            vertical_axis_title = "Counting/Frequency"
            
        y_hist = DATASET[column_with_variable_to_be_analyzed]
        number_of_bins = histogram_dict['number_of_bins']
        
        upper_spec = specification_limits['upper_spec_lim']
        lower_spec = specification_limits['lower_spec_lim']
        target = (upper_spec + lower_spec)/2 # center of the specification range
        
        # Let's put a small degree of transparency (1 - OPACITY) = 0.05 = 5%
        # so that the bars do not completely block other views.
        OPACITY = 0.95
        
        # Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
        fig = plt.figure(figsize = (12, 8))
        ax = fig.add_subplot()
        
        #STANDARD MATPLOTLIB METHOD:
        #bins = number of bins (intervals) of the histogram. Adjust it manually
        #increasing bins will increase the histogram's resolution, but height of bars
        
        ax.hist(y_hist, bins = number_of_bins, alpha = OPACITY, label = f'counting_of\n{column_with_variable_to_be_analyzed}', color = 'darkblue')
        
        # desired_normal = {'x': x_of_normal, 'y': y_normal}
        # actual_pdf = {'x': array_to_analyze, 'y': array_of_probs}
        
        # Plot the probability density function for the data:
        pdf_x = actual_pdf['x']
        pdf_y = actual_pdf['y']
        
        ax.plot(pdf_x, pdf_y, color = 'darkgreen', linestyle = '-', alpha = OPACITY, label = 'probability\ndensity')
        
        # Check if a normal curve was obtained:
        x_of_normal = desired_normal['x']
        y_normal = desired_normal['y']
        
        if (len(x_of_normal) > 0):
            # Non-empty list, add the normal curve:
            ax.plot(x_of_normal, y_normal, color = 'crimson', linestyle = 'dashed', alpha = OPACITY, label = 'expected\nnormal_curve')
        
        # Add the specification limits and target vertical lines (axvline):
        ax.axvline(upper_spec, color = 'black', linestyle = 'dashed', label = 'specification\nlimit', alpha = OPACITY)
        ax.axvline(lower_spec, color = 'black', linestyle = 'dashed', alpha = OPACITY)
        ax.axvline(target, color = 'aqua', linestyle = 'dashed', label = 'target\nvalue', alpha = OPACITY)

        # If there is a reference value, plot it as a vertical line:
        if (reference_value is not None):
            ax.axvline(reference_value, color = 'fuchsia', linestyle = 'dashed', label = 'reference\nvalue', alpha = OPACITY)
        
        #ROTATE X AXIS IN XX DEGREES
        plt.xticks(rotation = x_axis_rotation)
        # XX = 0 DEGREES x_axis (Default)
        #ROTATE Y AXIS IN XX DEGREES:
        plt.yticks(rotation = y_axis_rotation)
        # XX = 0 DEGREES y_axis (Default)

        ax.set_title(plot_title)
        ax.set_xlabel(horizontal_axis_title)
        ax.set_ylabel(vertical_axis_title)

        ax.grid(grid) # show grid or not
        ax.legend(loc = 'upper right')
        # position options: 'upper right'; 'upper left'; 'lower left'; 'lower right';
        # 'right', 'center left'; 'center right'; 'lower center'; 'upper center', 'center'
        # https://www.statology.org/matplotlib-legend-position/

        if (export_png == True):
            # Image will be exported
            import os

            #check if the user defined a directory path. If not, set as the default root path:
            if (directory_to_save is None):
                #set as the default
                directory_to_save = ""

            #check if the user defined a file name. If not, set as the default name for this
            # function.
            if (file_name is None):
                #set as the default
                file_name = "capability_plot"

            #check if the user defined an image resolution. If not, set as the default 110 dpi
            # resolution.
            if (png_resolution_dpi is None):
                #set as 330 dpi
                png_resolution_dpi = 330

            #Get the new_file_path
            new_file_path = os.path.join(directory_to_save, file_name)

            #Export the file to this new path:
            # The extension will be automatically added by the savefig method:
            plt.savefig(new_file_path, dpi = png_resolution_dpi, quality = 100, format = 'png', transparent = False) 
            #quality could be set from 1 to 100, where 100 is the best quality
            #format (str, supported formats) = 'png', 'pdf', 'ps', 'eps' or 'svg'
            #transparent = True or False
            # For other parameters of .savefig method, check https://indianaiproduction.com/matplotlib-savefig/
            print (f"Figure exported as \'{new_file_path}.png\'. Any previous file in this root path was overwritten.")

        #Set image size (x-pixels, y-pixels) for printing in the notebook's cell:
        #plt.figure(figsize = (12, 8))
        #fig.tight_layout()

        ## Show an image read from an image file:
        ## import matplotlib.image as pltimg
        ## img=pltimg.imread('mydecisiontree.png')
        ## imgplot = plt.imshow(img)
        ## See linkedIn Learning course: "Supervised machine learning and the technology boom",
        ##  Ex_Files_Supervised_Learning, Exercise Files, lesson '03. Decision Trees', '03_05', 
        ##  '03_05_END.ipynb'
        plt.show()

        return stats_dict
