# -*- coding: utf-8 -*-

# Standard library imports
from collections import OrderedDict, defaultdict
from glob import glob
import logging

# Third party imports
import numpy as np
import pandas as pd
from scipy.ndimage import gaussian_filter, gaussian_filter1d
import plotly.graph_objs as go
import plotly.offline as py

# Local lib import
from pycoQC.common import pycoQCError, pycoQCWarning

# Logger setup
logging.basicConfig(level=logging.INFO, format='%(message)s')
logger = logging.getLogger(__name__)

# Set seed for deterministic random sampling
np.random.RandomState(seed=42)

##~~~~~~~ MAIN CLASS ~~~~~~~#
class pycoQC ():

    #~~~~~~~FUNDAMENTAL METHODS~~~~~~~#
    def __init__ (self,
        seq_summary_file,
        runid_list = [],
        min_pass_qual = 7,
        filter_calibration=True,
        verbose_level = 0):
        """
        Parse Albacore sequencing_summary.txt file and clean-up the data
        * seq_summary_file: STR
            Path to the sequencing_summary generated by Albacore 1.0.0 +. One can also pass a UNIX style regex to match
            multiple files with glob https://docs.python.org/3.6/library/glob.html
        * runid_list: LIST of STR [Default []]
            Select only specific runids to be analysed. Can also be used to force pycoQC to order the runids for
            temporal plots, if the sequencing_summary file contain several sucessive runs. By default pycoQC analyses
            all the runids in the file and uses the runid order as defined in the file.
        * filter_calibration BOOL [Default True]
            If True read flagged as calibration strand by the software are removed
        * min_pass_qual INT [Default 7]
            Minimum quality to consider a read as 'pass'
        * verbose_level INT [Default 0]
            Level of verbosity, from 2 (Chatty) to 0 (Nothing)
        """
        # Set logging level
        logLevel_dict = {2:logging.DEBUG, 1:logging.INFO, 0:logging.WARNING}
        logger.setLevel (logLevel_dict.get (verbose_level, logging.INFO))

        # Check that file is readable and import the summary file in a dataframe
        logger.info ("Import raw data from sequencing summary files")
        try:
            df_list = []
            fp_list = glob (seq_summary_file)
            if len(fp_list) == 0:
                raise pycoQCError ("Could not find any files matching {}".format(seq_summary_file))
            logger.debug ("\tSequencing summary files found: {}".format(fp_list))
            for fp in fp_list:
                df_list.append (pd.read_csv(fp, sep ="\t"))
            df = pd.concat(df_list, ignore_index=True, sort=False, join="inner")
        except IOError:
            raise pycoQCError ("File {} is not readeable".format(fp))
        if len(df) == 0:
            raise pycoQCError ("No valid read found in input file")
        logger.info ("\t{:,} reads found in initial file".format(len(df)))

        # Define specific parameters depending on the run_type
        logger.info ("Verify fields and discard unused columns")

        if "sequence_length_template" in df:
            logger.info ("\t1D Run type")
            self.run_type = "1D"
            required_colnames = ["read_id", "run_id", "channel", "start_time", "sequence_length_template", "mean_qscore_template"]
            optional_colnames = ["calibration_strand_genome_template", "barcode_arrangement"]
            rename_colmanes = {
                "sequence_length_template":"num_bases", "mean_qscore_template":"mean_qscore",
                "calibration_strand_genome_template":"calibration","barcode_arrangement":"barcode"}

        elif "sequence_length_2d" in df:
            logger.info ("\t1D2 Run type")
            self.run_type = "1D2"
            required_colnames = ["read_id", "run_id", "channel", "start_time", "sequence_length_2d", "mean_qscore_2d"]
            optional_colnames = ["calibration_strand_genome_template", "barcode_arrangement"]
            rename_colmanes = {
                "sequence_length_2d":"num_bases", "mean_qscore_2d":"mean_qscore",
                "calibration_strand_genome_template":"calibration", "barcode_arrangement":"barcode"}
        else:
            raise pycoQCError ("Invalid sequencing summary file")

        # Verify that the required and optional columns, Drop unused fields and standardise field names.
        col = self._check_columns (df=df, required_colnames=required_colnames, optional_colnames=optional_colnames)
        logger.debug ("\tColumns found: {}".format(col))
        df = df[col]
        df = df.rename(columns=rename_colmanes)

        # Drop lines containing NA values
        logger.info ("Drop lines containing NA values")
        l = len(df)
        df = df.dropna()
        logger.info ("\t{:,} reads discarded".format(l-len(df)))
        if len(df) <= 1:
            raise pycoQCError("No valid read left after NA values filtering")


        # Filter out zero length reads
        if (df["num_bases"]==0).any():
            logger.info ("Filter out zero length reads")
            l = len(df)
            df = df[(df["num_bases"] > 0)]
            logger.info ("\t{} reads discarded".format(l-len(df)))
            if len(df) <= 1:
                raise pycoQCError("No valid read left after zero_len filtering")

        # Filter out calibration strands read if the "calibration_strand_genome_template" field is available
        if filter_calibration and "calibration" in df:
            logger.info ("Filter out calibration strand reads")
            l = len(df)
            df = df[(df["calibration"].isin(["filtered_out", "no_match"]))]
            logger.info ("\t{:,} reads discarded".format(l-len(df)))
            if len(df) <= 1:
                raise pycoQCError("No valid read left after calibration strand filtering")

        # Filter and reorder based on runid_list list if passed by user
        if runid_list:
            logger.info ("Select run_ids passed by user")
            df = df[(df["run_id"].isin(runid_list))]
            if len(df) <= 1:
                raise pycoQCError("No valid read left after run ID filtering")

        # Else sort the runids by output per time assuming that the throughput  decreases over time
        else:
            logger.info ("Sort run IDs by decreasing throughput")
            d = {}
            for run_id, sdf in df.groupby("run_id"):
                d[run_id] = len(sdf)/sdf["start_time"].ptp()
            runid_list = [i for i, j in sorted (d.items(), key=lambda t: t[1], reverse=True)]
            logger.debug ("\tRun-id order {}".format(runid_list))

        # Modify start time per run ids to order them following the runid_list
        logger.info ("\tReorder runids")
        increment_time = 0
        runid_start = OrderedDict()
        for runid in runid_list:
            logger.debug ("\tProcessing reads with Run_ID {} / time offset: {}".format(runid, increment_time))
            max_val = df['start_time'][df["run_id"] == runid].max()
            df.loc[df["run_id"] == runid, 'start_time'] += increment_time
            runid_start[runid] = increment_time
            increment_time += max_val+1
        df = df.sort_values ("start_time")

        # Reindex final df
        logger.info ("Reindex dataframe by read_ids")
        df = df.reset_index (drop=True)
        df = df.set_index ("read_id")

        # Save self var
        self.all_df = df
        df = df[df["mean_qscore"]>=min_pass_qual]
        if len(df) <= 1:
            raise pycoQCError("No valid read left after quality filtering")
        self.pass_df = df[df["mean_qscore"]>=min_pass_qual]
        self._min_pass_qual = min_pass_qual

        if len(self.all_df) < 100:
            logger.info ("Low number of valid reads found. This is likely to lead to errors when trying to generate plots")
        if len(self.pass_df) < 100:
            logger.info ("Low number of pass reads found. This is likely to lead to errors when trying to generate plots")

        # Detailed report for debug level only
        logger.debug (str(self))

    def __str__(self):
        """
        readable description of the object
        """
        msg = "[{}]\n".format(self.__class__.__name__)
        msg += "\tTotal reads: {:,}\n".format(len(self.all_df))
        msg += "\tPass reads: {:,}\n".format(len(self.pass_df))
        msg += "\tMinimal Pass Quality: {}\n".format(self._min_pass_qual)
        msg += "\tRun Duration: {} h\n".format(round((self.all_df["start_time"].ptp())/3600, 2))
        msg += "\tTotal Bases: {:,}\n".format(self.all_df["num_bases"].sum())
        msg += "\tBarcode found: {}\n".format(self.has_barcodes)

        return msg

    #~~~~~~~PROPERTY METHODS~~~~~~~#
    @property
    def has_barcodes (self):
        return "barcode" in self.all_df

    @property
    def is_promethion (self):
        return self.all_df["channel"].max() > 512

    #~~~~~~~SUMMARY METHOD AND HELPER~~~~~~~#
    def summary (self,
        width = None,
        height = None,
        plot_title="Run summary"):
        """
        Plot an interactive summary table
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        """
        # Prepare all data
        dd1 = self.__summary_data (df=self.all_df)
        dd2 = self.__summary_data (df=self.pass_df)

        # Plot initial data
        data = [go.Table(header = dd1["header"][0], cells = dd1["cells"][0], columnwidth = [60, 20])]

        # Create update buttons
        updatemenus = [
            dict (type="buttons", active=0, x=-0.05, y=1, xanchor='right', yanchor='top', buttons = [
                dict (label='All Reads', method='restyle', args=[dd1]),
                dict (label='Pass Reads', method='restyle', args=[dd2])])]

        # Autodefine height depending on the numbers of run_ids
        if not height:
            height = 300+(30*self.all_df["run_id"].nunique())
        # tweak plot layout
        layout = go.Layout (updatemenus=updatemenus, width=width, height=height, title=plot_title)

        return go.Figure (data=data, layout=layout)

    def __summary_data (self, df):
        """
        Private function preparing data for summary
        """

        header=["Run_ID", "Reads", "Bases", "Med Read Length", "N50 Length", "Med Read Quality", "Active Channels", "Run Duration (h)"]
        if "barcode" in df:
            header.append("Unique Barcodes")

        cells = []
        cells.append (self.__df_to_cell("All Run_IDs", df))
        for run_id, sdf in df.groupby ("run_id"):
            cells.append (self.__df_to_cell(run_id, sdf))

        # Transpose list of list
        cells = [*zip(*cells)]

        data_dict = dict (
            header = [{"values":header, "fill":{"color":"lightgrey"}, "align":"center", "font":{"color":'black', "size":12}, "height":40}],
            cells  = [{"values":cells, "fill":{"color":["lightgrey", "white"]}, "align":"center", "font":{"color":'black', "size":12}, "height":30}])

        return data_dict

    def __df_to_cell (self, run_id, df):
        """Extract information from sub-dataframes and return a list of values"""
        l = []
        l.append (run_id)
        l.append ("{:,}".format(len(df)))
        l.append ("{:,}".format(df["num_bases"].sum()))
        l.append ("{:,.2f}".format(df["num_bases"].median()))
        l.append ("{:,.2f}".format(self._compute_N50(df["num_bases"])))
        l.append ("{:,.2f}".format(df["mean_qscore"].median()))
        l.append ("{:,}".format(df["channel"].nunique()))
        l.append ("{:,.2f}".format(df["start_time"].ptp()/3600))
        if "barcode" in df:
            l.append ("{:,}".format(df["barcode"].nunique()))
        return l

    #~~~~~~~1D DISTRIBUTION METHODS AND HELPER~~~~~~~#

    def reads_len_1D (self,
        color="lightsteelblue",
        width=None,
        height=500,
        nbins=200,
        smooth_sigma=2,
        sample=100000,
        plot_title="Distribution of read length"):
        """
        Plot a distribution of read length (log scale)
        * color: Color of the area (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        * nbins: Number of bins to devide the x axis in
        * smooth_sigma: standard deviation for Gaussian kernel
        * sample: If given, a n number of reads will be randomly selected instead of the entire dataset
        """
        # Prepare all data
        dd1, ld1 = self.__reads_1D_data (self.all_df, field_name="num_bases", xscale="log", nbins=nbins, smooth_sigma=smooth_sigma, sample=sample)
        dd2, ld2 = self.__reads_1D_data (self.pass_df, field_name="num_bases", xscale="log", nbins=nbins, smooth_sigma=smooth_sigma, sample=sample)

        # Plot initial data
        line_style = {'color':'gray','width':1,'dash': 'dot'}
        data = [
            go.Scatter (x=dd1["x"][0], y=dd1["y"][0], name=dd1["name"][0], fill='tozeroy', fillcolor=color, mode='none', showlegend=True),
            go.Scatter (x=dd1["x"][1], y=dd1["y"][1], name=dd1["name"][1], text=dd1["text"][1], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][2], y=dd1["y"][2], name=dd1["name"][2], text=dd1["text"][2], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][3], y=dd1["y"][3], name=dd1["name"][3], text=dd1["text"][3], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][4], y=dd1["y"][4], name=dd1["name"][4], text=dd1["text"][4], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][5], y=dd1["y"][5], name=dd1["name"][5], text=dd1["text"][5], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style)]

        # Create update buttons
        updatemenus = [
            dict (type="buttons", active=0, x=-0.2, y=0, xanchor='left', yanchor='bottom', buttons = [
                dict (label='All Reads', method='update', args=[dd1, ld1]),
                dict (label='Pass Reads', method='update', args=[dd2, ld2])])]

        # tweak plot layout
        layout = go.Layout (
            hovermode="closest",
            legend={"x":-0.2, "y":1,"xanchor":'left',"yanchor":'top'},
            updatemenus=updatemenus,
            width=width,
            height=height,
            title = plot_title,
            xaxis = {"title":"Read length (log scale)", "type":"log", "zeroline":False, "showline":True},
            yaxis = {"title":"Read density", "zeroline":False, "showline":True, "fixedrange":True, "range":ld1["yaxis.range"]})

        return go.Figure (data=data, layout=layout)

    def reads_qual_1D (self,
        color="salmon",
        width=None,
        height=500,
        nbins=200,
        smooth_sigma=2,
        sample=100000,
        plot_title="Distribution of read quality scores"):
        """
        Plot a distribution of quality scores
        * color: Color of the area (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        * nbins: Number of bins to devide the x axis in
        * smooth_sigma: standard deviation for Gaussian kernel
        * sample: If given, a n number of reads will be randomly selected instead of the entire dataset
        """

        # Prepare all data
        dd1, ld1 = self.__reads_1D_data (self.all_df, field_name="mean_qscore", nbins=nbins, smooth_sigma=smooth_sigma, sample=sample)
        dd2, ld2 = self.__reads_1D_data (self.pass_df, field_name="mean_qscore", nbins=nbins, smooth_sigma=smooth_sigma, sample=sample)

        # Plot initial data
        line_style = {'color':'gray','width':1,'dash': 'dot'}
        data = [
            go.Scatter (x=dd1["x"][0], y=dd1["y"][0], name=dd1["name"][0], fill='tozeroy', fillcolor=color, mode='none', showlegend=True),
            go.Scatter (x=dd1["x"][1], y=dd1["y"][1], name=dd1["name"][1], text=dd1["text"][1], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][2], y=dd1["y"][2], name=dd1["name"][2], text=dd1["text"][2], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][3], y=dd1["y"][3], name=dd1["name"][3], text=dd1["text"][3], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][4], y=dd1["y"][4], name=dd1["name"][4], text=dd1["text"][4], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style),
            go.Scatter (x=dd1["x"][5], y=dd1["y"][5], name=dd1["name"][5], text=dd1["text"][5], mode="lines+text", hoverinfo="skip", textposition='top center', line= line_style)]

        # Create update buttons
        updatemenus = [
            dict (type="buttons", active=0, x=-0.2, y=0, xanchor='left', yanchor='bottom', buttons = [
                dict (label='All Reads', method='update', args=[dd1, ld1]),
                dict (label='Pass Reads', method='update', args=[dd2, ld2])])]

        # tweak plot layout
        layout = go.Layout (
            hovermode = "closest",
            legend = {"x":-0.2, "y":1,"xanchor":'left',"yanchor":'top'},
            updatemenus = updatemenus,
            width = width,
            height = height,
            title = plot_title,
            xaxis = {"title":"Read quality scores", "zeroline":False, "showline":True},
            yaxis = {"title":"Read density", "zeroline":False, "showline":True, "fixedrange":True, "range":ld1["yaxis.range"]})

        return go.Figure (data=data, layout=layout)

    def __reads_1D_data (self, df, field_name="num_bases", xscale="linear", nbins=200, smooth_sigma=2, sample=100000):
        """Private function preparing data for reads_len_1D and reads_qual_1D"""
        # Downsample if needed
        if sample and len(df)>sample:
            df = df.sample(sample)

        #Extract data field from df
        data = df[field_name].values

        # Count each categories in log or linear space
        min = np.nanmin(data)
        max = np.nanmax(data)
        if xscale == "log":
            count_y, bins = np.histogram (a=data, bins=np.logspace (np.log10(min), np.log10(max)+0.1, nbins))
        elif xscale == "linear":
            count_y, bins = np.histogram (a=data, bins= np.linspace (min, max, nbins))

        # Remove last bin from labels
        count_x = bins[1:]

        # Smooth results with a savgol filter
        if smooth_sigma:
            count_y = gaussian_filter1d (count_y, sigma=smooth_sigma)

        # Get percentiles percentiles
        stat = np.percentile (data, [10,25,50,75,90])
        y_max = count_y.max()

        data_dict = dict (
            x = [count_x, [stat[0],stat[0]], [stat[1],stat[1]], [stat[2],stat[2]], [stat[3],stat[3]], [stat[4],stat[4]]],
            y = [count_y, [0,y_max], [0,y_max], [0,y_max], [0,y_max], [0,y_max]],
            name = ["Density", "10%", "25%", "Median", "75%", "90%"],
            text = ["",
                ["", "10%<br>{:,.2f}".format(stat[0])],
                ["", "25%<br>{:,.2f}".format(stat[1])],
                ["", "Median<br>{:,.2f}".format(stat[2])],
                ["", "75%<br>{:,.2f}".format(stat[3])],
                ["", "90%<br>{:,.2f}".format(stat[4])]],
        )

        # Make layout dict = Off set for labels on top
        layout_dict = {"yaxis.range": [0, y_max+y_max/6]}

        return [data_dict, layout_dict]

    #~~~~~~~2D DISTRIBUTION METHOD AND HELPER~~~~~~~#

    def reads_len_qual_2D (self,
        colorscale = [[0.0,'rgba(255,255,255,0)'], [0.1,'rgba(255,150,0,0)'], [0.25,'rgb(255,100,0)'], [0.5,'rgb(200,0,0)'], [0.75,'rgb(120,0,0)'], [1.0,'rgb(70,0,0)']],
        width = None,
        height = 600,
        len_nbins = 200,
        qual_nbins = 75,
        smooth_sigma = 2,
        sample = 100000,
        plot_title="Mean read quality per sequence length"):
        """
        Plot a 2D distribution of quality scores vs length of the reads
        * colorscale: a valid plotly color scale https://plot.ly/python/colorscales/ (Not recommanded to change)
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        * len_nbins: Number of bins to divide the read length values in (x axis)
        * qual_nbins: Number of bins to divide the read quality values in (y axis)
        * smooth_sigma: standard deviation for 2D Gaussian kernel
        * sample: If given, a n number of reads will be randomly selected instead of the entire dataset
        """

        # Prepare all data
        dd1 = self.__reads_2D_data (self.all_df, len_nbins=len_nbins, qual_nbins=qual_nbins, smooth_sigma=smooth_sigma, sample=sample)
        dd2 = self.__reads_2D_data (self.pass_df, len_nbins=len_nbins, qual_nbins=qual_nbins, smooth_sigma=smooth_sigma, sample=sample)

        # Plot initial data
        data = [
            go.Contour (x=dd1["x"][0], y=dd1["y"][0], z=dd1["z"][0], contours=dd1["contours"][0],
                name="Density", hoverinfo="name+x+y", colorscale=colorscale, showlegend=True, connectgaps=True, line={"width":0}),
            go.Scatter (x=dd1["x"][1], y=dd1["y"][1],
                mode='markers', name='Median', hoverinfo="name+x+y", marker={"size":12,"color":'black', "symbol":"x"})]

        # Create update buttons
        updatemenus = [
            dict (type="buttons", active=0, x=-0.2, y=0, xanchor='left', yanchor='bottom', buttons = [
                dict (label='All Reads', method='restyle', args=[dd1]),
                dict (label='Pass Reads', method='restyle', args=[dd2])])]

        # tweak plot layout
        layout = go.Layout (
            hovermode = "closest",
            legend = {"x":-0.2, "y":1,"xanchor":'left',"yanchor":'top'},
            updatemenus = updatemenus,
            width = width,
            height = height,
            title = plot_title,
            xaxis = {"title":"Estimated read length", "showgrid":True, "zeroline":False, "showline":True, "type":"log"},
            yaxis = {"title":"Read quality scores", "showgrid":True, "zeroline":False, "showline":True,})

        return go.Figure (data=data, layout=layout)

    def __reads_2D_data (self, df, len_nbins, qual_nbins, smooth_sigma=1.5, sample=100000):
        """ Private function preparing data for reads_len_qual_2D """
        # Downsample if needed
        if sample and len(df)>sample:
            df = df.sample(sample)

        len_data = df["num_bases"]
        qual_data = df["mean_qscore"]

        len_min, len_med, len_max = np.percentile (len_data, (0, 50, 100))
        qual_min, qual_med, qual_max = np.percentile (qual_data, (0, 50, 100))

        len_bins = np.logspace (start=np.log10((len_min)), stop=np.log10(len_max)+0.1, num=len_nbins, base=10)
        qual_bins = np.linspace (start=qual_min, stop=qual_max, num=qual_nbins)
        z, y, x = np.histogram2d (x=qual_data, y=len_data, bins=[qual_bins, len_bins])

        if smooth_sigma:
            z = gaussian_filter(z, sigma=smooth_sigma)

        z_min, z_max = np.percentile (z, (0, 100))

        # Extract label and values
        data_dict = dict (
            x = [x, [len_med]],
            y = [y, [qual_med]],
            z = [z, None],
            contours = [dict(start=z_min, end=z_max, size=(z_max-z_min)/15),None])

        return data_dict

    #~~~~~~~OUTPUT_OVER_TIME METHODS AND HELPER~~~~~~~#

    def output_over_time (self,
        cumulative_color="rgb(204,226,255)",
        interval_color="rgb(102,168,255)",
        width=None,
        height=500,
        time_bins=500,
        sample=100000,
        plot_title="Output over experiment time"):
        """
        Plot a yield over time
        * cumulative_color: Color of cumulative yield area (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * interval_color: Color of interval yield line (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        * time_bins: Number of bins to divide the time values in (x axis)
        * sample: If given, a n number of reads will be randomly selected instead of the entire dataset
        """

        # Prepare all data
        dd1, ld1 = args=self.__output_over_time_data (self.all_df, level="reads", time_bins=time_bins, sample=sample)
        dd2, ld2 = args=self.__output_over_time_data (self.pass_df, level="reads", time_bins=time_bins, sample=sample)
        dd3, ld3 = args=self.__output_over_time_data (self.all_df, level="bases", time_bins=time_bins, sample=sample)
        dd4, ld4 = args=self.__output_over_time_data (self.pass_df, level="bases", time_bins=time_bins, sample=sample)

        # Plot initial data
        line_style = {'color':'gray','width':1,'dash':'dot'}
        data = [
            go.Scatter (x=dd1["x"][0], y=dd1["y"][0], name=dd1["name"][0], fill='tozeroy', fillcolor=cumulative_color, mode='none'),
            go.Scatter (x=dd1["x"][1], y=dd1["y"][1], name=dd1["name"][1], mode='lines', line={'color':interval_color,'width':2}),
            go.Scatter (x=dd1["x"][2], y=dd1["y"][2], name=dd1["name"][2], text=dd1["text"][2], mode="lines+text", hoverinfo="skip", textposition='top center', line=line_style),
            go.Scatter (x=dd1["x"][3], y=dd1["y"][3], name=dd1["name"][3], text=dd1["text"][3], mode="lines+text", hoverinfo="skip", textposition='top center', line=line_style),
            go.Scatter (x=dd1["x"][4], y=dd1["y"][4], name=dd1["name"][4], text=dd1["text"][4], mode="lines+text", hoverinfo="skip", textposition='top center', line=line_style),
            go.Scatter (x=dd1["x"][5], y=dd1["y"][5], name=dd1["name"][5], text=dd1["text"][5], mode="lines+text", hoverinfo="skip", textposition='top center', line=line_style),
            go.Scatter (x=dd1["x"][6], y=dd1["y"][6], name=dd1["name"][6], text=dd1["text"][6], mode="lines+text", hoverinfo="skip", textposition='top center', line=line_style)]

        # Create update buttons
        updatemenus = [
            dict (type="buttons", active=0, x=-0.06, y=0, xanchor='right', yanchor='bottom', buttons = [
                dict (label='All Reads',  method='update', args=[dd1, ld1]),
                dict (label='Pass Reads', method='update', args=[dd2, ld2]),
                dict (label='All Bases',  method='update', args=[dd3, ld3]),
                dict (label='Pass Bases', method='update', args=[dd4, ld4])])]

        # tweak plot layout
        layout = go.Layout (
            width=width,
            height=height,
            updatemenus=updatemenus,
            legend={"x":-0.05, "y":1,"xanchor":'right',"yanchor":'top'},
            title=plot_title,
            xaxis={"title":"Experiment time (h)", "zeroline":False, "showline":True},
            yaxis={"title":"Count", "zeroline":False, "showline":True, "fixedrange":True, "range":ld1["yaxis.range"]})

        return go.Figure (data=data, layout=layout)

    def __output_over_time_data (self, df, level="reads", time_bins=500, sample=100000):
        """Private function preparing data for output_over_time"""

        # Downsample if needed
        scaling_factor=1
        if sample and len(df)>sample:
            scaling_factor = len(df)/sample
            df = df.sample(sample)

        # Bin data in categories
        t = (df["start_time"]/3600).values
        x = np.linspace (t.min(), t.max(), num=time_bins)
        t = np.digitize (t, bins=x, right=True)

        # Count reads or bases per categories
        if level == "reads":
            y = np.bincount(t)
        elif level == "bases":
            y = np.bincount(t, weights=df["num_bases"].values)

        # Scale counts in case of downsampling
        y = y*scaling_factor

        # Transform to cummulative distribution
        y_cum = np.cumsum(y)
        y_cum_max = y_cum[-1]

        # Smooth and rescale interval trace
        y = gaussian_filter1d (y, sigma=1)
        y = y*y_cum_max/y.max()

        # Find percentages of data generated
        lab_text = []
        lab_name = []
        lab_x = []
        for lab in (50, 75, 90, 99, 100):
            val = y_cum_max*lab/100
            idx = (np.abs(y_cum-val)).argmin()
            lab_text.append(["", '{}%<br>{}h<br>{:,} {}'.format(lab, round(x[idx],2), int(y_cum[idx]), level)])
            lab_x.append ([x[idx], x[idx]])
            lab_name.append ("{}%".format(lab))

        # make data dict
        data_dict = dict(
            x = [x, x]+lab_x,
            y = [y_cum, y, [0,y_cum_max], [0,y_cum_max], [0,y_cum_max], [0,y_cum_max], [0,y_cum_max]],
            name = ["Cumulative", "Interval"]+lab_name,
            text = ["", ""]+lab_text)

        # Make layout dict = offset for labels on top
        layout_dict = {"yaxis.range": [0, y_cum_max+y_cum_max/6]}

        return [data_dict, layout_dict]

    #~~~~~~~QUAL_OVER_TIME METHODS AND HELPER~~~~~~~#

    def len_over_time (self,
        median_color="rgb(102,168,255)",
        quartile_color="rgb(153,197,255)",
        extreme_color="rgba(153,197,255,0.5)",
        smooth_sigma=1,
        width=None,
        height=500,
        time_bins=500,
        sample=100000,
        plot_title="Read length over experiment time"):
        """
        Plot a read length over time
        * median_color: Color of median line color (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * quartile_color: Color of inter quartile area and lines (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * extreme_color:: Color of inter extreme area and lines (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-col
        * smooth_sigma: sigma parameter for the Gaussian filter line smoothing
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        * time_bins: Number of bins to divide the time values in (x axis)
        * sample: If given, a n number of reads will be randomly selected instead of the entire dataset
        """

        # Prepare all data
        dd1 = self.__over_time_data (self.all_df, field_name="num_bases", smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)
        dd2 = self.__over_time_data (self.pass_df, field_name="num_bases", smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)

        # Plot initial data
        data= [
            go.Scatter(x=dd1["x"][0], y=dd1["y"][0], name=dd1["name"][0], mode="lines", line={"color":extreme_color}, connectgaps=True, legendgroup="Extreme"),
            go.Scatter(x=dd1["x"][1], y=dd1["y"][1], name=dd1["name"][1], mode="lines", fill="tonexty", line={"color":extreme_color}, connectgaps=True, legendgroup="Extreme"),
            go.Scatter(x=dd1["x"][2], y=dd1["y"][2], name=dd1["name"][2], mode="lines", line={"color":quartile_color}, connectgaps=True, legendgroup="Quartiles"),
            go.Scatter(x=dd1["x"][3], y=dd1["y"][3], name=dd1["name"][3], mode="lines", fill="tonexty", line={"color":quartile_color}, connectgaps=True, legendgroup="Quartiles"),
            go.Scatter(x=dd1["x"][4], y=dd1["y"][4], name=dd1["name"][4], mode="lines", line={"color":median_color}, connectgaps=True)]

        # Create update buttons
        updatemenus = [
            go.layout.Updatemenu (type="buttons", active=0, x=-0.07, y=0, xanchor='right', yanchor='bottom', buttons = [
                go.layout.updatemenu.Button (
                    label='All Reads', method='restyle', args=[dd1]),
                go.layout.updatemenu.Button (
                    label='Pass Reads', method='restyle', args=[dd2])])]

        # tweak plot layout
        layout = go.Layout (
            width=width,
            height=height,
            updatemenus=updatemenus,
            legend={"x":-0.07, "y":1,"xanchor":'right',"yanchor":'top'},
            title=plot_title,
            yaxis={"title":"Read length (log scale)", "type":"log", "zeroline":False, "showline":True, "rangemode":'nonnegative', "fixedrange":True},
            xaxis={"title":"Experiment time (h)", "zeroline":False, "showline":True, "rangemode":'nonnegative'})

        return go.Figure (data=data, layout=layout)


    def qual_over_time (self,
        median_color="rgb(250,128,114)",
        quartile_color="rgb(250,170,160)",
        extreme_color="rgba(250,170,160,0.5)",
        smooth_sigma=1,
        width=None,
        height=500,
        time_bins=500,
        sample=100000,
        plot_title="Read quality over experiment time"):
        """
        Plot a mean quality over time
        * median_color: Color of median line color (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * quartile_color: Color of inter quartile area and lines (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * extreme_color:: Color of inter extreme area and lines (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-col
        * smooth_sigma: sigma parameter for the Gaussian filter line smoothing
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        * time_bins: Number of bins to divide the time values in (x axis)
        * sample: If given, a n number of reads will be randomly selected instead of the entire dataset
        """

        # Prepare all data
        dd1 = self.__over_time_data (self.all_df, field_name="mean_qscore", smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)
        dd2 = self.__over_time_data (self.pass_df, field_name="mean_qscore", smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)

        # Plot initial data
        data= [
            go.Scatter(x=dd1["x"][0], y=dd1["y"][0], name=dd1["name"][0], mode="lines", line={"color":extreme_color}, connectgaps=True, legendgroup="Extreme"),
            go.Scatter(x=dd1["x"][1], y=dd1["y"][1], name=dd1["name"][1], mode="lines", fill="tonexty", line={"color":extreme_color}, connectgaps=True, legendgroup="Extreme"),
            go.Scatter(x=dd1["x"][2], y=dd1["y"][2], name=dd1["name"][2], mode="lines", line={"color":quartile_color}, connectgaps=True, legendgroup="Quartiles"),
            go.Scatter(x=dd1["x"][3], y=dd1["y"][3], name=dd1["name"][3], mode="lines", fill="tonexty", line={"color":quartile_color}, connectgaps=True, legendgroup="Quartiles"),
            go.Scatter(x=dd1["x"][4], y=dd1["y"][4], name=dd1["name"][4], mode="lines", line={"color":median_color}, connectgaps=True)]

        # Create update buttons
        updatemenus = [
            go.layout.Updatemenu (type="buttons", active=0, x=-0.07, y=0, xanchor='right', yanchor='bottom', buttons = [
                go.layout.updatemenu.Button (
                    label='All Reads', method='restyle', args=[dd1]),
                go.layout.updatemenu.Button (
                    label='Pass Reads', method='restyle', args=[dd2])])]

        # tweak plot layout
        layout = go.Layout (
            width=width,
            height=height,
            updatemenus=updatemenus,
            legend={"x":-0.07, "y":1,"xanchor":'right',"yanchor":'top'},
            title=plot_title,
            yaxis={"title":"Mean quality", "zeroline":False, "showline":True, "rangemode":'nonnegative', "fixedrange":True},
            xaxis={"title":"Experiment time (h)", "zeroline":False, "showline":True, "rangemode":'nonnegative'})

        return go.Figure (data=data, layout=layout)

    def __over_time_data (self, df, field_name="num_bases", smooth_sigma=1.5, time_bins=500, sample=100000):
        """Private function preparing data for qual_over_time"""

        # Downsample if needed
        if sample and len(df)>sample:
            df = df.sample(sample)

        # Bin data in categories
        t = (df["start_time"]/3600).values
        x = np.linspace (t.min(), t.max(), num=time_bins)
        t = np.digitize (t, bins=x, right=True)

        # List quality value per categories
        bin_dict = defaultdict (list)
        for bin_idx, val in zip (t, df[field_name].values) :
            bin = x[bin_idx]
            bin_dict[bin].append(val)

        # Aggregate values per category
        val_name = ["Min", "Max", "25%", "75%", "Median"]
        stat_dict = defaultdict(list)
        for bin in x:
            if bin in bin_dict:
                p = np.percentile (bin_dict[bin], [0, 100, 25, 75, 50])
            else:
                p = [np.nan,np.nan,np.nan,np.nan,np.nan]
            for val, stat in zip (val_name, p):
                stat_dict[val].append(stat)

        # Values smoothing
        if smooth_sigma:
            for val in val_name:
                stat_dict [val] = gaussian_filter1d (stat_dict [val], sigma=smooth_sigma)

        # make data dict
        data_dict = dict(
            x = [x,x,x,x,x],
            y = [stat_dict["Min"], stat_dict["Max"], stat_dict["25%"], stat_dict["75%"], stat_dict["Median"]],
            name = val_name)

        return data_dict

    #~~~~~~~BARCODE_COUNT METHODS AND HELPER~~~~~~~#
    def barcode_counts (self,
        min_percent_barcode = 0.1,
        colors = ["#f8bc9c", "#f6e9a1", "#f5f8f2", "#92d9f5", "#4f97ba"],
        width =  None,
        height = 500,
        plot_title="Percentage of reads per barcode"):
        """
        Plot a mean quality over time
        * min_percent_barcode: minimal percentage od total reads for a barcode to be reported
        * colors: List of colors (hex, rgb, rgba, hsl, hsv or any CSV named colors https://www.w3.org/TR/css-color-3/#svg-color
        * width: With of the ploting area in pixel
        * height: height of the ploting area in pixel
        """
        # Verify that barcode information are available
        if not self.has_barcodes:
            raise pycoQCError ("No barcode information available")

        # Prepare all data
        dd1 = self.__barcode_counts_data (self.all_df, min_percent_barcode=min_percent_barcode)
        dd2 = self.__barcode_counts_data (self.pass_df, min_percent_barcode=min_percent_barcode)

        # Plot initial data
        data= [go.Pie (labels=dd1["labels"][0] , values=dd1["values"][0] , sort=False, marker=dict(colors=colors))]

        # Create update buttons
        updatemenus = [
            dict (type="buttons", active=0, x=-0.2, y=0, xanchor='left', yanchor='bottom', buttons = [
                dict (label='All Reads', method='restyle', args=[dd1]),
                dict (label='Pass Reads', method='restyle', args=[dd2])])]

        # tweak plot layout
        layout = go.Layout (
            legend = {"x":-0.2, "y":1,"xanchor":'left',"yanchor":'top'},
            updatemenus = updatemenus,
            width = width,
            height = height,
            title = plot_title)

        return go.Figure (data=data, layout=layout)

    def __barcode_counts_data (self, df, min_percent_barcode=0.1):
        """Private function preparing data for barcode_counts"""

        counts = df["barcode"].value_counts()
        counts = counts.sort_index()

        if min_percent_barcode:
            cuttoff = counts.sum()*min_percent_barcode/100
            counts = counts[counts>cuttoff]

        # Extract label and values
        data_dict = dict (
            labels = [counts.index],
            values = [counts.values])

        return data_dict

    #~~~~~~~BARCODE_COUNT METHODS AND HELPER~~~~~~~#

    def channels_activity (self,
        colorscale = [[0.0,'rgba(255,255,255,0)'], [0.01,'rgb(255,255,200)'], [0.25,'rgb(255,200,0)'], [0.5,'rgb(200,0,0)'], [0.75,'rgb(120,0,0)'], [1.0,'rgb(0,0,0)']],
        smooth_sigma=1,
        width=None,
        height=600,
        time_bins=150,
        sample=100000,
        plot_title="Output per channel over experiment time"):
        """
        Plot a yield over time
        * colorscale: a valid plotly color scale https://plot.ly/python/colorscales/ (Not recommanded to change)
        * smooth_sigma: sigma parameter for the Gaussian filter line smoothing
        * width: With of the ploting area in pixel
        * height: Height of the ploting area in pixel
        * time_bins: Number of bins to divide the time values in (y axis)
        * sample: If given, a n number of reads will be randomly selected instead of the entire dataset
        """

        # Define maximal number of channels
        n_channels = 40000 if self.is_promethion else 512

        # Prepare all data
        dd1 = args=self.__channels_activity_data(self.all_df, level="reads", n_channels=n_channels, smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)
        dd2 = args=self.__channels_activity_data(self.pass_df, level="reads", n_channels=n_channels, smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)
        dd3 = args=self.__channels_activity_data(self.all_df, level="bases", n_channels=n_channels, smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)
        dd4 = args=self.__channels_activity_data(self.pass_df, level="bases", n_channels=n_channels, smooth_sigma=smooth_sigma, time_bins=time_bins, sample=sample)

        # Plot initial data
        data = [go.Heatmap(x=dd1["x"][0], y=dd1["y"][0], z=dd1["z"][0], xgap=0.5, colorscale=colorscale, hoverinfo="x+y+z")]

        # Create update buttons
        updatemenus = [
            dict (type="buttons", active=0, x=-0.06, y=0, xanchor='right', yanchor='bottom', buttons = [
                dict (label='All Reads', method='restyle', args=[dd1]),
                dict (label='Pass Reads', method='restyle', args=[dd2]),
                dict (label='All Bases', method='restyle', args=[dd3]),
                dict (label='Pass Bases', method='restyle', args=[dd4])])]

        # tweak plot layout
        layout = go.Layout (
            width=width,
            height=height,
            updatemenus=updatemenus,
            title=plot_title,
            xaxis={"title":"Channel id", "zeroline":False, "showline":False, "nticks":20, "showgrid":False},
            yaxis={"title":"Experiment time (h)", "zeroline":False, "showline":False, "hoverformat":".2f", "fixedrange":True})

        return go.Figure (data=data, layout=layout)

    def __channels_activity_data (self, df, level="bases", n_channels=512, smooth_sigma=2, time_bins=150, sample=100000):
        """Private function preparing data for channels_activity"""

        # Downsample if needed
        scaling_factor=1
        if sample and len(df)>sample:
            scaling_factor = len(df)/sample
            df = df.sample(sample)

        # Bin data in categories
        t = (df["start_time"]/3600).values
        bins = np.linspace (t.min(), t.max(), num=time_bins)
        t = np.digitize (t, bins=bins, right=True)

        # Count values per categories
        z = np.ones((len(bins), n_channels), dtype=np.int)
        if level == "bases":
            for t_idx, channel, n_bases in zip(t, df["channel"], df["num_bases"]):
                z[t_idx][channel-1]+=n_bases
        elif level == "reads":
            for t_idx, channel in zip(t, df["channel"]):
                z[t_idx][channel-1]+=1
        # Scale counts in case of downsampling
        z=z*scaling_factor

        # Time series smoothing
        if smooth_sigma:
            z = gaussian_filter1d (z.astype(np.float32), sigma=smooth_sigma, axis=0)

        # Define x and y axis
        x = ["c {}".format(i) for i in range(1, n_channels+1)]
        y = bins[1:]

        # Make data dict
        data_dict = dict (x=[x], y=[y], z=[z])

        return data_dict

    #~~~~~~~PRIVATE METHODS~~~~~~~#
    def _check_columns (self, df, required_colnames, optional_colnames):
        col_found = []
        # Verify the presence of the columns required for pycoQC
        for col in required_colnames:
            if col in df:
                col_found.append(col)
            else:
                raise pycoQCError("Column {} not found in the provided sequence_summary file".format(col))
        for col in optional_colnames:
            if col in df:
                col_found.append(col)
        return col_found

    def _compute_N50 (self, data):
        data = data.values
        data.sort()
        half_sum = data.sum()/2
        cum_sum = 0
        for v in data:
            cum_sum += v
            if cum_sum >= half_sum:
                return v
