#!/usr/bin/env python3
##########################################################
## Jose F. Sanchez                                        ##
## Copyright (C) 2019 Lauro Sumoy Lab, IGTP, Spain        ##
##########################################################
"""
Prepares samples for further analysis.
"""
## useful imports
import time
import io
import os
import re
import sys
import pandas as pd
from termcolor import colored
import shutil
import concurrent.futures

## import my modules
from BacterialTyper.scripts import functions
from BacterialTyper.config import set_config

def help_fastq_format():
    """
    Explanation of fastq format details.
    """

    functions.boxymcboxface("Name format for samples")

    print ("Format for fastq files can be:")
    print ("name.fastq.gz, name_1.fastq.gz, name_R2.fastq.gz, name_L001_R1.fastq.gz, name_L001_R1_001.fastq.gz etc.")
    print ("\nThere are many options and here we provide some guidelines on the name format.")
    print ("\n")

    functions.print_sepLine("*",20,"red")
    print ("[1] Length limitation")
    functions.print_sepLine("*",20,"red")
    print ("There is a limitation for the sample ID ('name') of 25 characters.")
    print (colored("** BacterialTyper provides an option to rename samples if necessary: module prep option --rename **", 'yellow'))
    print ("\n")

    functions.print_sepLine("*",20,"red")
    print ("[2] Single end files")
    functions.print_sepLine("*",20,"red")
    print (colored('** Use option --single-end in the different BacterialTyper modules **', 'yellow'))
    print ("name.fastq.gz")
    print ("name.fastq")
    print ("name.fq")
    print ("\n")

    functions.print_sepLine("*",20,"red")
    print ("[3] Paired-end files")
    functions.print_sepLine("*",20,"red")
    print ("Paired-end files are full supported. The format for these files are:")
    print ("Read1 => name_1.fastq.g or name_R1.fastq.gz")
    print ("Read2 => name_2.fastq.gz or name_R2.fastq.gz")
    print (colored('** See additional details for Lane information **', 'yellow'))
    print ("\n")

    functions.print_sepLine("*",55,"red")
    print ("[4] Lane information:")
    functions.print_sepLine("*",55,"red")
    print ("In some cases, files might contain lane information (*L00x* and/or *00x*).")
    print ("BacterialTyper supports these names as long as follow these examples:")
    print ("name_L00x_R1.fastq.gz\tname_L00x_R2.fastq.gz")
    print ("name_L00x_1.fastq.gz\tname_L00x_2.fastq.gz")
    print ("name_L00x_R1.fastq.gz\tname_L00x_R2.fastq.gz")
    print ("name_L00x_R1_00x.fastq.gz\tname_L00x_R2_00x.fastq.gz")
    print ("\n")
    print ("Sometimes it might be appropriate to include lane tags (*L00X*) within the name.")
    print (colored("** Use option --include-lane within each module", 'yellow'))
    
    print (colored("\n** If you need to merge fastq files from different lanes, use option within module prep **", 'yellow'))
    print("As an example:")
    print (colored("\n** Option --merge within module prep **", 'yellow'))
    print ("sample1_L001_R1.fastq.gz\tsample1_L001_R2.fastq.gz")
    print ("sample1_L002_R1.fastq.gz\tsample1_L002_R2.fastq.gz")
    print ("sample1_L003_R1.fastq.gz\tsample1_L003_R2.fastq.gz")
    print ("sample1_L004_R1.fastq.gz\tsample1_L004_R2.fastq.gz")
    print ("Result:")
    print ("--------------------------------------------------")
    print ("sample1_R1.fastq.gz\tsample1_R2.fastq.gz")
    print ("\n")
    print (colored("\n** Option --merge-by-lane within module prep **", 'yellow'))
    print ("sample1_L001_R1_001.fastq.gz\tsample1_L001_R2_001.fastq.gz")
    print ("sample1_L001_R1_002.fastq.gz\tsample1_L001_R2_002.fastq.gz")
    print ("sample1_L002_R1_001.fastq.gz\tsample1_L002_R2_001.fastq.gz")
    print ("sample1_L002_R1_002.fastq.gz\tsample1_L002_R2_002.fastq.gz")
    print ("--------------------------------------------------")
    print ("Result:")
    print ("sample1_L001_R1.fastq.gz\tsample1_L001_R2.fastq.gz")
    print ("sample1_L002_R1.fastq.gz\tsample1_L002_R2.fastq.gz")
    print (colored("** Remember to use option --include_lane within each module", 'yellow'))
    print ("\n")
    
    functions.print_sepLine("*",55,"red")
    print ("[5] Include all information:")
    functions.print_sepLine("*",55,"red")
    print ("In some cases, files might contain other information and it is necessay to " +
           "include it all as a tag nane. See as an example:")
    print ("sample1_L001_XYZ_R1_001.fastq.gz\tsample1_L001_XYZ_R2_001.fastq.gz")
    print (colored("** Remember to use option --include_all within each module", 'yellow'))
    print (colored("** It might be appropiate to change samples names using --rename option under prep module", 'yellow'))
    
    print ("\n")
    functions.print_sepLine("*",15,"red")
    print ("[6] Extensions:")
    functions.print_sepLine("*",15,"red")
    print ("name_L00x_R2.fastq\tname_L00x_R2.fq\nname_L00x_R2.fastq.gz\tname_L00x_R2.fq.gz")
    print ("\n")



###############
def get_fields(file_name_list, pair, Debug, include_all):
    """
    Get information from files
    
    Creates and returns a dataframe containing information for each sample for:
    "sample", "dirname", "name", "new_name", "name_len", "lane", "read_pair","lane_file","ext","gz", "tag"
    
    :param file_name_list: List of files to parse
    :param pair: True/false for Paired-end files
    :param Debug: True/false for debugging messages
    
    :type file_name_list: list
    :type pair: bool
    :type Debug: bool
    
    :returns: Pandas dataframe.
    
    """
    ## init dataframe
    name_columns = ("sample", "dirname", "name", "new_name", "name_len", 
                "lane", "read_pair","lane_file","ext","gz", "tag", "file")
    name_frame = pd.DataFrame(columns=name_columns)
    
    ## loop through list
    for path_files in file_name_list:
        ## get file name
        file_name = os.path.basename(path_files)
        dirN = os.path.dirname(path_files)
        trim_search = re.search(r".*trim.*", file_name)
        lane_search = re.search(r".*\_L\d+\_.*", file_name)
        ## get name
        if (include_all):
            if (pair):
                name_search = re.search(r"(.*)\_(R1|1|R2|2)\_{0,1}(.*)\.(f.*q)(\..*){0,1}", file_name)
            else:
                name_search = re.search(r"(.*)\.(f.*q)(\..*){0,1}", file_name)

        else:    
            if (pair):
                ## pair could be: R1|R2 or 1|2
                if (trim_search):
                    name_search = re.search(r"(.*)\_trim\_(R1|1|R2|2)\.(f.*q)(\..*){0,1}", file_name)
                else:
                    ## Lane files: need to merge by file_name: 33i_S5_L004_R1_001.fastq.gz
                    ## lane should contain L00x            
                    if (lane_search):
                        name_search = re.search(r"(.*)\_(L\d+)\_(R1|1|R2|2)\_{0,1}(.*)\.(f.*q)(\..*){0,1}", file_name)
                    else:
                        name_search = re.search(r"(.*)\_(R1|1|R2|2)\.(f.*q)(\..*){0,1}", file_name)
            else:
                if (trim_search):
                    name_search = re.search(r"(.*)\_trim\.(f.*q)(\..*){0,1}", file_name)
                else:
                    name_search = re.search(r"(.*)\.(f.*q)(\..*){0,1}", file_name)

        ### declare
        name= ""
        lane_id= ""
        read_pair= ""
        lane_file= ""
        ext= ""
        gz= ""
    
        if name_search:
            name = name_search.group(1)
            name_len = len(name)
            if (pair):
                if (include_all):
                    lane_id = ""
                    read_pair = name_search.group(2)
                    lane_file = name_search.group(3)
                    ext = name_search.group(4)
                    gz = name_search.group(5)
                                
                ## Lane files: need to merge by file_name: 33i_S5_L004_R1_001.fastq.gz
                elif (lane_search):
                    lane_id = name_search.group(2)
                    read_pair = name_search.group(3)
                    lane_file = name_search.group(4)
                    ext = name_search.group(5)
                    gz = name_search.group(6)
                else:
                    ## could exist or not
                    read_pair = name_search.group(2)
                    ext = name_search.group(3)
                    gz = name_search.group(4)
            else:
                ext = name_search.group(2)
                gz = name_search.group(3)
    
            ## populate dataframe
            name_frame.loc [len(name_frame)] = (path_files, dirN, name, name, name_len, 
                                            lane_id, read_pair, lane_file, ext, gz, "reads", os.path.basename(path_files))
    
        else:
            ## debug message
            if (Debug):
                print (colored("**DEBUG: sampleParser.get_fields **", 'yellow'))
                print (colored("*** ATTENTION: Sample did not match the possible parsing options...", 'yellow'))
                print (file_name)
            
            print (colored("*** ATTENTION: Sample (%s) did not match the possible parsing options..." %path_files, 'yellow'))

    return (name_frame)

####################
def select_samples (list_samples, samples_prefix, pair=True, exclude=False, Debug=False, lane=False, include_all=False):
    """
    Select samples
    
    Given a sample prefix (any or a given list), this function retrieves
    sample files from a list given. If exclude option provided, excludes 
    the files retrieved from the total.
    
    :param list_samples: List of absolute path for fastq files
    :param samples_prefix: List of prefix to search 
    :param pair: True/false for paired-end files
    :param exclude: True/false for exclude found files from the total
    :param Debug: True/false for debugging messages
    :param lane:
    :param include_all:
    
    :type list_samples: list
    :type samples_prefix: list
    :type pair: bool
    :type exclude: bool
    :type Debug: bool
    :type lane:
    :type include_all:
    
    :returns: something
    """
    #Get all files in the folder "path_to_samples"
    sample_list = pd.DataFrame(columns=('sample', 'file'))
    
    for names in samples_prefix:
        for path_fastq in list_samples:    
            fastq = os.path.basename(path_fastq)
            samplename_search = re.search(r"(%s)\_{0,1}(R1|1|R2|2){0,1}(.*){0,1}\.f.*q.*" % names, fastq)
            enter = ""
            if samplename_search:
                if (exclude): ## exclude==True
                    enter = False
                else: ## exclude==True
                    enter = True
            else:
                if (exclude): ## exclude==True
                    enter = True
                else: ## exclude==True
                    enter = False
                    
            if enter:
                if fastq.endswith('.gz') or fastq.endswith('fastq') or fastq.endswith('fq'):
                    sample_list.loc[len(sample_list)] = (names, path_fastq) 
                else:
                    ## debug message
                    if (Debug):
                        print (colored("**DEBUG: sampleParser.select_samples **", 'yellow'))
                        print (colored("** ERROR: %s is a file that is neither in fastq.gz or .fastq format, so it is not included" %path_fastq, 'yellow'))
                            
    ## discard duplicates if any
    non_duplicate_names = sample_list['sample'].to_list() #
    non_duplicate_names = list(set(non_duplicate_names))
    
    ## it might be a bug in exclude list.
    ## if sample X1 is provided to be excluded, we might be also excluding
    ## sample X12, sample X13, etc.
    ## TODO: check this

    ## debugging messages
    if Debug:
        print (colored("** DEBUG: select_samples",'yellow'))
        print ("non_duplicate_names:")
        print (non_duplicate_names)
    
    ## check they match with given input
    if (exclude): ## exclude==True
        if bool(set(samples_prefix).intersection(non_duplicate_names)):
            print(colored("** ERROR: Some non desired samples are included", 'red'))
    else: ## exclude==True
        non_duplicate_names = set(sorted(samples_prefix)).intersection(sorted(non_duplicate_names))

    ## get fields    
    tmp = sample_list[ sample_list['sample'].isin(non_duplicate_names) ]
    non_duplicate_samples = tmp['file'].to_list()
    
    ## debugging messages
    if Debug:
        print (colored("** DEBUG: select_samples",'yellow'))
        print ("non_duplicate_names:")
        print (non_duplicate_names)
        print ("samples_prefix")
        print (samples_prefix)
        print ("non_duplicate_samples")
        print (non_duplicate_samples)
        print ("tmp dataframe")
        functions.print_all_pandaDF(tmp)
    
    ## 
            
    ## get info
    name_frame_samples = get_fields(non_duplicate_samples, pair, Debug, include_all)    
    number_files = name_frame_samples.index.size
    total_samples = set(name_frame_samples['name'].to_list())
    
    ##
    if (lane):
        ## include lane tag within name
        name_frame_samples['name'] = name_frame_samples['name'] + '_' + name_frame_samples['lane']
        name_frame_samples['new_name'] = name_frame_samples['name']
            
    ## debugging messages
    if Debug:
        print (colored("** DEBUG: select_samples",'yellow'))
        print ("name_frame_samples:")
        print (name_frame_samples)
        print ("number_files:")
        print (number_files)
        print ("total_samples:")
        print (total_samples)
    
    ### get some stats
    if (number_files == 0):
        print (colored("\n**ERROR: No samples were retrieved. Check the input provided\n",'red'))
        exit()
    print (colored("\t" + str(number_files) + " files selected...", 'yellow'))
    print (colored("\t" + str(len(total_samples)) + " samples selected...", 'yellow'))
    if (pair):
        print (colored("\tPaired-end mode selected...", 'yellow'))
    else:
        print (colored("\tSingle end mode selected...", 'yellow'))
    
    ## return info
    return (name_frame_samples)

###############
def select_other_samples (project, list_samples, samples_prefix, mode, extensions, exclude=False, Debug=False):

    ## init dataframe
    name_columns = ("sample", "dirname", "name", "ext", "tag")

    ## initiate dataframe
    df_samples = pd.DataFrame(columns=name_columns)

    ## debug message
    if (Debug):
        print (colored("**DEBUG: samples_prefix **", 'yellow'))
        print (samples_prefix)
        print (colored("**DEBUG: mode **", 'yellow'))
        print (mode)
        print (colored("**DEBUG: extensions **", 'yellow'))
        print (extensions)

    #Get all files in the folder "path_to_samples"    
    sample_list = []
    for names in samples_prefix:
        for path_file in list_samples:    
            f = os.path.basename(path_file)
            dirN = os.path.dirname(path_file)
            #samplename_search = re.search(r"(%s).*" % names, f)
            samplename_search = re.search(r"(%s).*" % names, path_file)
            
            enter = ""
            if samplename_search:
                if (exclude): ## exclude==True
                    enter = False
                else: ## exclude==True
                    enter = True
            else:
                if (exclude): ## exclude==True
                    enter = True
                else: ## exclude==True
                    enter = False
                    
            if enter:
                
                ## project mode:
                if project:
                    if mode == 'annot':
                        #### /path/to/folder/annot/name.faa
                        for ext in extensions:
                            f_search = re.search(r".*\/%s\/(.*)\.%s$" %(mode, ext), path_file)
                            if f_search:
                                file_name = f_search.group(1) 
                                df_samples.loc[len(df_samples)] = [path_file, dirN, file_name, ext, mode]    

                    elif mode== 'assembly':
                        #### name_assembly.faa
                        for ext in extensions:
                            f_search = re.search(r"(.*)\_%s\.%s$" %(mode, ext), f)
                            if f_search:
                                file_name = f_search.group(1) 
                                df_samples.loc[len(df_samples)] = [path_file, dirN, file_name, ext, mode]    

                    elif mode== 'mash':
                        #### name.sig
                        for ext in extensions:
                            f_search = re.search(r".*\/%s\/(.*)\.%s$" %(mode, ext), path_file)
                            if f_search:
                                file_name = f_search.group(1) 
                                df_samples.loc[len(df_samples)] = [path_file, dirN, file_name, ext, mode]    

                    else:
                        for ext in extensions:
                            f_search = re.search(r".*\/(.*)\/%s\/(.*)\_summary\.%s$" %(mode, ext), path_file)
                            if f_search:
                                ### get information
                                if mode == 'profile':
                                    name = f_search.group(1)
                                    db_name = f_search.group(2).split('_')[-1]
                                    if not name.startswith('report'):
                                        df_samples.loc[len(df_samples)] = [path_file, dirN, name, db_name, mode]    
    
                                elif mode == 'ident':
                                    name = f_search.group(1)
                                    df_samples.loc[len(df_samples)] = [path_file, dirN, name, 'csv', mode]    

                ## detached mode
                else:
                    for ext in extensions:
                        if f.endswith(ext):
                            file_name, ext1 = os.path.splitext(f)
                            df_samples.loc[len(df_samples)] = [path_file, dirN, file_name, db_name, mode]    
                        
    ## debug message
    if (Debug):
        print (colored("**DEBUG: df_samples **", 'yellow'))
        print (df_samples)
    
    ##
    number_samples = df_samples.index.size
    if (number_samples == 0):
        print (colored("\n**ERROR: No samples were retrieved for this option. Continue processing...\n",'red'))
        return (df_samples)
    print (colored("\t" + str(number_samples) + " samples selected from the input provided...", 'yellow'))

    df_samples['new_name'] = df_samples['name']
    return (df_samples)
    

###############    
def gunzip_merge(outfile, list_files):
    """
    Merge gunzip files into final file
    
    :param outfile: String for output file
    :param list_files: List of files to merge
    
    :type outfile: string
    :type list_files: list
        
    """
    list_files = list(list_files)
    list_files.sort()
    print ("\tMerging files into: ", outfile)
    #print ("\tFiles: ", list_files)

    with open(outfile, 'wb') as wfp:
        for fn in list_files:
            with open(fn, 'rb') as rfp:
                shutil.copyfileobj(rfp, wfp)

    return()
    
###############    
def one_file_per_sample(dataFrame, outdir_dict, threads, outdir, Debug=False):
    """
    Merge fastq files from different lanes positions for each sample
    
    """
    ## merge sequencing files for sample, no matter of sector or lane generated.    
    list_samples = set(dataFrame['new_name'].tolist())
    print (colored("\t" + str(len(list_samples)) + " samples to be merged from the input provided...", 'yellow'))
    print ("+ Merging sequencing files for samples")

    ##
    sample_frame = dataFrame.groupby(["new_name", "read_pair"])
    
    ### get extension for files
    ext_list = dataFrame.ext.unique()
    gz_list = dataFrame.gz.unique()
    ext = ext_list[0] + gz_list[0] ## might generate a bug if several extension or some zip/unzip files provided

    # We can use a with statement to ensure threads are cleaned up promptly
    with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: ## need to do 1 by one as there is a problem with the working directory
        commandsSent = { executor.submit(gunzip_merge, 
                                        outdir_dict[name[0]] + '/' + name[0] + '_' + name[1] + '.' + ext, 
                                        sorted(set(cluster["sample"].tolist()))): name for name, cluster in sample_frame }
        for cmd2 in concurrent.futures.as_completed(commandsSent):
            details = commandsSent[cmd2]
            try:
                data = cmd2.result()
            except Exception as exc:
                print ('***ERROR:')
                print (cmd2)
                print('%r generated an exception: %s' % (details, exc))                
                            
    ## return output name merged generated in dataframe
    name_columns = ("new_name", "dirname", "read_pair", "new_file", "ext", "gz")
    name_frame = pd.DataFrame(columns=name_columns)

    ## print to a file
    timestamp = functions.create_human_timestamp()
    merge_details = outdir + '/' + timestamp + '_prep_mergeDetails.txt'
    merge_details_hd = open(merge_details, 'w')

    for name, cluster in sample_frame: ## loop over samples
        outfile = outdir_dict[name[0]] + '/' + name[0] + '_' + name[1] + ext
        
        merge_details_hd.write("####################\n")        
        merge_details_hd.write("Sample: " + name[0] + '\n')
        merge_details_hd.write("New name: " + name[0] + '\n')
        
        merge_details_hd.write("Read: " + name[1] + '\n')
        merge_details_hd.write("Files:\n")
        merge_details_hd.write(",".join(sorted(cluster["sample"].tolist())))
        merge_details_hd.write('\n')
        merge_details_hd.write("####################\n")        
        
        name_frame.loc[len(name_frame)] = (name[0], outdir_dict[name[0]], name[1], outfile, ext_list[0], gz_list[0])

    merge_details_hd.close()
    return(name_frame)

################################
def get_files(options, input_dir, mode, extension):
    """
    Parser function to get files
    
    Given an input dir and a mode in retrieves
    matching files with the extension desired.
    
    :param options: BacterialTyper options as parser.parse_args options.
    :param input_dir: Absolute path to input dir containing samples.
    :param mode: Options are: fastq, trim, annot, assembly.
    :param extension: List of possible extension to retrieve.
    
    :type options: parser.parse_args
    :type input_dir: string 
    :type mode: string 
    :type extension: list
    
    :returns: Pandas dataframe with sample and file information.
    """
    ## get list of input files
    files = []
    
    print ()
    functions.print_sepLine("-",50, False)
    print ('+ Getting files from input folder... ')
    print ('+ Mode: ', mode,'. Extension:', extension)
    if (options.project):
        ### a folder containing a project is provided
        if os.path.isdir(input_dir):
            print ('+ Input folder exists')
            ## get files in folder
            files = []
            for ext in extension:
                if mode == 'trim':
                    files_tmp = functions.get_fullpath_list(input_dir)
                    files = [s for s in files_tmp if ext in s]
                else:
                    files_tmp = functions.retrieve_matching_files(input_dir, ext)                
                    files.extend(files_tmp)
            
            files = set(files)
            
        else:
            ## input folder is not a dir, is it a batch input file?
            if (options.batch):
                if os.path.isfile(input_dir):
                    if (options.debug):
                        print (colored("\n**DEBUG: sampleParser.get_files input folder is a batch file, get full path **", 'yellow'))
                    dir_list = [line.rstrip('\n') for line in open(input_dir)]
                    for d in dir_list:
                        if os.path.exists(d):
                            print ('+ Folder (%s) exists' %d)
                            files.extend(functions.get_fullpath_list(d))
                        else:
                            ## input folder does not exist...
                            if (options.debug):
                                print (colored("\n**DEBUG: sampleParser.get_files batch option; input folder does not exists **", 'yellow'))
                                print (d)
                                print ("\n")
            else:
                ## input folder does not exist...
                if (options.debug):
                    print (colored("\n**DEBUG: sampleParser.get_files input folder does not exists **", 'yellow'))
                    print (input_dir)
                    print ("\n")
    
                print (colored('***ERROR: input folder does not exist or it is not readable', 'red'))
                exit()
    else:
        ### provide a single folder or a file with multiple paths (option batch)
        if (options.batch):
            if os.path.isfile(input_dir):
                dir_list = [line.rstrip('\n') for line in open(input_dir)]
                files=[]
                for d in dir_list:
                    if os.path.exists(d):
                        print ('+ Folder (%s) exists' %d)
                        files.extend(functions.get_fullpath_list(d))
                    else:
                        ## input folder does not exist...
                        if (options.debug):
                            print (colored("\n**DEBUG: sampleParser.get_files batch option; input folder does not exists **", 'yellow'))
                            print (d)
                            print ("\n")
                        
        else:
            if os.path.exists(input_dir):
                print ('+ Input folder exists')
                ## get files in folder
                files = functions.get_fullpath_list(input_dir)
            else:
                ## input folder does not exist...
                if (options.debug):
                    print (colored("\n**DEBUG: sampleParser.get_files input folder does not exists **", 'yellow'))
                    print (input_dir)
                    print ("\n")
    
                print (colored('***ERROR: input folder does not exist or it is not readable', 'red'))
                exit()

    ## get list of samples
    samples_names = []
    exclude=False

    if (options.in_sample):
        in_file = os.path.abspath(options.in_sample)
        samples_names = [line.rstrip('\n') for line in open(in_file)]
        print ('+ Retrieve selected samples to obtain from the list files available.')        
        exclude=False

        ## in sample list...
        if (options.debug):
            print (colored("\n**DEBUG: sampleParser.get_files include sample list **", 'yellow'))
            print (samples_names, '\n')


    elif (options.ex_sample):
        ex_file = os.path.abspath(options.ex_sample)
        samples_names = [line.rstrip('\n') for line in open(ex_file)]
        print ('+ Retrieve selected samples to exclude from the list files available.')        
        exclude=True

        ## in sample list...
        if (options.debug):
            print (colored("\n**DEBUG: sampleParser.get_files exclude sample list **", 'yellow'))
            print (samples_names, '\n')

    else:
        samples_names = ['.*']

    ## discard empty sample_names
    samples_names = list(filter(None, samples_names)) ## empty space

    ## discard some files obtain
    files = [s for s in files if 'single_copy_busco_sequences' not in s]
    files = [s for s in files if 'orphan' not in s]
    files = [s for s in files if 'augustus_output' not in s]
    files = [s for s in files if 'hmmer_output' not in s]
    files = [s for s in files if 'configs' not in s]
    files = [s for s in files if '00.0_0.cor.fastq.gz' not in s]
    files = [s for s in files if 'report_summary' not in s]
    files = list(filter(None, files)) ## empty space
    
    ## files list...
    if (options.debug):
        print (colored("\n**DEBUG: sampleParser.get_files files list to check **", 'yellow'))
        if (len(files) > 50):
            print ('DO NOT PRINT THIS LIST: It could be very large...')
            print ('Length:' + str(len(files)))    
        else:
            print (files, '\n')

    ## get information
    if mode in ['fastq', 'trim']:
        pd_samples_retrieved = select_samples(files, samples_names, options.pair, exclude, options.debug, options.include_lane, options.include_all)
    else:
        pd_samples_retrieved = select_other_samples(options.project, files, samples_names, mode, extension, exclude, options.debug)        
        
    return(pd_samples_retrieved)
