
import os  
import pandas as pd
import numpy as np
#from scipy import signal

import pickle

import os
import multiprocessing
import time
import fastqsplitter
from pathlib import Path
from ..util.cli import (
                        opt_barcode_length,opt_barcode_start,opt_parallel_demultiplexing,opt_clipped_demultiplexing,opt_mismatch_tolerence,opt_index_tolerence,opt_demulti_overwrite,opt_fasta,opt_library,opt_fastq1,opt_fastq2)

#secondary barcode names: secondary_signiture
params = [
    # Inputs
    opt_fasta,
    opt_fastq1,
    opt_fastq2,
    opt_library,
    opt_barcode_start,
    opt_barcode_length,

    #options
    opt_parallel_demultiplexing,
    opt_clipped_demultiplexing,
    opt_mismatch_tolerence,
    opt_index_tolerence,
    opt_demulti_overwrite


]


def makes_dict_from_fastq(fpath):
        #do I make a set of the cordinates? ideally that's the only part thats different..?


        f = open(fpath)
        lines = f.readlines()
        f.close()
        reads={}
        read_lines=[]
        for x in range(len(lines)):

            l=lines[x]
            read_lines.append(l)


            if x%4==0:
                spl=l.split()
                identifier_pre=spl[0]#5/6
                values=identifier_pre.split(":")
                identifier=(values[4]+":"+values[5]+":"+values[6])

            if x%4==3:
                reads[identifier]=read_lines
                read_lines=[]
        return reads       

def reverse_compliment(sequence):
    rev_seq=sequence[::-1]
    new_str=""
    for x in range(len(rev_seq)):
        if rev_seq[x]=="A":
            new_str+="T"
        if rev_seq[x]=="G":
            new_str+="C"
        if rev_seq[x]=="C":
            new_str+="G"
        if rev_seq[x]=="T":
            new_str+="A"
    return new_str


class Sequence_Obj():
    def __init__(self,
    sequence,
    name,
    fastq1_path,
    fastq2_path,
    workspace,
    paired=True,
    fwd_primer="",
    rev_primer="",
    secondary_signature="",secondary_signature_start="",secondary_signature_end="",
    rev_secondary_signature="",rev_secondary_signature_start="",rev_secondary_signature_end="",
    barcode_start=-1,barcode_end=-1,barcode="",
    rev_barcode="",rev_barcode_start="",rev_barcode_end=""):

        """
        This object will be hold the reference objects:
            including a dictionary of the ranges and the bv/mh associated with each of those
        """

        #input_df=pd.read_csv()
        self.demutliplex_fasta=""
        self.sequence=sequence
        self.sample_name=name
        self.paired=paired
        self.sequence=sequence
        self.fwd_primer=fwd_primer
        self.rev_primer=rev_primer
        self.paired=paired
        self.workspace=workspace


        self.fastq1_path=fastq1_path
        self.fastq2_path=fastq2_path
        self.fastq_paths={1:fastq1_path,2:fastq2_path}
        #barcode info
        self.barcode_start=barcode_start
        self.barcode_end=barcode_end
        self.barcode=barcode

        self.rev_barcode=rev_barcode
        self.rev_barcode_start=rev_barcode_start
        self.rev_barcode_end=rev_barcode_end

        #secondary signiture info
        self.secondary_signature=secondary_signature
        self.secondary_signature_start=secondary_signature_start
        self.secondary_signature_end=secondary_signature_end

        self.rev_secondary_signature=rev_secondary_signature
        self.rev_secondary_signature_start=rev_secondary_signature_start
        self.rev_secondary_signature_end=rev_secondary_signature_end



        #self.sequence_folder=workspace
        self.fastqs={1:"",2:""}

        self.fastq1_init_bc=set()
        self.fastq2_init_bc=set()
        
        clip_grepped_fq1={}
        clip_grepped_fq2={}

        #reads grepped by rev barcode search
        self.fastq1_rev_bc=set()
        self.fastq2_rev_bc=set()

        rev_clip_grepped_fq1={}
        rev_clip_grepped_fq2={}

        #ids of reads filtered out due to the secondary signiture
        self.fastq1_sec_filtered=set()
        self.fastq2_sec_filtered=set()

        #ids of reads filtered out due to reverse compliment of the secondary signiture
        self.fastq1_sec_rev_filtered=set()
        self.fastq2_sec_rev_filtered=set()

        #this is the total before secondary signiture filtering
        self.fastq1_unfiltered=set()
        self.fastq2_unfiltered=set()


        #this is post filtering or if no sec_sign filtering is going to happen
        self.fastq1_filtered=set()
        self.fastq2_filtered=set()


        #these too will only really need one since theyre the same reads either way
        #set of distnct reads from either fq1 or fq2
        self.fastq_union=set()

        #set of reads found in both fastq
        self.fastq_intersection=set()

        #organized by the amount of NT clipped from barcode and then searched for
        clip_grepped_fq1={}
        clip_grepped_fq2={}

        #grepped_done_file=

class super_fastq():

    def __init__(self,fpath:str,split_count=10,fastq_name:str=None,super_dir="")-> None:
        #print(super_dir)
        if(fastq_name==None):
            fq=fpath.split("/")[-1]
            dir_name=super_dir+fq.split("_R")[0]+"_"+fq.split("_R")[-1][0]+"_pickle_split/"
        else:
            dir_name=super_dir+fastq_name
        #dir_name+=super_dir
        os.makedirs(dir_name,exist_ok=True)
        try:

            fq_size=os.path.getsize(fpath)
        except:
            raise Exception("File could not be opened, please check if the path you gave is correct!")


        split_list=[]
        pickle_file_list=[]
        self.original_fastq=fpath
        self.id_to_pickle_dict={}
        if fq_size<100000:
            self.split_count=1
        else:
            self.split_count=split_count

        for x in range(self.split_count):
            sp=dir_name + "split_"+str(x)+".fastq"
            sp_pickle=dir_name + "split_"+str(x)+".p"
            self.id_to_pickle_dict[x]=dict()

            pickle_file_list.append(sp_pickle)
            split_list.append(sp)

        self.split_list=split_list
        self.pickle_file_list=pickle_file_list
        
        self.id_to_pickle_dict_pickle_file=dir_name+"read_ids_per_pickle.p"

        self.split_bool=False
        self.split_fastqs_present=False
        #self.full_id_set_pickle=dir_name+"read_ids_per_pickle.p"

    def check_exists(self):
        
        for p in self.pickle_file_list:
            path_v=Path(p)
            b=os.path.exists(path_v)
            if (b==False):
                return b
        return True

        
    def check_set(self):
        return set(make_dict_from_fasta(self.original_fastq))
    def fastq_to_dict(self,fpath):
        f = open(fpath)
        lines = f.readlines()
        f.close()
        reads={}
        read_lines=[]
        for x in range(len(lines)):

            l=lines[x]
            read_lines.append(l)


            if x%4==0:
                spl=l.split()
                identifier_pre=spl[0]#5/6
                values=identifier_pre.split(":")
                identifier=(values[4]+":"+values[5]+":"+values[6])

            if x%4==3:
                reads[identifier]=read_lines
                read_lines=[]
        return reads

    def split_fastq(self,delete_text_fastqs:bool,temp_delete_idSets_to_pickle_dict:bool):
        print("splitting: ",self.original_fastq)
        big_set=set()

        if self.check_exists():
            
            print("already exists")
            self.split_bool=True
            id_dict=pickle.load(open(self.id_to_pickle_dict_pickle_file,"rb"))

            for k in id_dict.keys():
                #print(k," : ",type(id_dict[k]))

                big_set=big_set.union(id_dict[k])
                #print(len(big_set))
            #print("big set: ",len(big_set))
            return big_set

        fastqsplitter.split_fastqs(self.original_fastq,self.split_list)
        

        for i in range(len(self.split_list)):
            #print(type(self.split_list[i]))
            temp_dict=self.fastq_to_dict(self.split_list[i])
            big_set=big_set.union(set(temp_dict))
            self.id_to_pickle_dict[i]=set(temp_dict.keys())
            pickle.dump(temp_dict,open(self.pickle_file_list[i],"wb"))

        if(delete_text_fastqs):

            for fq in self.split_list:
                os.remove(fq)
            self.split_fastqs_present=False

        else:

            self.split_fastqs_present=True
        self.split_bool=True
        
        """
        this set could be pretty big, could be worth temp_deleting
        """
        
        pickle.dump(self.id_to_pickle_dict,open(self.id_to_pickle_dict_pickle_file,"wb"))
        #self.id_to_pickle_dict.clear()

        #print("in split: ",big_set)
        return big_set
            
    
    """
    takes a dict of reads 
    this method is for efficently organizing the reads based on which pickle
    they are in so pickles only need to be opened once

    union dict a diction representing the sequence to its 

    they will write to the "directory_to_write_to" which is just a string 
    names file as construct name (key in union dict) +"_R"+fastq_id+".fastq

    also clears the union dict to save space

    """
    def super_write_fastqs(self,union_dict:dict,directoy_to_write_to:str,fastq_id:int,sequence_objects:dict,sample_name:str):

        """
        organizes reads into sets per k,
        based on which pickle the read is in 
        """
        sample_fq_dir=directoy_to_write_to+sample_name+"/"
        os.makedirs(sample_fq_dir,exist_ok=True)

        organized_joint_dict={}
        for k in union_dict.keys():
            #print(k)
            joint_set=union_dict[k]

            temp_dictionary_pickle_loc={}

            
            self.id_to_pickle_dict=pickle.load(open(self.id_to_pickle_dict_pickle_file,"rb"))
            #print(self.id_to_pickle_dict.keys())

            for i in range(len(self.pickle_file_list)):
                temp_dictionary_pickle_loc[i]=set()

            
            for ident in joint_set:

                for num in self.id_to_pickle_dict.keys(): 

                    if(ident in self.id_to_pickle_dict[num]):

                        temp_dictionary_pickle_loc[num].add(ident)

            organized_joint_dict[k]=temp_dictionary_pickle_loc
        
        #union_dict.clear()
        """
        now that the reads are organized we have to open each pickle file write fastqs 
        """
        #fqs=[]
        for k in sequence_objects.keys():
            
            if(len(union_dict[k])>1):
                fq=sample_fq_dir+k+"_R"+str(fastq_id)+".fastq"
                fq_buff=open(fq,"wt")
                fq_buff.close()

        for x in range(len(self.pickle_file_list)):

            p_file_object=open(self.pickle_file_list[x],"rb")
            pickled_fastq_dict=pickle.load(p_file_object)
            p_file_object.close()
            
            for k in organized_joint_dict.keys():

                if(len(union_dict[k])>1):

                    #print("writing filtered fastqs"+str(k))
                    fq=sample_fq_dir+k+"_R"+str(fastq_id)+".fastq"
                    #fqs.append(fq)

                    sequence_objects[k].fastqs[fastq_id]=fq

                    specific_reads= organized_joint_dict[k][x]

                    fq_buff=open(fq,"a")

                    for r in specific_reads:
                        lines=pickled_fastq_dict[r]
                        fq_buff.writelines(lines)
                    fq_buff.close()
        #fqs.sort()
    """
    this methods removes the pickles that contain split fastq information
    """
    def destroy_temp_data(self):
        for x in range(len(self.pickle_file_list)):
            os.remove(self.pickle_file_list[x])



        





"""
tolerence is added to both ends of search range

if tolerence==-1 then disregards indexes 
    this will increase computation time
no index ranges and high mismatch threshhold will result in extremely high comp time 
"""

def run_seqkit_grep_function(pattern:str,
                            search_start_ind:int,
                            search_end_index:int,
                            fastq_to_search:str,
                            fastq_to_write:str,
                            threads:int=20,
                            mismatch_threshhold:int=0,
                            append_bool:bool=False,
                            tolerance:int=0,
                            delete_fq:bool=False
                            ):
    """
    1 indexed? 
    
    """
    append_char=">>" if append_bool else ">"

    if(search_start_ind -tolerance<0):
        search_start_ind =0
    else:
        search_start_ind-=tolerance
    search_end_index+=tolerance
    if(search_start_ind<1):
        search_end_index+=abs(search_start_ind)+1
        search_start_ind=1
    cmd=f'seqkit --threads {threads} grep -s -R {search_start_ind}:{search_end_index+tolerance} -p "{pattern}" -m {mismatch_threshhold} -P {fastq_to_search} > {fastq_to_write}'
    print(cmd)#debug-
    return_code=os.system(cmd)#debug
    if (delete_fq):
        os.remove(fastq_to_write)
    return (set(makes_dict_from_fastq(fastq_to_write).keys()))








        
"""
simple fastq dict for retrieveing sequences from fastq
"""


def make_dict_from_fasta(fasta_path) -> dict:
    fa=open(fasta_path,"rt").readlines()
    temp_dict={}

    for i in range(0,len(fa),2):
        temp_dict[fa[i][1:].strip()]=fa[i+1].strip()
    
    return temp_dict
"""
input csv, library that represents each sequence to be dumultiplexed with many different coloumns 

workspace directory that demultiplexing is being done in 

fq1/fq2 path for big fastq which will be demultiplexed? maybe should be removed


this whole method could be replaced with a dataframe that organizes all of these attributes


"""
def make_sequence_objects_from_csv(input_csv,barcode_start,barcode_length,fasta,fastq1_path,fastq2_path,paired,workspace) -> dict:
    
    sequence_object_dict={}
    fasta_dict=make_dict_from_fasta(fasta)
    if (input_csv==""):

        for name in fasta_dict.keys():

            seq=fasta_dict[name]
            rev_seq=reverse_compliment(seq)
            bc=seq[barcode_start :barcode_start+barcode_length]

            rev_barcode=reverse_compliment(bc)
            rev_bc_start=rev_seq.index(rev_barcode)
            rev_bc_end=rev_bc_start+len(rev_barcode)
            

            sequence_object_dict[name]=Sequence_Obj(
                    sequence=fasta_dict[name],
                    fastq1_path=fastq1_path,
                    fastq2_path=fastq2_path,
                    name=name,
                    paired=paired,
                    barcode_start=barcode_start,
                    barcode_end=barcode_start+barcode_length,
                    barcode=bc,
                    rev_barcode=rev_barcode,
                    rev_barcode_start=rev_bc_start,
                    rev_barcode_end=rev_bc_end,
                    secondary_signature_start=-1,
                    secondary_signature_end=-1,
                    secondary_signature=-1,
                    rev_secondary_signature_start=-1,
                    rev_secondary_signature_end=-1,
                    rev_secondary_signature=-1,
                    workspace=workspace+name+"/"
                )

    else:
        df=pd.read_csv(input_csv)

        
        #fasta_dict=make_dict_from_fasta(fasta)

        #sequence_object_dict={}
        cols=set(df.columns)

        if(barcode_start==barcode_length) and ("barcode_start" not in cols):
            raise Exception("no barcode info given")
        
            
        for x in df.index:

            name=df.at[x,"reference"]
            seq=fasta_dict[name]
            rev_seq=reverse_compliment(seq)

            
            if(barcode_start==barcode_length):
                barcode_start=df.at[x,"barcode_start"]
                barcode_length=df.at[x,"barcode_length"]
                bc=seq[barcode_start :barcode_start+barcode_length]
            else:
                bc=seq[barcode_start:barcode_start+barcode_length]
            rev_barcode=reverse_compliment(bc)
            rev_bc_start=rev_seq.index(rev_barcode)
            rev_bc_end=rev_bc_start+len(rev_barcode)
            print("\nhere"+str(cols))
            if("secondary_signature_start" in cols ):
                secondary_sign_start=df.at[x,"secondary_signature_start"]
                secondary_sign_end=secondary_sign_start+df.at[x,"secondary_signature_length"]
                secondary_sign=fasta_dict[name][secondary_sign_start:secondary_sign_end]

                rev_sec_sign=reverse_compliment(secondary_sign)
                rev_sec_sign_start = rev_seq.index(rev_sec_sign)
                rev_sec_sign_end= rev_sec_sign_start +len(rev_sec_sign)


            else:
                secondary_sign_start = -1
                secondary_sign_end = -1
                secondary_sign = -1

                rev_sec_sign = -1
                rev_sec_sign_start = -1
                rev_sec_sign_end = -1

        

            

            sequence_object_dict[name]=Sequence_Obj(
                sequence=fasta_dict[name],
                fastq1_path=fastq1_path,
                fastq2_path=fastq2_path,
                name=name,
                paired=paired,
                barcode_start=barcode_start,
                barcode_end=barcode_start+len(bc),
                barcode=bc,
                rev_barcode=rev_barcode,
                rev_barcode_start=rev_bc_start,
                rev_barcode_end=rev_bc_end,
                secondary_signature_start=secondary_sign_start,
                secondary_signature_end=secondary_sign_end,
                secondary_signature=secondary_sign,
                rev_secondary_signature_start=rev_sec_sign_start,
                rev_secondary_signature_end=rev_sec_sign_end,
                rev_secondary_signature=rev_sec_sign,
                workspace=workspace+name+"/"
            )
    return sequence_object_dict
"""
seqkit grep is 1 indexed and inclusive of the final value of its range 



clipped int that represents how much 
fastq_id fastq 1 or fastq 2
"""

"""
runs grep and accepts a clipped argument and appends the set to the main dictionary

index tolerence can only apply to the initial but there are cases where that could false 
"""
def run_multi_greps(read_id_dict:dict,clipped:int,index_tolerence:int,delete_fastqs:bool,mismatches_allowed:int,pattern_type:str,pattern:str,pattern_start:int,pattern_end:int,fastq:str,seq_folder:str,front:bool=False):
    #debug#print("running multi_greps for ", pattern_type)
    new_reads_dict={}
    folder=seq_folder
    #file_name_start=
    new_reads_dict["init_"+pattern_type]=run_seqkit_grep_function(pattern=pattern,search_start_ind=pattern_start,search_end_index=pattern_end,fastq_to_search=fastq,fastq_to_write=seq_folder+"init_"+pattern_type+".fastq",mismatch_threshhold=mismatches_allowed,tolerance=index_tolerence)
    #if(len(pattern)-pattern_end)<clipped:
    append_these=[seq_folder+"init_"+pattern_type+".fastq"]
    clipped+=1
    #print("initial grep run")#debug
    if not front:
        for x in range(1,clipped):
            
            append_these.append(seq_folder+pattern_type+"_clipped_"+str(x)+".fastq")
            new_reads_dict[pattern_type+"_clipped_"+str(x)]=run_seqkit_grep_function(pattern=pattern[:len(pattern)-x],search_start_ind=pattern_start+x,search_end_index=pattern_end,fastq_to_search=fastq,fastq_to_write=seq_folder+pattern_type+"_clipped_"+str(x)+".fastq",mismatch_threshhold=0,tolerance=0)
    else:
        for x in range(1,clipped):
            append_these.append(seq_folder+pattern_type+"_clipped_"+str(x)+".fastq")
            new_reads_dict[pattern_type+"_clipped_"+str(x)]=run_seqkit_grep_function(pattern=pattern[x:],search_start_ind=pattern_start,search_end_index=pattern_end-x,fastq_to_search=fastq,fastq_to_write=seq_folder+pattern_type+"_clipped_"+str(x)+".fastq",mismatch_threshhold=0,tolerance=0)
    #deug#print("clipped run")
    #for k in new_reads_dict.keys():
    #keys=list(new_reads_dict.keys())

    for x in range(1,clipped):

        new_reads_dict[pattern_type+"_clipped_"+str(x)]-=new_reads_dict["init_"+pattern_type]
        for i in range(x+1,clipped):
            new_reads_dict[pattern_type+"_clipped_"+str(x)]-=new_reads_dict[pattern_type+"_clipped_"+str(x)]
    #debug#print("organizing reads")
    for k in new_reads_dict.keys():
        read_id_dict[k]=new_reads_dict[k]
    #debug#print("appending reads")
    cmd="cat "
    return_fastq=seq_folder+pattern_type+ "_appended.fastq"
    #print(append_these  )
    for x in range(len(append_these)):
        cmd+=(append_these[x] + " ")
    cmd+="> "+ seq_folder+pattern_type+ "_appended.fastq"
    
    os.system(cmd)
    #debug#print("appended!")
    return return_fastq

    
    
def append_files(files,new_file_name):
    cmd="cat "
    for x in range(len(files)):
        cmd+=(files[x] + " ")
    cmd+="> "+ new_file_name
    os.system(cmd)



def run_seqkit_grep(sequence_object:Sequence_Obj,clipped:int,rev_clipped:int,index_tolerence:int,delete_fastqs:bool,fastq_id:int,mismatches_allowed:int):
    #f=open(str(sequence_object.sample_name)+"_test.txt","wt")
    #f.write(str(vars(sequence_object)))
    fastq=sequence_object.fastq_paths[fastq_id]
    #makes folder for demultiplex
    os.makedirs(sequence_object.workspace,exist_ok=True)

    fastq_folder=sequence_object.workspace+"fq"+str(fastq_id)+"/"
    os.makedirs(fastq_folder,exist_ok=True)

    #fastqs
    
    fastq_unfiltered=fastq_folder + "unfiltered.fastq"


    fastq_sec_filtered=fastq_folder + "sec_fitlered_filtered.fastq"
    fastq_rev_sec_filtered=fastq_folder + "rev_sec_filtered.fastq"
    
    fastq_filtered=fastq_folder + "filtered_out.fastq"

    read_ids={}
    bc_fastq=run_multi_greps(read_id_dict=read_ids,clipped=clipped,index_tolerence=index_tolerence,delete_fastqs=delete_fastqs,mismatches_allowed=mismatches_allowed,pattern_type="barcode",pattern=sequence_object.barcode,pattern_start=sequence_object.barcode_start,pattern_end=sequence_object.barcode_end,fastq=fastq,seq_folder=fastq_folder)
    #debug#print("first_multi_run")
    rev_bc_fastq=run_multi_greps(read_id_dict=read_ids,clipped=0,index_tolerence=index_tolerence,delete_fastqs=delete_fastqs,mismatches_allowed=mismatches_allowed,pattern_type="rev_barcode",pattern=sequence_object.rev_barcode,pattern_start=sequence_object.rev_barcode_start,pattern_end=sequence_object.rev_barcode_end,fastq=fastq,seq_folder=fastq_folder)

    append_files([bc_fastq,rev_bc_fastq],fastq_unfiltered)
    read_ids["unfiltered"]=set(makes_dict_from_fastq(fastq_unfiltered).keys())

    
    complete_set=set()

    if(sequence_object.secondary_signature!=-1):
        v_threshold=0
        v_section=pattern=sequence_object.secondary_signature
        if(len(v_section)>20):
            v_threshold=4
        elif(len(v_section)<=20 and len(v_section)>15 ):
            v_threshold=3
        elif(len(v_section)<=15 and len(v_section)>=5 ):
            v_threshold=2
        elif(len(v_section)<5):
            v_threshold=1
    
        secondary_sign_mismatches=v_threshold#TODO hardcoded :(


        read_ids["sec_filter"]=run_seqkit_grep_function(pattern=sequence_object.secondary_signature,search_start_ind=sequence_object.secondary_signature_start,search_end_index=sequence_object.secondary_signature_end,fastq_to_search=fastq_unfiltered,fastq_to_write=fastq_sec_filtered,mismatch_threshhold=secondary_sign_mismatches,tolerance=2)
        read_ids["rev_sec_filter"]=run_seqkit_grep_function(pattern=sequence_object.rev_secondary_signature,search_start_ind=sequence_object.rev_secondary_signature_start,search_end_index=sequence_object.rev_secondary_signature_end,fastq_to_search=fastq_unfiltered,fastq_to_write=fastq_rev_sec_filtered,mismatch_threshhold=secondary_sign_mismatches,tolerance=2)
        complete_set=read_ids["sec_filter"].union(read_ids["rev_sec_filter"])
        read_ids["reads_lost_to_filter"]=read_ids["unfiltered"]-complete_set
        #complete_set=read_ids["sec_filter"].union(read_ids["rev_sec_filter"])
    else:
        complete_set=read_ids["unfiltered"]
    #here the sets need to be compared in order to identify which ids were unique to which search
    #TODO
    
    #print(vars(sequence_object))
    #complete_set=read_ids["sec_filter"].union(read_ids["rev_sec_filter"])

    complete_set_pickle=fastq_folder+"complete_set_of_reads.p"
    pickle.dump(complete_set,open(complete_set_pickle,"wb"))
    read_id_data_pickle=fastq_folder+"read_id_data.p"
    pickle.dump(read_ids,open(read_id_data_pickle,"wb"))
    #print(read_ids)

    open(sequence_object.workspace+"grepped.txt","wt").write("done")


"""
latest and greatest
"""
def check_done(sequence_folder:str) -> bool:
    #print("looking here: ",sequence_folder+"grepped.txt")

    comp_set_fq1="fq1/complete_set_of_reads.p"

    if  (os.path.exists(sequence_folder+"fq1/complete_set_of_reads.p"))\
    and (os.path.exists(sequence_folder+"fq2/complete_set_of_reads.p")) \
        \
    and (os.path.exists(sequence_folder+"fq1/read_id_data.p"))\
    and (os.path.exists(sequence_folder+"fq2/read_id_data.p")) \
        \
    and (os.path.exists(sequence_folder+"grepped.txt")):

        return True
    else:
        return False

def check_all_done(seq_objects:dict()):
    return_dict={}

    
    for k in seq_objects.keys():
        print("checking: "+k,end="\r")
        if not (check_done(seq_objects[k].workspace)):
            
            print("check done not done! ",check_done(seq_objects[k].workspace))
            return_dict[k]=seq_objects[k]
    return return_dict

def grep_both_fastq(sequence_object:Sequence_Obj,clipped:int,rev_clipped:int,index_tolerence:int,delete_fastqs:bool,mismatches_allowed:int):

    run_seqkit_grep(sequence_object=sequence_object, clipped=clipped, rev_clipped=rev_clipped, index_tolerence=index_tolerence, delete_fastqs=delete_fastqs, mismatches_allowed=mismatches_allowed,fastq_id=1)
    run_seqkit_grep(sequence_object=sequence_object, clipped=clipped, rev_clipped=rev_clipped, index_tolerence=index_tolerence, delete_fastqs=delete_fastqs, mismatches_allowed=mismatches_allowed,fastq_id=2)

def parallel_grepping(sequence_objects:dict,fwd_clips:int,rev_clips:int,index_tolerence:int,delete_fastq:bool,paired:bool=True,mismatches:int=0,threads=10,iteration:int=0,overwrite:bool=True):
    """
    runs grep in parallel 
    """
    itr_val=iteration
    print("iteration value:XXX ",itr_val)
    countx=0
    procs=[]
    seq_keys=list(sequence_objects.keys())
    seq_count=0
    THREADS=threads
    #print(seq_keys)
    run_all=True
    
    while(seq_count<len(seq_keys)+6):
        
        print(seq_count)
        procs=[]
            
        seq_index=seq_count
        print(range(seq_index,seq_index+THREADS))
        for i in range(seq_index,(seq_index+THREADS)):
            #seq_index+i
            #print(seq_index)
            if(i<len(seq_keys)):
                seq_keys[i]
                if(not overwrite):
                    previously_run=check_done(sequence_objects[seq_keys[i]].workspace)
                else:
                    previously_run=False
                #print(previously_run)
                if(previously_run and not overwrite):
                    print("grepped")
                else:
                    #(sequence_object:Sequence_Obj,clipped:int,rev_clipped:int,index_tolerence:int,delete_fastqs:bool,mismatches_allowed:int)
                    x=multiprocessing.Process(target=grep_both_fastq, args=(sequence_objects[seq_keys[i]],fwd_clips,rev_clips,index_tolerence,delete_fastq,mismatches))
                    x.start()
                    procs.append(x)
            #seq_index=+1
            else:
                #info#print("no more to run")
                pass

        for p in procs:
            p.join()


        #time.sleep(1000)
        start=time.time()
        seq_count+=THREADS
        itr_val+=1
    
    print("checking")
    not_done_dict=check_all_done(sequence_objects)
    #print(not_done_list)

    while len(not_done_dict) > 0 and itr_val >4:
        print("rip")
        itr_val=iteration
        #itr_val=iteration
        #not_done_dict=check_all_done(sequence_objects)
        parallel_grepping(sequence_objects=not_done_dict,fwd_clips=fwd_clips,rev_clips=rev_clips,index_tolerence=index_tolerence,delete_fastq=delete_fastq,iteration=itr_val+1,mismatches=mismatches)
    if(itr_val>4):
        print("could not finish some of the reads: ",list(not_done_dict.keys()))
    
        #warning could not finish one of the reads
    return not_done_dict    

def regular_grepping(sequence_objects:dict,fwd_clips:int,rev_clips:int,index_tolerence:int,delete_fastq:bool,paired:bool=True,mismatches:int=0,iteration:int=0,overwrite:bool=False):
    """
    runs grep in parallel 
    """
    print("regular grepping")
    itr_val=iteration

    for k in sequence_objects.keys():
            previously_run=check_done(sequence_objects[k].workspace)
            if(previously_run and (not overwrite)):
                print("grepped")
            else:
                grep_both_fastq(sequence_objects[k],fwd_clips,rev_clips,index_tolerence,delete_fastq,mismatches)



    not_done_dict=check_all_done(sequence_objects)


    while len(not_done_dict) > 0 and itr_val >4:
        itr_val=iteration
        """if itr_val>0:
            not_done_dict=check_all_done(sequence_objects)"""
        
        #not_done_dict=check_all_done(sequence_objects)
        regular_grepping(sequence_objects=not_done_dict,fwd_clips=0,rev_clips=0,index_tolerence=0,delete_fastq=False,iteration=itr_val+1,mismatches=mismatches)
    if(itr_val>4):
        print("could not finish one of the reads: ",list(not_done_dict.keys()))


"""
checks each sequence for a grepped.txt and returns true if found 
"""


def finds_multigrepped_reads(sequence_objects:dict,remove:bool=True,resolve:bool=False,print_multi_grep_dict:bool=True,demultiplex_workspace:str=None) -> dict:
    """
    filters reads based on weather or not they map to multiple constructs 
    returns a dictionary mapping each read to a list of the reads it mapped to
    """
    union_dictionary={}
    fq1_sets={}
    fq2_sets={}

    for k in sequence_objects.keys():


        #print(k)
        seq_object_fastq_folder=sequence_objects[k].workspace
        fq1_complete_set_pickle=seq_object_fastq_folder+"fq1/complete_set_of_reads.p"
        fq2_complete_set_pickle=seq_object_fastq_folder+"fq2/complete_set_of_reads.p"

        fq1_complete_set=pickle.load(open(fq1_complete_set_pickle,"rb"))
        fq2_complete_set=pickle.load(open(fq2_complete_set_pickle,"rb"))

        fq1_sets[k]=fq1_complete_set
        fq2_sets[k]=fq2_complete_set
        #joint set of ids from complete sets of fqs
        union_dictionary[k]=fq1_complete_set.union(fq2_complete_set)
        """
        now that all reads are collected into a big dictionary of sets 
        we can see which ones are mapped to multiple reads
        """
    used_reads={}

    for k in sequence_objects.keys():

        for read in union_dictionary[k]:

            if read not in used_reads.keys():
                used_reads[read]=[k]
            else:
                used_reads[read].append(k)
    
    multi_grep_dict={}

    
    for read in used_reads.keys():
        used_by_list=used_reads[read]


        if len(used_by_list)>1:
            temp_list=[]
            #multi_grep_dict[read]=used_by_list
            
            for name in used_by_list:

                if(read in fq1_sets[name] and read in fq2_sets[name]):
                    temp_list.append([0,0,1])
                elif(read in fq1_sets[name] and read not in fq2_sets[name]):
                    temp_list.append([1,0,0])
                elif(read not in fq1_sets[name] and read in fq2_sets[name]):
                    temp_list.append([0,1,0])
                else:
                    temp_list.append([0,0,0])

                if (remove):
                    union_dictionary[name].remove(read)
            multi_grep_dict[read]=[used_by_list,temp_list]

    if(print_multi_grep_dict):
        pickle.dump(multi_grep_dict,open(demultiplex_workspace+"multigrepped_reads.p","wb"))

    return union_dictionary

def resolve_or_analyze_multigrepped_reads(union_sets:dict,remove:bool=True,resolve:bool=False):
    pass
 

def create_report(sequence_objects:dict,fq1:str,fq2:str,working_directory:str,unioned_sets:dict):


    orginal_len={}
    mixed_total_dict={}

    fastqs={}
    dict_of_lists={}
    FQS=["fq1","fq2"]
    #basic_info=["construct_name_","final_count_","lost_to_multigrep_",]
    temp_read_data_pickle=sequence_objects[list(sequence_objects.keys())[0]].workspace+"fq1/read_id_data.p"
    read_data_keys=list(pickle.load(open(temp_read_data_pickle,"rb")).keys())


    for fq in FQS:
        if(fq=="fq1"):
            fastqs[fq]=fq1
        else:
            fastqs[fq]=fq2

        mixed_total_dict[fq]=set(makes_dict_from_fastq(fastqs[fq]))
        orginal_len[fq]=len(mixed_total_dict[fq])
        print("orginal len: ",orginal_len[fq])
        dict_of_lists[fq+"_construct_name_"]=[]
        dict_of_lists[fq+"_final_count_"]=[]
        dict_of_lists[fq+"_lost_to_multigrep_"]=[]
        for data_key in read_data_keys:
            dict_of_lists[fq+"_"+data_key]=[]


        

    for k in sequence_objects.keys():
        #sequence_objects
        seq_obj=sequence_objects[k]
        final_reads=unioned_sets[k]

        print("final_counts: ",len(final_reads))

        for fq in FQS:
            mixed_total_dict[fq]-=final_reads
            fq_complete_set=pickle.load(open(seq_obj.workspace+fq+"/complete_set_of_reads.p","rb"))
            fq_read_data=pickle.load(open(seq_obj.workspace+fq+"/read_id_data.p","rb"))
            
            dict_of_lists[fq+"_construct_name_"].append(k)
            dict_of_lists[fq+"_final_count_"].append(len(final_reads))
            dict_of_lists[fq+"_lost_to_multigrep_"].append(len(fq_complete_set)-len(final_reads))

            for data_key in read_data_keys:
                dict_of_lists[fq+"_"+data_key].append(len(fq_read_data[data_key]))
    df=pd.DataFrame()

    for k in dict_of_lists.keys():
        df[k]=dict_of_lists[k]
    print("len: mixed: ",len(mixed_total_dict[FQS[0]]))
    print("len: mixed: ",len(mixed_total_dict[FQS[0]])," / ",orginal_len[FQS[0]]," = ",len(mixed_total_dict[FQS[0]])/orginal_len[FQS[0]])
    len_col=[" "]*len(sequence_objects.keys())
    len_col[0]=len(mixed_total_dict[FQS[0]])
    df["percent_reads_used"]=[(1-(len(mixed_total_dict[FQS[0]])/orginal_len[FQS[0]]))*100]*len(sequence_objects.keys())
    df["total_read_count"]=len_col

    print(working_directory+"demultiplex_info.csv")
    df.to_csv(working_directory+"demultiplex_info.csv",index=False)



"""
split is default to 10. disregarding extremes, the higher the split the lighter the memeory load
library csv 
    each construct must have a secondary signiture start index and len in order to process, 
    barcode given in main arguements 
"""
def demultiplex_run(library_csv,demulti_workspace,report_folder,mixed_fastq1,mixed_fastq2,fasta,barcode_start=0,barcode_length=0,split:int=10,clipped:int=0,rev_clipped:int=0,index_tolerance:int=0,parallel:bool=False,mismatch_tolerence:int=0,overwrite:bool=False):

    if(type(mixed_fastq1)==type(("x","x"))):
        mixed_fastq1=mixed_fastq1[0]
        mixed_fastq2=mixed_fastq2[0]
    sample_name=mixed_fastq1.split("_R1")[0].split("/")[-1]

    """
    makes dictionary of sequence objects
    """
    #print("demulti_workspace: ",demulti_workspace)
    temp_ws=demulti_workspace+"/"+sample_name+"_demultiplex_folders_and_files/"
    #final_sample_folder=temp_ws+"sample_fqs/"

    #print(temp_ws)

    print(repr(temp_ws))
    os.makedirs(temp_ws,exist_ok=True)
    #print("tempworkspace: ",temp_ws)
    """
    all the little stuff gets stored per sequence here, at least temporarily 
    """
    seq_data_folder=temp_ws+"sequence_data/"
    
    os.makedirs(seq_data_folder,exist_ok=True)

    sequence_objects=make_sequence_objects_from_csv(
        input_csv=library_csv,
        barcode_start=barcode_start,
        barcode_length=barcode_length,
        fasta=fasta,
        fastq1_path=mixed_fastq1,
        fastq2_path=mixed_fastq2,
        paired=True,
        workspace=seq_data_folder)

    #print("\nhere:\n"+(sequence_objects))
    #print("workspace: ",vars(sequence_objects["3042-O-flank_1=hp1-DB"]))
    #demultiplex_workspace=demulti_workspace#"demultiplexed_sequences/"
    """
    makes a super fastq for memory efficent access to fastq reads

    checks values of ids to verify that the two fastq provided contain matching paired end reads
    """  
    super_fq1=super_fastq(mixed_fastq1,split,super_dir=temp_ws)
    super_set1=super_fq1.split_fastq(True,False)

    super_fq2=super_fastq(mixed_fastq2,split,super_dir=temp_ws)
    super_set2=super_fq2.split_fastq(True,False)
    #print(super_set2)
    c=len(super_set1.difference(super_set2))

    super_set1.clear()
    super_set2.clear()

    if c!=0:
        raise Exception("Fastq ids do not match, please verify that the file")


    

    """
    runs grep in parallel 
    """
    if(parallel):
        parallel_grepping(sequence_objects=sequence_objects,fwd_clips=clipped,rev_clips=rev_clipped,index_tolerence=index_tolerance,delete_fastq=False,mismatches=mismatch_tolerence,overwrite=overwrite)
    else:
        regular_grepping(sequence_objects=sequence_objects,fwd_clips=clipped,rev_clips=rev_clipped,index_tolerence=index_tolerance,delete_fastq=False,mismatches=mismatch_tolerence,overwrite=overwrite)
    #print(cum+plus)



    """
    filters reads based on weather or not they map to multiple constructs 
    default is to delete multigrepped reads

    """
    unioned_sets_dictionary=finds_multigrepped_reads(sequence_objects=sequence_objects,demultiplex_workspace=seq_data_folder)
    #debug#print(unioned_sets_dictionary.keys())

    #multigrepped_read_analysis()

    """
    organizes reads for writing
    uses super_fastq object to write fastqs in efficent way
    """

    fq1_paths=super_fq1.super_write_fastqs(unioned_sets_dictionary,report_folder,1,sequence_objects,sample_name=sample_name)
    #super_fq1.destroy_temp_data()

    fq2_paths=super_fq2.super_write_fastqs(unioned_sets_dictionary,report_folder,2,sequence_objects,sample_name=sample_name)
    #super_fq2.destroy_temp_data()

    """
    makes report on what amount of reads were found in which stage
    """
    #sequence_objects:dict,fq1:str,fq2:str,working_directory:str,unioned_sets:dict)
    print("creating report!!!")
    create_report(sequence_objects,mixed_fastq1,mixed_fastq2,report_folder,unioned_sets_dictionary)


    return (),(),(report_folder+sample_name+"/",)
