from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
import sys
import logging
import csv
import os
import subprocess
import multiprocessing
import time
import shutil
import gzip
import edlib
import pandas as pd
import json
import platform
using_distro = False
try:
    import distro
    using_distro = True
except ImportError:
    pass

from builtins import range
from Bio import SeqIO
from Bio.SeqIO.QualityIO import FastqGeneralIterator
from natsort import natsorted
from amptk.__version__ import __version__
try:
    from urllib.request import urlopen
except ImportError:
    from urllib2 import urlopen

parentdir = os.path.join(os.path.dirname(__file__))

ASCII = {'!':'0','"':'1','#':'2','$':'3','%':'4','&':'5',
         "'":'6','(':'7',')':'8','*':'9','+':'10',',':'11',
         '-':'12','.':'13','/':'14','0':'15','1':'16','2':'17',
         '3':'18','4':'19','5':'20','6':'21','7':'22','8':'23',
         '9':'24',':':'25',';':'26','<':'27','=':'28','>':'29',
         '?':'30','@':'31','A':'32','B':'33','C':'34','D':'35',
         'E':'36','F':'37','G':'38','H':'39','I':'40','J':'41',
         'K':'42','L':'43','M':'44','N':'45','O':'46','P':'47',
         'Q':'48','R':'49','S':'50'}

with open(os.path.join(os.path.dirname(__file__), 'primers.json')) as infile:
    primer_db = json.load(infile)


degenNuc = [("R", "A"), ("R", "G"),
            ("M", "A"), ("M", "C"),
            ("W", "A"), ("W", "T"),
            ("S", "C"), ("S", "G"),
            ("Y", "C"), ("Y", "T"),
            ("K", "G"), ("K", "T"),
            ("V", "A"), ("V", "C"), ("V", "G"),
            ("H", "A"), ("H", "C"), ("H", "T"),
            ("D", "A"), ("D", "G"), ("D", "T"),
            ("B", "C"), ("B", "G"), ("B", "T"),
            ("N", "G"), ("N", "A"), ("N", "T"), ("N", "C"),
            ("X", "G"), ("X", "A"), ("X", "T"), ("X", "C")]

degenNucSimple = [("R", "A"), ("R", "G"),
            ("M", "A"), ("M", "C"),
            ("W", "A"), ("W", "T"),
            ("S", "C"), ("S", "G"),
            ("Y", "C"), ("Y", "T"),
            ("K", "G"), ("K", "T"),
            ("V", "A"), ("V", "C"), ("V", "G"),
            ("H", "A"), ("H", "C"), ("H", "T"),
            ("D", "A"), ("D", "G"), ("D", "T"),
            ("B", "C"), ("B", "G"), ("B", "T")]

class colr(object):
    GRN = '\033[92m'
    END = '\033[0m'
    WARN = '\033[93m'

#setup functions
def download(url, name):
    file_name = name
    u = urlopen(url)
    f = open(file_name, 'wb')
    meta = u.info()
    file_size = 0
    for x in meta.items():
        if x[0].lower() == 'content-length':
            file_size = int(x[1])
    print("Downloading: {0} Bytes: {1}".format(url, file_size))
    file_size_dl = 0
    block_sz = 8192
    while True:
        buffer = u.read(block_sz)
        if not buffer:
            break
        file_size_dl += len(buffer)
        f.write(buffer)
        p = float(file_size_dl) / file_size
        status = r"{0}  [{1:.2%}]".format(file_size_dl, p)
        status = status + chr(8)*(len(status)+1)
        sys.stdout.write(status)
    f.close()

#functions for system checks, etc
def git_version():
    def _minimal_ext_cmd(cmd):
        # construct minimal environment
        out = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = open(os.devnull, 'w'), cwd=parentdir).communicate()[0]
        return out
    try:
        out = _minimal_ext_cmd(['git', 'rev-parse', '--short', 'HEAD'])
        GIT_REVISION = out.strip().decode('ascii')
    except OSError:
        GIT_REVISION = False
    return GIT_REVISION

def getSize(filename):
    st = os.stat(filename)
    return st.st_size

def execute(cmd):
    DEVNULL = open(os.devnull, 'w')
    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             universal_newlines=True, stderr=DEVNULL)
    for stdout_line in iter(popen.stdout.readline, ""):
        yield stdout_line
    popen.stdout.close()
    return_code = popen.wait()
    if return_code:
        raise subprocess.CalledProcessError(return_code, cmd)


def minimap_otutable(otus, fastq, output, method='ont', min_mapq=1, cpus=1,
                     min_pident=0.90):
    # function to map reads with minimap2 and create OTU table
    # requires barcodelable= or barcode= or sample= in fastq header
    if method == 'pacbio':
        cmd = ['minimap2', '-x', 'map-pb', '-c', '-t', str(cpus),
               '--secondary=no', otus, fastq]
    else:
        cmd = ['minimap2', '-x', 'map-ont', '-c', '-t', str(cpus),
               '--secondary=no', otus, fastq]
    tmpOut = 'minimap2_{}.paf'.format(os.getpid())
    runSubprocess2(cmd, log, tmpOut)
    Results = {}
    with open(tmpOut, 'r') as infile:
        for line in infile:
            line = line.rstrip()
            qname, qlen, qstart, qend, strand, tname, tlen, tstart, tend, match, alnlen, mapq = line.split('\t')[:12]
            theRest = line.split('\t')[12:]
            pident = 0.0
            barcodelabel = None
            if int(mapq) < min_mapq:
                continue
            if ';' not in qname:
                continue
            for x in qname.split(';'):
                if x.startswith('barcodelabel='):
                    barcodelabel = x.replace('barcodelabel=', '')
                    break
                elif x.startswith('barcode='):
                    barcodelabel = x.replace('barcode=', '')
                    break
                elif x.startswith('sample='):
                    barcodelabel = x.replace('sample=', '')
                    break
            if not barcodelabel:
                continue
            for y in theRest:
                if y.startswith('de:f:'):
                    pident = 1 - float(y.replace('de:f:', ''))
            if pident < min_pident:
                continue
            #print(tname, barcodelabel, str(pident))
            if barcodelabel not in Results:
                Results[barcodelabel] = {tname: 1}
            else:
                if tname not in Results[barcodelabel]:
                    Results[barcodelabel][tname] = 1
                else:
                    Results[barcodelabel][tname] += 1
    df = pd.DataFrame(Results)
    df.index.rename('#OTU ID', inplace=True)
    df.fillna(0, inplace=True)
    df = df.astype(int)
    df.to_csv(output, sep='\t')
    os.remove(tmpOut)


def checkfile(input):
    if os.path.isfile(input):
        filesize = getSize(input)
        if int(filesize) < 1:
            return False
        else:
            return True
    elif os.path.islink(input):
        return True
    else:
        return False

def SafeRemove(input):
    if os.path.isdir(input):
        shutil.rmtree(input)
    elif os.path.isfile(input):
        os.remove(input)
    else:
        return

def number_present(s):
    return any(i.isdigit() for i in s)

def get_version():
    if git_version():
        version = __version__+'-'+git_version
    else:
        version = __version__
    return version

def get_usearch_version(usearch):
    try:
        version = subprocess.Popen([usearch, '-version'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
    except OSError:
        log.error("%s not found in your PATH, exiting." % usearch)
        sys.exit(1)
    vers = version[0].decode('utf-8').replace('usearch v', '')
    vers2 = vers.split('_')[0]
    return vers2

def get_vsearch_version():
    version = subprocess.Popen(['vsearch', '--version'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
    for v in version:
        v = v.decode('utf-8')
        if v.startswith('vsearch'):
            vers = v.replace('vsearch v', '')
            vers2 = vers.split('_')[0]
            return vers2

def versiontuple(v):
    return tuple(map(int, (v.split("."))))

def gvc(input, check):
    if versiontuple(input) >= versiontuple(check):
        return True
    else:
        return False

def versionDependencyChecks(usearch, method='vsearch'):
    #to run amptk need usearch > 9.0.2132 and vsearch > 2.2.0
    usearch_pass = '9.0.2132'
    vsearch_pass = '2.2.0'
    amptk_version = get_version()
    if method == 'usearch':
        usearch_version = get_usearch_version(usearch)
        if not gvc(usearch_version, usearch_pass):
            log.error("USEARCH v%s detected, needs to be atleast v%s" % (usearch_version, usearch_pass))
            sys.exit(1)
    vsearch_version = get_vsearch_version()
    if not gvc(vsearch_version, vsearch_pass):
        log.error("VSEARCH v%s detected, needs to be atleast v%s" % (vsearch_version, vsearch_pass))
        sys.exit(1)
    if method == 'usearch':
        log.info("AMPtk v%s, USEARCH v%s, VSEARCH v%s" % (amptk_version, usearch_version, vsearch_version))
    else:
        log.info("AMPtk v%s, VSEARCH v%s" % (amptk_version, vsearch_version))

def checkusearch10(usearch):
    #to run amptk need usearch > 10.0.2132 and vsearch > 2.2.0
    amptk_version = get_version()
    usearch_version = get_usearch_version(usearch)
    vsearch_version = get_vsearch_version()
    usearch_pass = '10.0.240'
    vsearch_pass = '2.2.0'
    if not gvc(usearch_version, usearch_pass):
        log.error("USEARCH v%s detected, needs to be atleast v%s" % (usearch_version, usearch_pass))
        sys.exit(1)
    if not gvc(vsearch_version, vsearch_pass):
        log.error("VSEARCH v%s detected, needs to be atleast v%s" % (vsearch_version, vsearch_pass))
        sys.exit(1)
    #log.info("AMPtk v%s, USEARCH v%s, VSEARCH v%s" % (amptk_version, usearch_version, vsearch_version))

def checkRversion():
    #need to have R version > 3.2
    cmd = ['Rscript', '--vanilla', os.path.join(parentdir, 'check_version.R')]
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    versions = stdout.decode('utf-8').replace(' \n', '')
    Rvers = versions.split(',')[0]
    dada2 = versions.split(',')[1]
    phyloseq = versions.split(',')[2]
    lulu = versions.split(',')[3]
    return (Rvers, dada2, phyloseq, lulu)

def checkfastqsize(input):
    filesize = os.path.getsize(input)
    return filesize

def getCPUS():
    cores = multiprocessing.cpu_count()
    return cores

def MemoryCheck():
    import psutil
    mem = psutil.virtual_memory()
    RAM = int(mem.total)
    return round(RAM / 1024000000)

def systemOS():
    if sys.platform == 'darwin':
        system_os = 'MacOSX '+ platform.mac_ver()[0]
    elif sys.platform == 'linux':
        linux_version = ''
        if using_distro:
            linux_version = distro.linux_distribution()
        else:
            linux_version = platform.linux_distribution()
        system_os = linux_version[0]+ ' '+linux_version[1]
    else:
        system_os = sys.platform
    return system_os

def SystemInfo():
    system_os = systemOS()
    python_vers = str(sys.version_info[0])+'.'+str(sys.version_info[1])+'.'+str(sys.version_info[2])
    log.info("OS: %s, %i cores, ~ %i GB RAM. Python: %s" % (system_os, multiprocessing.cpu_count(), MemoryCheck(), python_vers))
    #print python modules to logfile
    mod_versions()

def mod_versions():
    import pkg_resources
    modules = ['numpy', 'pandas', 'matplotlib', 'psutil', 'natsort', 'biopython', 'edlib', 'biom-format']
    results = []
    for x in modules:
        try:
            vers = pkg_resources.get_distribution(x).version
            hit = "%s v%s" % (x, vers)
        except pkg_resources.DistributionNotFound:
            hit = "%s NOT installed!" % x
        results.append(hit)
        if x == 'edlib':
            if '.post' in vers:
                vers = vers.split('.post')[0]
            if not gvc(vers, '1.2.1'):
                log.error("Edlib v%s detected, at least v1.2.1 required for degenerate nucleotide search, please upgrade.  e.g. pip install -U edlib or conda install edlib" % vers)
                sys.exit(1)
    log.debug("Python Modules: %s" % ', '.join(results))


class gzopen(object):
    def __init__(self, fname):
        f = open(fname)
        # Read magic number (the first 2 bytes) and rewind.
        magic_number = f.read(2)
        f.seek(0)
        # Encapsulated 'self.f' is a file or a GzipFile.
        if magic_number == '\x1f\x8b':
            self.f = gzip.GzipFile(fileobj=f)
        else:
            self.f = f
    # Define '__enter__' and '__exit__' to use in
    # 'with' blocks. Always close the file and the
    # GzipFile if applicable.
    def __enter__(self):
        return self
    def __exit__(self, type, value, traceback):
        try:
            self.f.fileobj.close()
        except AttributeError:
            pass
        finally:
            self.f.close()
    # Reproduce the interface of an open file
    # by encapsulation.
    def __getattr__(self, name):
        return getattr(self.f, name)
    def __iter__(self):
        return iter(self.f)
    def __next__(self):
        return next(self.f)

def Funzip(input, output, cpus):
    '''
    function to unzip as fast as it can, pigz -> bgzip -> gzip
    '''
    if which('pigz'):
        cmd = ['pigz', '--decompress', '-c', '-p', str(cpus), input]
    else:
        cmd = ['gzip', '--decompress', '-c', input]
    try:
        runSubprocess2(cmd, log, output)
    except NameError:
        with open(output, 'w') as outfile:
            subprocess.call(cmd, stdout=outfile)

def Fzip(input, output, cpus):
    '''
    function to zip as fast as it can, pigz -> bgzip -> gzip
    '''
    if which('pigz'):
        cmd = ['pigz', '-c', '-p', str(cpus), input]
    else:
        cmd = ['gzip', '-c', input]
    try:
        runSubprocess2(cmd, log, output)
    except NameError:
        with open(output, 'w') as outfile:
            subprocess.call(cmd, stdout=outfile)

def Fzip_inplace(input):
    '''
    function to zip as fast as it can, pigz -> bgzip -> gzip
    '''
    cpus = multiprocessing.cpu_count()
    if which('pigz'):
        cmd = ['pigz', '-f', '-p', str(cpus), input]
    else:
        cmd = ['gzip', '-f', input]
    try:
        runSubprocess(cmd, log)
    except NameError:
        subprocess.call(cmd)

def fastalen2dict(input):
    Lengths = {}
    with open(input, 'r') as infile:
        for rec in SeqIO.parse(infile, 'fasta'):
            if not rec.id in Lengths:
                Lengths[rec.id] = len(rec.seq)
    return Lengths

def myround(x, base=10):
    return int(base * round(float(x)/base))

def GuessRL(input):
    #read first 50 records, get length then exit
    lengths = []
    for title, seq, qual in FastqGeneralIterator(gzopen(input)):
        if len(lengths) < 50:
            lengths.append(len(seq))
        else:
            break
    return myround(max(set(lengths)))

def countfasta(input):
    count = 0
    with open(input, 'r') as f:
        for line in f:
            if line.startswith(">"):
                count += 1
    return count

def countfastq(input):
    lines = sum(1 for line in gzopen(input))
    count = int(lines) // 4
    return count

def line_count(fname):
    with open(fname) as f:
        i = -1
        for i, l in enumerate(f):
            pass
    return i + 1

def softwrap(string, every=80):
    lines = []
    for i in range(0, len(string), every):
        lines.append(string[i:i+every])
    return '\n'.join(lines)

def line_count2(fname):
    count = 0
    with open(fname, 'r') as f:
        for line in f:
            if not '*' in line:
                count += 1
    return count

def getreadlength(input):
    with gzopen(input) as fp:
        for i, line in enumerate(fp):
            if i == 1:
                read_length = len(line) - 1 #offset to switch to 1 based counts
            elif i > 2:
                break
    return read_length

def runSubprocess(cmd, logfile):
    logfile.debug(' '.join(cmd))
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    if stdout:
        logfile.debug(stdout.decode("utf-8"))
    if stderr:
        logfile.debug(stderr.decode("utf-8"))

def runSubprocess2(cmd, logfile, output):
    #function where output of cmd is STDOUT, capture STDERR in logfile
    logfile.debug(' '.join(cmd))
    with open(output, 'w') as out:
        proc = subprocess.Popen(cmd, stdout=out, stderr=subprocess.PIPE)
    stderr = proc.communicate()
    if stderr:
        if stderr[0] != None:
            logfile.debug(stderr.decode("utf-8"))

def runSubprocess3(cmd, logfile, folder, output):
    #function where output of cmd is STDOUT, capture STDERR in logfile
    logfile.debug(' '.join(cmd))
    with open(output, 'w') as out:
        proc = subprocess.Popen(cmd, stdout=out, stderr=out, cwd=folder)
    stderr = proc.communicate()
    if stderr:
        if stderr[0] != None:
            logfile.debug(stderr.decode("utf-8"))

def runSubprocess4(cmd, logfile, logfile2):
    #function where cmd is issued in logfile, and log captured in logfile 2
    logfile.debug(' '.join(cmd))
    with open(logfile2, 'w') as out:
        proc = subprocess.Popen(cmd, stdout=out, stderr=out)
    stderr = proc.communicate()
    if stderr:
        if stderr[0] != None:
            logfile.debug(stderr.decode("utf-8"))

def runSubprocess5(cmd):
    #function where no logfile and stdout/stderr to fnull
    FNULL = open(os.devnull, 'w')
    #print(' '.join(cmd))
    subprocess.call(cmd, stdout=FNULL, stderr=FNULL)

def getSize(filename):
    st = os.stat(filename)
    return st.st_size

def check_valid_file(input):
    if os.path.isfile(input):
        filesize = getSize(input)
        if int(filesize) < 1:
            return False
        else:
            return True
    else:
        return False

def bam2fastq(input, output):
    from . import pybam
    with open(output, 'w') as fastqout:
        with open(input, 'r') as bamin:
            for title, seq, qual in pybam.read(bamin,['sam_qname', 'sam_seq','sam_qual']):
                fastqout.write("@%s\n%s\n+\n%s\n" % (title, seq, qual))

def scan_linepos(path):
    """return a list of seek offsets of the beginning of each line"""
    linepos = []
    offset = 0
    with gzopen(path) as inf:
        # WARNING: CPython 2.7 file.tell() is not accurate on file.next()
        for line in inf:
            linepos.append(offset)
            offset += len(line)
    return linepos

def return_lines(path, linepos, nstart, nstop):
    """return nsamp lines from path where line offsets are in linepos"""
    offsets = linepos[int(nstart):int(nstop)]
    lines = []
    with gzopen(path) as inf:
        for offset in offsets:
            inf.seek(offset)
            lines.append(inf.readline())
    return lines

def split_fastq(input, numseqs, outputdir, chunks):
    #get number of sequences and then number of sequences in each chunk
    numlines = numseqs*4
    n = numlines // chunks
    #make sure n is divisible by 4 (number of lines in fastq)
    if ( n % 4 ) != 0:
        n = ((n // 4) + 1) * 4
    splits = []
    count = 0
    for i in range(chunks):
        start = count
        end = count+n
        if end > numlines:
            end = numlines
        splits.append((start, end))
        count += n
    #get line positions from file
    linepos = scan_linepos(input)
    #make sure output directory exists
    if not os.path.isdir(outputdir):
        os.makedirs(outputdir)
    #loop through the positions and write output
    for i, x in enumerate(splits):
        num = i+1
        with open(os.path.join(outputdir, 'chunk_'+str(num)+'.fq'), 'w') as output:
            lines = return_lines(input, linepos, x[0], x[1])
            output.write('%s' % ''.join(lines))

def split_fastqPE(R1, R2, numseqs, outputdir, chunks):
    #get number of sequences and then number of sequences in each chunk
    numlines = numseqs*4
    n = numlines // chunks
    #make sure n is divisible by 4 (number of lines in fastq)
    if ( n % 4 ) != 0:
        n = ((n // 4) + 1) * 4
    splits = []
    count = 0
    for i in range(chunks):
        start = count
        end = count+n
        if end > numlines:
            end = numlines
        splits.append((start, end))
        count += n
    #get line positions from file
    linepos = scan_linepos(R1)
    #make sure output directory exists
    if not os.path.isdir(outputdir):
        os.makedirs(outputdir)
    #loop through the positions and write output
    for i, x in enumerate(splits):
        num = i+1
        with open(os.path.join(outputdir, 'chunk_'+str(num)+'_R1.fq'), 'w') as output1:
            with open(os.path.join(outputdir, 'chunk_'+str(num)+'_R2.fq'), 'w') as output2:
                lines1 = return_lines(R1, linepos, x[0], x[1])
                output1.write('%s' % ''.join(lines1))
                lines2 = return_lines(R2, linepos, x[0], x[1])
                output2.write('%s' % ''.join(lines2))

def split_fastqPEandI(R1, R2, I1, numseqs, outputdir, chunks):
    #get number of sequences and then number of sequences in each chunk
    numlines = numseqs*4
    n = numlines // chunks
    #make sure n is divisible by 4 (number of lines in fastq)
    if ( n % 4 ) != 0:
        n = ((n // 4) + 1) * 4
    splits = []
    count = 0
    for i in range(chunks):
        start = count
        end = count+n
        if end > numlines:
            end = numlines
        splits.append((start, end))
        count += n
    #get line positions from file
    linepos = scan_linepos(R1)
    linepos2 = scan_linepos(I1)
    #make sure output directory exists
    if not os.path.isdir(outputdir):
        os.makedirs(outputdir)
    #loop through the positions and write output
    for i, x in enumerate(splits):
        num = i+1
        with open(os.path.join(outputdir, 'chunk_'+str(num)+'_R1.fq'), 'w') as output1:
            with open(os.path.join(outputdir, 'chunk_'+str(num)+'_R2.fq'), 'w') as output2:
                with open(os.path.join(outputdir, 'chunk_'+str(num)+'_R3.fq'), 'w') as output3:
                    lines1 = return_lines(R1, linepos, x[0], x[1])
                    output1.write('%s' % ''.join(lines1))
                    lines2 = return_lines(R2, linepos, x[0], x[1])
                    output2.write('%s' % ''.join(lines2))
                    lines3 = return_lines(I1, linepos2, x[0], x[1])
                    output3.write('%s' % ''.join(lines3))


def split_fasta(input, outputdir, chunks):
    #function to return line positions of fasta files for chunking
    fastapos = []
    position = 0
    numseqs = 0
    with open(input, 'r') as infile:
        for line in infile:
            if line.startswith('>'):
                numseqs += 1
                fastapos.append(position)
            position += 1
    splits = []
    n = numseqs // chunks
    num = 0
    lastline = line_count(input)
    for i in range(chunks):
        if i == 0:
            start = 0
            num = n
            lastpos = fastapos[n+1]
        elif i == chunks-1: #last one
            start = lastpos
            lastpos = lastline
        else:
            start = lastpos
            num = num + n
            try:
                lastpos = fastapos[num+1] #find the n+1 seq
            except IndexError:
                lastpos = fastapos[-1]
        splits.append((start, lastpos))
    #check if output folder exists, if not create it
    if not os.path.isdir(outputdir):
        os.makedirs(outputdir)
    #get line positions from file
    linepos = scan_linepos(input)
    #loop through the positions and write output
    for i, x in enumerate(splits):
        num = i+1
        with open(os.path.join(outputdir, 'chunk_'+str(num)+'.fasta'), 'w') as output:
            lines = return_lines(input, linepos, x[0], x[1])
            output.write('{:}'.format(''.join(lines)))


def trim3prime(input, trimlen, output, removelist):
    with open(output, 'w') as outfile:
        for title, seq, qual in FastqGeneralIterator(gzopen(input)):
            if not title.split(' ')[0] in removelist:
                Seq = seq[:trimlen]
                Qual = qual[:trimlen]
                outfile.write("@%s\n%s\n+\n%s\n" % (title, Seq, Qual))


def PEsanitycheck(R1, R2):
    R1count = line_count(R1)
    R2count = line_count(R2)
    if R1count != R2count:
        return False
    else:
        return True


def PEandIndexCheck(R1, R2, R3):
    R1count = countfastq(R1)
    R2count = countfastq(R2)
    R3count = countfastq(R3)
    if R1count == R2count == R3count:
        return True
    else:
        return False


def mapIndex(seq, mapDict, bcmismatch):
    besthit = []
    for index_name,index in mapDict.items():
        align = edlib.align(index, seq, mode="SHW", k=bcmismatch, additionalEqualities=degenNuc)
        if align["editDistance"] < 0:
            continue
        elif align["editDistance"] == 0:
            return (index_name, 0)
        else:
            if len(besthit) < 3:
                besthit = [index, index_name, align["editDistance"]]
            else:
                if align["editDistance"] < int(besthit[2]):
                    besthit = [index, index_name, align["editDistance"]]
    if len(besthit) == 3:
        return (besthit[1], besthit[2])
    else:
        return (None,None)


def DemuxIllumina(R1, R2, I1, mapDict, mismatch,
                  fwdprimer, revprimer, primer_mismatch,
                  outR1, outR2, trim_primers=True):
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    Total = 0
    FPrimer = 0
    RPrimer = 0
    BCFound = 0
    #function to loop through PE reads, renaming according to index
    file1 = FastqGeneralIterator(gzopen(R1))
    file2 = FastqGeneralIterator(gzopen(R2))
    file3 = FastqGeneralIterator(gzopen(I1))
    counter = 1
    with open(outR1, 'w') as outfile1:
        with open(outR2, 'w') as outfile2:
            for read1, read2, index in zip(file1, file2, file3):
                Total += 1
                Name,Diffs = mapIndex(index[1], mapDict, mismatch)
                if Name:
                    BCFound += 1
                    if trim_primers:
                        #strip primers if found
                        R1ForPos = trimForPrimer(fwdprimer, read1[1], primer_mismatch)
                        R1RevPos = trimRevPrimer(revprimer, read1[1], primer_mismatch)
                        R2ForPos = trimForPrimer(revprimer, read2[1], primer_mismatch)
                        R2RevPos = trimRevPrimer(fwdprimer, read2[1], primer_mismatch)
                        if R1ForPos > 0:
                            FPrimer += 1
                        if R1RevPos > 0:
                            RPrimer += 1
                        header = 'R_'+str(counter)+';barcodelabel='+Name+';bcseq='+index[1]+';bcdiffs='+str(Diffs)+';'
                        outfile1.write('@%s\n%s\n+\n%s\n' % (header, read1[1][R1ForPos:R1RevPos], read1[2][R1ForPos:R1RevPos]))
                        outfile2.write('@%s\n%s\n+\n%s\n' % (header, read2[1][R2ForPos:R2RevPos], read2[2][R2ForPos:R2RevPos]))
                    else:
                        header = 'R_'+str(counter)+';barcodelabel='+Name+';bcseq='+index[1]+';bcdiffs='+str(Diffs)+';'
                        outfile1.write('@%s\n%s\n+\n%s\n' % (header, read1[1], read1[2]))
                        outfile2.write('@%s\n%s\n+\n%s\n' % (header, read2[1], read2[2]))
                    counter += 1
    return Total, BCFound, FPrimer, RPrimer

def DemuxIllumina4SRA(R1, R2, I1, mapDict, mismatch, outdir):
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    Total = 0
    BCFound = 0
    #function to loop through PE reads, renaming according to index
    file1 = FastqGeneralIterator(gzopen(R1))
    file2 = FastqGeneralIterator(gzopen(R2))
    file3 = FastqGeneralIterator(gzopen(I1))
    counter = 1
    for read1, read2, index in zip(file1, file2, file3):
        Total += 1
        Name, Diffs = mapIndex(index[1], mapDict, mismatch)
        if Name:
            BCFound += 1
            with open(os.path.join(outdir, Name+'_R1.fastq'), 'a') as outfile1:
                with open(os.path.join(outdir, Name+'_R2.fastq'), 'a') as outfile2:
                    #header = 'R_'+str(counter)+';barcodelabel='+Name+';bcseq='+index[1]+';bcdiffs='+str(Diffs)+';'
                    outfile1.write('@%s\n%s\n+\n%s\n' % (read1[0], read1[1], read1[2]))
                    outfile2.write('@%s\n%s\n+\n%s\n' % (read2[0], read2[1], read2[2]))
                    counter += 1
    return Total, BCFound

def stripPrimersPE(R1, R2, RL, samplename, fwdprimer, revprimer, primer_mismatch, require_primer, full_length, outR1, outR2):
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    #can walk through dataset in pairs
    file1 = FastqGeneralIterator(gzopen(R1))
    file2 = FastqGeneralIterator(gzopen(R2))
    counter = 1
    Total = 0
    multihits = 0
    findForPrimer = 0
    findRevPrimer = 0
    with open(outR1, 'w') as outfile1:
        with open(outR2, 'w') as outfile2:
            for read1, read2 in zip(file1, file2):
                Total += 1
                ffp = False
                frp = False
                R1Seq = read1[1][:RL]
                R1Qual = read1[2][:RL]
                R2Seq = read2[1][:RL]
                R2Qual = read2[2][:RL]
                ForTrim, RevTrim = (0,)*2
                #look for forward primer in forward read
                R1foralign = edlib.align(fwdprimer, R1Seq, mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                if R1foralign['editDistance'] < 0:
                    if require_primer == 'on' or full_length: #not found
                        continue
                else:
                    if len(R1foralign['locations']) > 1: #multiple hits
                        multihits += 1
                        continue
                    try:
                        ForTrim = R1foralign["locations"][0][1]+1
                        findForPrimer += 1
                        ffp = True
                    except IndexError:
                        pass
                R1revalign = edlib.align(RevComp(revprimer), R1Seq, mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                if R1revalign['editDistance'] < 0:
                    R1RevCut = RL
                else:
                    R1RevCut = R1revalign["locations"][0][0]
                    findRevPrimer += 1
                    frp = True
                #look for reverse primer in reverse read
                R2foralign = edlib.align(revprimer, R2Seq, mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                if R2foralign['editDistance'] < 0:
                    if require_primer == 'on' or full_length: #not found
                        continue
                else:
                    if len(R2foralign['locations']) > 1: #multiple hits
                        multihits += 1
                        continue
                    try:
                        RevTrim = R2foralign["locations"][0][1]+1
                        if not frp:
                            findRevPrimer += 1
                    except IndexError:
                        pass
                R2revalign = edlib.align(RevComp(fwdprimer), R2Seq, mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                if R2revalign['editDistance'] < 0:
                    R2RevCut = RL
                else:
                    R2RevCut = R2revalign["locations"][0][0]
                    if not ffp:
                        findForPrimer += 1
                header = 'R_{:};barcodelabel={:};'.format(counter,samplename)
                outfile1.write('@%s\n%s\n+\n%s\n' % (header, R1Seq[ForTrim:R1RevCut], R1Qual[ForTrim:R1RevCut]))
                outfile2.write('@%s\n%s\n+\n%s\n' % (header, R2Seq[RevTrim:R2RevCut], R2Qual[RevTrim:R2RevCut]))
                counter += 1
    return Total, counter-1, multihits, findForPrimer, findRevPrimer

def primerFound(primer, seq, mismatch):
    align = edlib.align(primer, seq, mode="HW", k=mismatch, additionalEqualities=degenNuc)
    if align['editDistance'] < 0:
        return False
    else:
        return True

def illuminaReorient(R1, R2, fwdprimer, revprimer, mismatch, ReadLength, outR1, outR2):
    '''
    function to re-orient reads based on primer sequences
    only useful if primers in the reads
    drops reads that don't have matching primers
    '''
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    Total = 0
    Correct = 0
    Flipped = 0
    Dropped = 0
    #function to loop through PE reads, renaming according to index
    file1 = FastqGeneralIterator(gzopen(R1))
    file2 = FastqGeneralIterator(gzopen(R2))
    with open(outR1, 'w') as outfile1:
        with open(outR2, 'w') as outfile2:
            for read1, read2 in zip(file1, file2):
                Total += 1
                if primerFound(fwdprimer, read1[1], mismatch) and primerFound(revprimer, read2[1], mismatch):
                    Correct += 1
                    outfile1.write('@%s\n%s\n+\n%s\n' % (read1[0], read1[1][:ReadLength], read1[2][:ReadLength]))
                    outfile2.write('@%s\n%s\n+\n%s\n' % (read2[0], read2[1][:ReadLength], read2[2][:ReadLength]))
                elif primerFound(fwdprimer, read2[1], mismatch) and primerFound(revprimer, read1[1], mismatch):
                    Flipped += 1
                    outfile1.write('@%s\n%s\n+\n%s\n' % (read2[0], read2[1][:ReadLength], read2[2][:ReadLength]))
                    outfile2.write('@%s\n%s\n+\n%s\n' % (read1[0], read1[1][:ReadLength], read1[2][:ReadLength]))
                else:
                    Dropped += 1
    return Total, Correct, Flipped, Dropped


def demuxIlluminaPE(R1, R2, fwdprimer, revprimer, samples, forbarcodes, revbarcodes, barcode_mismatch, primer_mismatch, outR1, outR2, stats):
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    #function to loop through PE reads, renaming according to index
    file1 = FastqGeneralIterator(gzopen(R1))
    file2 = FastqGeneralIterator(gzopen(R2))
    counter = 1
    Total = 0
    NoBarcode = 0
    NoRevBarcode = 0
    NoPrimer = 0
    NoRevPrimer = 0
    ValidSeqs = 0
    with open(outR1, 'w') as outfile1:
        with open(outR2, 'w') as outfile2:
            for read1, read2 in zip(file1, file2):
                Total += 1
                #look for valid barcode in forward read
                if len(forbarcodes) > 0:
                    BC, BCLabel = AlignBarcode(read1[1], forbarcodes, barcode_mismatch)
                    if BC == '':
                        NoBarcode += 1
                        continue
                if len(samples) > 0: #sample dictionary so enforce primers and barcodes from here
                    FwdPrimer = samples[BCLabel]['ForPrimer']
                    RevPrimer = samples[BCLabel]['RevPrimer']
                    foralign = edlib.align(FwdPrimer, read1[1], mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                    if foralign['editDistance'] < 0: #not found
                        NoPrimer += 1
                        continue
                    stringent = {}
                    stringent[BCLabel] = samples[BCLabel]['RevBarcode']
                    revBC, revBCLabel = AlignBarcode(read2[1], stringent, barcode_mismatch)
                    if revBC == '':
                        NoRevBarcode += 1
                        continue
                    #look for reverse primer in reverse read
                    revalign = edlib.align(RevPrimer, read2[1], mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                    if revalign['editDistance'] < 0: #not found
                        NoRevPrimer += 1
                        continue
                else:
                    #look for forward primer
                    foralign = edlib.align(fwdprimer, read1[1], mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                    if foralign['editDistance'] < 0: #not found
                        NoPrimer += 1
                        continue
                    if len(revbarcodes) > 0:
                        #look for valid revbarcodes
                        if len(revbarcodes) > 0:
                            revBC, revBCLabel = AlignBarcode(read2[1], revbarcodes, barcode_mismatch)
                            if revBC == '':
                                NoRevBarcode += 1
                                continue
                    #look for reverse primer in reverse read
                    revalign = edlib.align(revprimer, read2[1], mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
                    if revalign['editDistance'] < 0: #not found
                        NoRevPrimer += 1
                        continue
                #if get here, then all is well, construct new header and trim reads
                if BCLabel == revBCLabel:
                    label = BCLabel
                else:
                    label = BCLabel+':-:'+revBCLabel
                ForTrim = foralign["locations"][0][1]+1
                RevTrim = revalign["locations"][0][1]+1
                header = 'R_{:};barcodelabel={:};'.format(counter,label)
                outfile1.write('@%s\n%s\n+\n%s\n' % (header, read1[1][ForTrim:], read1[2][ForTrim:]))
                outfile2.write('@%s\n%s\n+\n%s\n' % (header, read2[1][RevTrim:], read2[2][RevTrim:]))
                counter += 1
                ValidSeqs += 1
    with open(stats, 'w') as statsout:
        statsout.write('%i,%i,%i,%i,%i,%i\n' % (Total, NoBarcode, NoPrimer, NoRevBarcode, NoRevPrimer, ValidSeqs))

def demuxIlluminaPE2(R1, R2, fwdprimer, revprimer, samples, forbarcodes, revbarcodes, barcode_mismatch, primer_mismatch, outR1, outR2, stats):
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    #function to loop through PE reads, renaming according to index
    file1 = FastqGeneralIterator(gzopen(R1))
    file2 = FastqGeneralIterator(gzopen(R2))
    counter = 1
    Total = 0
    NoBarcode = 0
    NoRevBarcode = 0
    NoPrimer = 0
    NoRevPrimer = 0
    ValidSeqs = 0
    with open(outR1, 'w') as outfile1:
        with open(outR2, 'w') as outfile2:
            for read1, read2 in zip(file1, file2):
                Total += 1
                #look for forward primer first, should all have primer and in correct orientation
                R1ForTrim = trimForPrimer(fwdprimer, read1[1], primer_mismatch)
                if R1ForTrim == 0: #no primer found
                    continue
                if len(forbarcodes) > 0: #search for barcode match in seq upstream of primer
                    R1BCtrim = R1ForTrim - len(fwdprimer)
                    BC, BCLabel = AlignBarcode2(read1[1][:R1BCtrim], forbarcodes, barcode_mismatch)
                    if BC == '':
                        NoBarcode += 1
                        continue
                if len(samples) > 0: #sample dictionary so enforce primers and barcodes from here
                    FwdPrimer = samples[BCLabel]['ForPrimer']
                    RevPrimer = samples[BCLabel]['RevPrimer']
                    stringent = {}
                    stringent[BCLabel] = samples[BCLabel]['RevBarcode']
                    #find rev primer in reverse read
                    R2ForTrim = trimForPrimer(RevPrimer, read2[1], primer_mismatch)
                    if R2ForTrim == 0:
                        continue
                    #look for reverse barcode
                    R2BCTrim = R2ForTrim - len(RevPrimer)
                    revBC, revBCLabel = AlignBarcode2(read2[1][:R2BCTrim], stringent, barcode_mismatch)
                    if revBC == '':
                        NoRevBarcode += 1
                        continue
                    #okay, found both primers and barcodes, now 1 more cleanup step trip revcomped primers
                    R1RevTrim = trimRevPrimer(RevPrimer, read1[1], primer_mismatch)
                    R2RevTrim = trimRevPrimer(FwdPrimer, read2[1], primer_mismatch)
                else:
                    #no samples dictionary, so allow all combinations of matches
                    R2ForTrim = trimForPrimer(revprimer, read2[1], primer_mismatch)
                    if R2ForTrim == 0:
                        continue
                    if len(revbarcodes) > 0:
                        #look for reverse barcode
                        R2BCTrim = R2ForTrim - len(revprimer)
                        revBC, revBCLabel = AlignBarcode2(read2[1][:R2BCTrim], revbarcodes, barcode_mismatch)
                        if revBC == '':
                            NoRevBarcode += 1
                            continue
                    #okay, found both primers and barcodes, now 1 more cleanup step trip revcomped primers
                    R1RevTrim = trimRevPrimer(revprimer, read1[1], primer_mismatch)
                    R2RevTrim = trimRevPrimer(fwdprimer, read2[1], primer_mismatch)

                #if get here, then all is well, construct new header and trim reads
                if BCLabel == revBCLabel:
                    label = BCLabel
                else:
                    label = BCLabel+':-:'+revBCLabel
                header = 'R_{:};barcodelabel={:};'.format(counter,label)
                outfile1.write('@%s\n%s\n+\n%s\n' % (header, read1[1][R1ForTrim:R1RevTrim], read1[2][R1ForTrim:R1RevTrim]))
                outfile2.write('@%s\n%s\n+\n%s\n' % (header, read2[1][R2ForTrim:R2RevTrim], read2[2][R2ForTrim:R2RevTrim]))
                counter += 1
                ValidSeqs += 1
    with open(stats, 'w') as statsout:
        statsout.write('%i,%i,%i,%i,%i,%i\n' % (Total, NoBarcode, NoPrimer, NoRevBarcode, NoRevPrimer, ValidSeqs))


def trimForPrimer(primer, seq, primer_mismatch):
    foralign = edlib.align(primer, seq, mode="HW", k=primer_mismatch, additionalEqualities=degenNuc)
    if foralign['editDistance'] < 0:
        return 0
    else:
        CutPos = foralign["locations"][0][1]+1
        return CutPos

def trimRevPrimer(primer, seq, primer_mismatch):
    revprimer = RevComp(primer)
    revalign = edlib.align(revprimer, seq, mode="HW", k=primer_mismatch, task="locations", additionalEqualities=degenNuc)
    if revalign['editDistance'] < 0:
        return len(seq)
    else:
        CutPos = revalign["locations"][0][0]
        return CutPos

def losslessTrim(input, fwdprimer, revprimer, mismatch, trimLen, padding, minlength, output):
    '''
    function to trim primers if found from SE reads
    and then trim/pad to a set length
    '''
    with open(output, 'w') as outfile:
        for title, seq, qual in FastqGeneralIterator(gzopen(input)):
            #sometimes primers sneek through the PE merging pipeline, check quickly again trim if found
            ForTrim = trimForPrimer(fwdprimer, seq, mismatch)
            RevTrim = trimRevPrimer(revprimer, seq, mismatch)
            Seq = seq[ForTrim:RevTrim]
            Qual = qual[ForTrim:RevTrim]
            if len(Seq) < int(minlength): #need this check here or primer dimers will get through
                continue
            if len(Seq) < int(trimLen) and padding == 'on':
                pad = int(trimLen) - len(Seq)
                SeqF = Seq + pad*'N'
                QualF = Qual + pad*'I'
            else:
                SeqF = Seq[:trimLen]
                QualF = Qual[:trimLen]
            outfile.write('@%s\n%s\n+\n%s\n' % (title, SeqF, QualF))


def checkBCinHeader(input):
    #read first header
    for title, seq, qual in FastqGeneralIterator(open(input)):
        header = title.split(' ')
        info = header[-1]
        if info.split(':')[-1].isdigit():
            return False
        else:
            return True
        break

def illuminaBCmismatch(R1, R2, index):
    remove = []
    for file in [R1,R2]:
        for title, seq, qual in FastqGeneralIterator(open(file)):
            ID = title.split(' ')[0]
            BC = title.split(':')[-1]
            if BC != index:
                remove.append(ID)
    remove = set(remove)
    return remove

def fasta2barcodes(input, revcomp):
    BC = {}
    with open(input, 'r') as infile:
        for rec in SeqIO.parse(infile, 'fasta'):
            if not rec.id in BC:
                if revcomp:
                    Seq = RevComp(str(rec.seq))
                else:
                    Seq = str(rec.seq)
                BC[rec.id] = Seq
    return BC

def AlignBarcode(Seq, BarcodeDict, mismatch):
    besthit = []
    for BL in list(BarcodeDict.keys()):
        B = BarcodeDict[BL]
        #apparently people use N's in barcode sequences, this doesn't work well with
        #edlib, so if N then trim and then align, need to trim the seq as well
        if B.startswith('N'):
            origLen = len(B)
            B = B.lstrip('N')
            newLen = len(B)
            lenDiff = origLen - newLen
            newSeq = Seq[lenDiff:]
        else:
            newSeq = Seq
        align = edlib.align(B, newSeq, mode="SHW", k=mismatch, additionalEqualities=degenNuc)
        if align["editDistance"] < 0:
            continue
        elif align["editDistance"] == 0:
            return B, BL
        elif align["editDistance"] > 0 and mismatch == 0:
            continue
        else:
            if len(besthit) < 3:
                besthit = [B, BL, align["editDistance"]]
            else:
                if align["editDistance"] < int(besthit[2]):
                    besthit = [B, BL, align["editDistance"]]
    if len(besthit) == 3:
        return besthit[0], besthit[1]
    else:
        return "", ""

def AlignRevBarcode(Seq, BarcodeDict, mismatch):
    besthit = []
    for BL in list(BarcodeDict.keys()):
        B = BarcodeDict[BL]
        if B.endswith('N'):
            B = B.rstrip('N')
        align = edlib.align(B, Seq, mode="HW", k=mismatch, additionalEqualities=degenNuc)
        if align["editDistance"] < 0:
            continue
        elif align["editDistance"] == 0:
            return B, BL
        elif align["editDistance"] > 0 and mismatch == 0:
            continue
        else:
            if len(besthit) < 3:
                besthit = [B, BL, align["editDistance"]]
            else:
                if align["editDistance"] < int(besthit[2]):
                    besthit = [B, BL, align["editDistance"]]
    if len(besthit) == 3:
        return besthit[0], besthit[1]
    else:
        return "", ""

def AlignBarcode2(Seq, BarcodeDict, mismatch):
    besthit = []
    for BL in list(BarcodeDict.keys()):
        B = BarcodeDict[BL]
        #apparently people use N's in barcode sequences, this doesn't work well with
        #edlib, so if N then trim and then align, need to trim the seq as well
        if B.startswith('N'):
            origLen = len(B)
            B = B.lstrip('N')
            newLen = len(B)
            lenDiff = origLen - newLen
            newSeq = Seq[lenDiff:]
        else:
            newSeq = Seq
        align = edlib.align(B, newSeq, mode="HW", k=int(mismatch), additionalEqualities=degenNuc)
        if align["editDistance"] < 0:
            continue
        elif align["editDistance"] == 0:
            return B, BL
        elif align["editDistance"] > 0 and mismatch == 0:
            continue
        else:
            if len(besthit) < 3:
                besthit = [B, BL, align["editDistance"]]
            else:
                if align["editDistance"] < int(besthit[2]):
                    besthit = [B, BL, align["editDistance"]]
    if len(besthit) == 3:
        return besthit[0], besthit[1]
    else:
        return "", ""

def findFwdPrimer(primer, sequence, mismatch, equalities):
    #trim position
    TrimPos = None
    #search for match
    align = edlib.align(primer, sequence, mode="HW", k=int(mismatch), additionalEqualities=equalities)
    if align["editDistance"] >= 0: #we found a hit
        TrimPos = align["locations"][0][1]+1
    #return position will be None if not found
    return TrimPos

def findRevPrimer(primer, sequence, mismatch, equalities):
    #trim position
    TrimPos = None
    #search for match
    align = edlib.align(primer, sequence, mode="HW", task="locations", k=int(mismatch), additionalEqualities=equalities)
    if align["editDistance"] >= 0: #we found a hit
        TrimPos = align["locations"][0][0]
    #return position will be None if not found
    return TrimPos

def MergeReadsSimple(R1, R2, tmpdir, outname, minlen, usearch, rescue, method='vsearch'):
    #check that num sequences is identical
    if not PEsanitycheck(R1, R2):
        log.error("%s and %s are not properly paired, exiting" % (R1, R2))
        sys.exit(1)
    #next run USEARCH/vsearch mergepe
    merge_out = os.path.join(tmpdir, outname + '.merged.fq')
    skip_for = os.path.join(tmpdir, outname + '.notmerged.R1.fq')
    report = os.path.join(tmpdir, outname +'.merge_report.txt')
    log.debug("Now merging PE reads")
    if method == 'usearch':
        cmd = [usearch, '-fastq_mergepairs', R1, '-reverse', R2,
               '-fastqout', merge_out, '-fastq_trunctail', '5',
               '-fastqout_notmerged_fwd', skip_for, '-minhsp', '12',
               '-fastq_maxdiffs', '8', '-report', report,
               '-fastq_minmergelen', str(minlen), '-threads', '1']
    elif method == 'vsearch':
        log.debug('Merging PE reads using vsearch --fastq_mergepairs: {} {}'.format(os.path.basename(R1), os.path.basename(R2)))
        cmd = ['vsearch', '--fastq_mergepairs', R1, '--reverse', R2,
               '--fastqout', merge_out, '--fastqout_notmerged_fwd', skip_for,
               '--fastq_minmergelen', str(minlen), '--fastq_allowmergestagger', '--threads', '1']
    runSubprocess(cmd, log)
    #now concatenate files for downstream pre-process_illumina.py script
    final_out = os.path.join(tmpdir, outname)
    tmp_merge = os.path.join(tmpdir, outname+'.tmp')
    with open(tmp_merge, 'w') as cat_file:
        shutil.copyfileobj(open(merge_out,'r'), cat_file)
        if rescue == 'on':
            shutil.copyfileobj(open(skip_for,'r'), cat_file)
    phixsize = getSize(tmp_merge)
    phixcount = countfastq(tmp_merge)
    if method == 'usearch':
        #run phix removal
        #since most users have 32 bit usearch, check size of file, if > 3 GB, split into parts
        log.debug("Removing phix from %s" % outname)
        log.debug('File Size: %i bytes' % phixsize)
        if phixsize > 3e9:
            log.debug('FASTQ > 3 GB, splitting FASTQ file into chunks to avoid potential memory problems with 32 bit usearch')
            phixdir = os.path.join(tmpdir, 'phix_'+str(os.getpid()))
            os.makedirs(phixdir)
            num = round(int((phixsize / 3e9))) + 1
            split_fastq(tmp_merge, phixcount, phixdir, int(num))
            for file in os.listdir(phixdir):
                if file.endswith(".fq"):
                    output = os.path.join(phixdir, file+'.phix')
                    file = os.path.join(phixdir, file)
                    cmd = [usearch, '-filter_phix', file, '-output', output, '-threads', '1']
                    runSubprocess(cmd, log)
            with open(final_out, 'w') as finalout:
                for file in os.listdir(phixdir):
                    if file.endswith('.phix'):
                        with open(os.path.join(phixdir, file), 'r') as infile:
                            shutil.copyfileobj(infile, finalout)
            shutil.rmtree(phixdir)
        else:
            cmd = [usearch, '-filter_phix', tmp_merge, '-output', final_out, '-threads', '1']
            runSubprocess(cmd, log)
    else:
        os.rename(tmp_merge, final_out)
    #count output
    finalcount = countfastq(final_out)
    SafeRemove(merge_out)
    SafeRemove(skip_for)
    SafeRemove(tmp_merge)
    return phixcount, finalcount


def MergeReads(R1, R2, tmpdir, outname, read_length, minlen, usearch, rescue, method, index, mismatch):
    removelist = []
    if mismatch == 0 and index != '':
        if checkBCinHeader(R1):
            log.debug("Searching for index mismatches > 0: %s" % index)
            removelist = illuminaBCmismatch(R1, R2, index)
            log.debug("Removing %i reads with index mismatch > 0" % len(removelist))
    pretrim_R1 = os.path.join(tmpdir, outname + '.pretrim_R1.fq')
    pretrim_R2 = os.path.join(tmpdir, outname + '.pretrim_R2.fq')
    log.debug("Removing index 3prime bp 'A' from reads")
    trim3prime(R1, read_length, pretrim_R1, removelist)
    trim3prime(R2, read_length, pretrim_R2, removelist)

    #check that num sequences is identical
    if not PEsanitycheck(pretrim_R1, pretrim_R2):
        log.error("%s and %s are not properly paired, exiting" % (R1, R2))
        sys.exit(1)

    #next run USEARCH/vsearch mergepe
    merge_out = os.path.join(tmpdir, outname + '.merged.fq')
    skip_for = os.path.join(tmpdir, outname + '.notmerged.R1.fq')
    report = os.path.join(tmpdir, outname +'.merge_report.txt')
    log.debug("Now merging PE reads")
    if method == 'usearch':
        cmd = [usearch, '-fastq_mergepairs', pretrim_R1, '-reverse', pretrim_R2, '-fastqout', merge_out, '-fastq_trunctail', '5', '-fastqout_notmerged_fwd', skip_for,'-minhsp', '12','-fastq_maxdiffs', '8', '-report', report, '-fastq_minmergelen', str(minlen)]
    else:
        cmd = ['vsearch', '--fastq_mergepairs', pretrim_R1, '--reverse', pretrim_R2, '--fastqout', merge_out, '--fastqout_notmerged_fwd', skip_for, '--fastq_minmergelen', str(minlen), '--fastq_allowmergestagger']
    runSubprocess(cmd, log)
    #now concatenate files for downstream pre-process_illumina.py script
    final_out = os.path.join(tmpdir, outname)
    tmp_merge = os.path.join(tmpdir, outname+'.tmp')
    with open(tmp_merge, 'w') as cat_file:
        shutil.copyfileobj(open(merge_out,'r'), cat_file)
        if rescue == 'on':
            shutil.copyfileobj(open(skip_for,'r'), cat_file)
    #run phix removal
    #since most users have 32 bit usearch, check size of file, if > 3 GB, split into parts
    log.debug("Removing phix from %s" % outname)
    phixsize = getSize(tmp_merge)
    phixcount = countfastq(tmp_merge)
    log.debug('File Size: %i bytes' % phixsize)
    if phixsize > 3e9:
        log.debug('FASTQ > 3 GB, splitting FASTQ file into chunks to avoid potential memory problems with 32 bit usearch')
        phixdir = os.path.join(tmpdir, 'phix_'+str(os.getpid()))
        os.makedirs(phixdir)
        num = round(int((phixsize / 3e9))) + 1
        split_fastq(tmp_merge, phixcount, phixdir, int(num))
        for file in os.listdir(phixdir):
            if file.endswith(".fq"):
                output = os.path.join(phixdir, file+'.phix')
                file = os.path.join(phixdir, file)
                cmd = [usearch, '-filter_phix', file, '-output', output]
                runSubprocess(cmd, log)
        with open(final_out, 'wb') as finalout:
            for file in os.listdir(phixdir):
                if file.endswith('.phix'):
                    with open(os.path.join(phixdir, file), 'r') as infile:
                        shutil.copyfileobj(infile, finalout)
        shutil.rmtree(phixdir)
    else:
        cmd = [usearch, '-filter_phix', tmp_merge, '-output', final_out]
        runSubprocess(cmd, log)
    #count output
    origcount = countfastq(R1)
    finalcount = countfastq(final_out)
    log.debug("Removed %i reads that were phiX" % (origcount - finalcount - len(removelist)))
    pct_out = finalcount / float(origcount)
    #clean and close up intermediate files
    os.remove(merge_out)
    os.remove(pretrim_R1)
    os.remove(pretrim_R2)
    os.remove(skip_for)
    os.remove(tmp_merge)
    return log.info('{0:,}'.format(finalcount) + ' reads passed ('+'{0:.1%}'.format(pct_out)+')')

def validateorientation(tmp, reads, otus, output):
    orientcounts = os.path.join(tmp, 'orient.uc')
    cmd = ['vsearch', '--usearch_global', reads, '--db', otus, '--sizein', '--id', '0.97', '--strand', 'plus', '--uc', orientcounts]
    runSubprocess(cmd, log)
    OTUCounts = {}
    with open(orientcounts, 'r') as countdata:
        for line in countdata:
            line = line.rstrip()
            cols = line.split('\t')
            ID = cols[9]
            if ID == '*':
                continue
            size = cols[8].split('size=')[-1].replace(';', '')
            if not ID in OTUCounts:
                OTUCounts[ID] = int(size)
            else:
                OTUCounts[ID] += int(size)
    orientmap = os.path.join(tmp, 'orient-map.txt')
    cmd = ['vsearch', '--usearch_global', otus, '--db', otus, '--self', '--id', '0.95', '--strand', 'both', '--userout', orientmap, '--userfields', 'query+target+qstrand+id']
    runSubprocess(cmd, log)
    orient_remove = []
    keeper = []
    with open(orientmap, 'r') as selfmap:
        for line in selfmap:
            line = line.rstrip()
            cols = line.split('\t')
            if cols[2] == '-':
                qCount = OTUCounts.get(cols[0])
                tCount = OTUCounts.get(cols[1])
                if qCount > tCount:
                    if not cols[1] in orient_remove and not cols[1] in keeper:
                        orient_remove.append(cols[1])
                    if not cols[0] in keeper:
                        keeper.append(cols[0])
                else:
                    if not cols[0] in orient_remove and not cols[0]:
                        orient_remove.append(cols[0])
                    if not cols[1] in keeper:
                        keeper.append(cols[1])
    log.debug('Dropping {:,} OTUs: {:}'.format(len(orient_remove), ', '.join(orient_remove)))
    count = 0
    with open(output, 'w') as outfile:
        with open(otus, 'r') as infile:
            for rec in SeqIO.parse(infile, 'fasta'):
                if not rec.id in orient_remove:
                    count += 1
                    SeqIO.write(rec, outfile, 'fasta')
    return count, len(orient_remove)


def validateorientationDADA2(OTUCounts, otus, output):
    orientmap = 'orient-map.txt'
    cmd = ['vsearch', '--usearch_global', otus, '--db', otus, '--self', '--id', '0.95', '--strand', 'both', '--userout', orientmap, '--userfields', 'query+target+qstrand+id']
    runSubprocess(cmd, log)
    orient_remove = []
    keeper = []
    with open(orientmap, 'r') as selfmap:
        for line in selfmap:
            line = line.rstrip()
            cols = line.split('\t')
            if cols[2] == '-':
                qCount = OTUCounts.get(cols[0])
                tCount = OTUCounts.get(cols[1])
                if qCount > tCount:
                    if not cols[1] in orient_remove and not cols[1] in keeper:
                        orient_remove.append(cols[1])
                    if not cols[0] in keeper:
                        keeper.append(cols[0])
                else:
                    if not cols[0] in orient_remove and not cols[0]:
                        orient_remove.append(cols[0])
                    if not cols[1] in keeper:
                        keeper.append(cols[1])
    log.debug('Dropping {:,} OTUs: {:}'.format(len(orient_remove), ', '.join(natsorted(orient_remove))))
    count = 0
    with open(output, 'w') as outfile:
        with open(otus, 'r') as infile:
            for rec in SeqIO.parse(infile, 'fasta'):
                if not rec.id in orient_remove:
                    count += 1
                    SeqIO.write(rec, outfile, 'fasta')
    SafeRemove(orientmap)
    return count, len(orient_remove)


def dictFlip(input):
    #flip the list of dictionaries
    outDict = {}
    for k,v in input.items():
        for i in v:
            if not i in outDict:
                outDict[i] = k
            else:
                print("duplicate ID found: %s" % i)
    return outDict

def classifier2dict(input, pcutoff):
    ClassyDict = {}
    with open(input, 'r') as infile:
        for line in infile:
            cols = line.split('\t')
            ID = cols[0]
            tax = cols[1].split(',')
            passtax = []
            scores = []
            hit = False
            for i,level in enumerate(tax):
                if '(' in level:
                    score = level.split('(')[-1].replace(')', '')
                    if float(score) >= float(pcutoff):
                        hit = True
                        passtax.append(level.split('(')[0])
                        scores.append(score)
            if hit:
                if not ID in ClassyDict:
                    ClassyDict[ID] = (scores[-1], passtax)
    return ClassyDict

def usearchglobal2dict(input):
    GlobalDict = {}
    with open(input, 'r') as infile:
        for line in infile:
            line = line.rstrip()
            cols = line.split('\t')
            ID = cols[0]
            if cols[1] != '*':
                tax = cols[1].split('tax=')[-1]
                tax = tax.split(',')
                pident = float(cols[-1]) / 100
                besthit = cols[1].split(';tax=')[0]
            else:
                tax = ['No hit']
                besthit = 'None'
                pident = 0.0000
            pident = "{0:.4f}".format(pident)
            #since we can have multiple hits with same pident, need to get best taxonomy
            if not ID in GlobalDict:
                GlobalDict[ID] = [(pident, besthit, '', tax,)]
            else:
                GlobalDict[ID].append((pident, besthit, '', tax))
    Results = {}
    for k,v in natsorted(GlobalDict.items()):
        mostTax = []
        lcaTax = []
        mt = 0
        for hit in v:
            if len(hit[-1]) > mt:
                mt = len(hit[-1])
        #loop through getting the ones with most tax info
        for x in v:
            if len(x[-1]) == mt:
                mostTax.append(x)
                lcaTax.append(x[-1])
        #now if mostTax has more than 1 hit, need to do LCA
        if len(mostTax) <= 1:
            Results[k] = mostTax[0]
        else:
            lcaResult = lcaTax[0]
            lcaFinal = []
            for x in lcaTax[1:]:
                s = set(x)
                lcaFinal = [z for z in lcaResult if z in s]
                lcaResult = lcaFinal
            if len(lcaResult) < mt:
                Results[k] = (mostTax[0][0], mostTax[0][1], 'LCA', lcaResult)
            else:
                Results[k] = (mostTax[0][0], mostTax[0][1], '',lcaResult)
    return Results

def bestclassifier(utax, sintax, otus):
    #both are dictionaries from classifier2dict, otus is a list of OTUs in order
    BestClassify = {}
    for otu in otus:
        sintaxhit, utaxhit, hit = (None,)*3
        if otu in sintax:
            sintaxhit = sintax.get(otu) #this should be okay...
        if otu in utax:
            utaxhit = utax.get(otu) #returns tuple of (score, [taxlist]
        if sintaxhit and utaxhit:
            if len(utaxhit[1]) > len(sintaxhit[1]):
                hit = (utaxhit[0], 'U', utaxhit[1])
            elif len(utaxhit[1]) == len(sintaxhit[1]):
                if float(utaxhit[0]) >= float(sintaxhit[0]):
                    hit = (utaxhit[0], 'U', utaxhit[1])
                else:
                    hit = (sintaxhit[0], 'S', sintaxhit[1])
            else:
                hit = (sintaxhit[0], 'S', sintaxhit[1])
        elif not sintaxhit and utaxhit:
            hit = (utaxhit[0], 'U', utaxhit[1])
        elif sintaxhit and not utaxhit:
            hit = (sintaxhit[0], 'S', sintaxhit[1])
        BestClassify[otu] = hit
    return BestClassify

def bestTaxonomy(usearch, classifier):
    Taxonomy = {}
    for k,v in natsorted(list(usearch.items())):
        globalTax = v[-1] #this should be a list
        try:
            classiTax = classifier.get(k)[-1] #also should be list
        except TypeError:
            classiTax = None
        pident = float(v[0])
        besthitID = v[1]
        LCA = v[2]
        discrep = 'S'
        if pident < 0.9700 and classiTax: #if global alignment hit is less than 97%, then default to classifier result
            method = classifier.get(k)[1].split()[0] #get first letter, either U or S
            score = float(classifier.get(k)[0])
            tax = ','.join(classiTax)
            fulltax = method+discrep+"|{0:.4f}".format(score)+'|'+besthitID+';'+tax
        else: #should default to global alignment with option to update taxonomy from classifier if more information
            method = 'G'
            score = pident * 100
            if classiTax and len(globalTax) < len(classiTax): #iterate through and make sure all levels match else stop where no longer matches
                tax = ','.join(classiTax)
                for lvl in globalTax:
                    if not lvl in classiTax: #now we have a problem
                        error = globalTax.index(lvl) #move one position backwards and keep only that level of taxonomy
                        discrep = 'D'
                        tax = ','.join(globalTax[:error])
                        break
            else:
                tax = ','.join(globalTax)
            if discrep == 'S' and LCA == '':
                fulltax = method+discrep+"|{0:.1f}".format(score)+'|'+besthitID+';'+tax
            else:
                fulltax = method+discrep+"L|{0:.1f}".format(score)+'|'+besthitID+';'+tax
        Taxonomy[k] = fulltax
    return Taxonomy

def utax2qiime(input, output):
    domain = False
    with open(output, 'w') as outfile:
        outfile.write('#OTUID\ttaxonomy\n')
        with open(input, 'r') as infile:
            for line in infile:
                if line.startswith('#'):
                    continue
                line = line.replace('\n', '')
                OTU = line.split('\t')[0]
                tax = line.split('\t')[1]
                try:
                    levels = tax.split(';')[1]
                except IndexError:
                    levels = '*'
                levels = levels.replace(',', ';')
                if levels.startswith('d:'):
                    domain = True
                    changes = ['d','p','c','o','f','g','s']
                else:
                    changes = ['k','p','c','o','f','g','s']
                for i in changes:
                    levels = levels.replace(i+':', i+'__')
                try:
                    levList = levels.split(';')
                except IndexError:
                    levList = [levels]
                #now loop through and add empty levels
                if not domain:
                    if not levList[0].startswith('k__'):
                        levList = ['k__', 'p__', 'c__', 'o__', 'f__', 'g__', 's__']
                    if len(levList) < 2 and levList[0].startswith('k__'):
                        levList.extend(['p__', 'c__', 'o__', 'f__', 'g__', 's__'])
                    if len(levList) > 2 and not levList[2].startswith('c__'):
                        levList.insert(2,'c__')
                    if len(levList) > 3 and not levList[3].startswith('o__'):
                        levList.insert(3,'o__')
                    if len(levList) > 4 and not levList[4].startswith('f__'):
                        levList.insert(4,'f__')
                    if len(levList) > 5 and not levList[5].startswith('g__'):
                        levList.insert(5,'g__')
                    if len(levList) > 6 and not levList[6].startswith('s__'):
                        levList.insert(6,'s__')
                else:
                    if not levList[0].startswith('d__'):
                        levList = ['d__','p__', 'c__', 'o__', 'f__', 'g__', 's__']
                    if len(levList) < 2 and levList[0].startswith('d__'):
                        levList.extend(['p__', 'c__', 'o__', 'f__', 'g__', 's__'])
                    if len(levList) > 2 and not levList[2].startswith('c__'):
                        levList.insert(2,'c__')
                    if len(levList) > 3 and not levList[3].startswith('o__'):
                        levList.insert(3,'o__')
                    if len(levList) > 4 and not levList[4].startswith('f__'):
                        levList.insert(4,'f__')
                    if len(levList) > 5 and not levList[5].startswith('g__'):
                        levList.insert(5,'g__')
                    if len(levList) > 6 and not levList[6].startswith('s__'):
                        levList.insert(6,'s__')
                outfile.write('%s\t%s\n' % (OTU, ';'.join(levList)))


def barcodes2dict(input, mapDict, bcmismatch):
    #here expecting the index reads from illumina, create dictionary for naming?
    Results = {}
    NoMatch = []
    for title, seq, qual in FastqGeneralIterator(gzopen(input)):
        hit = [None, None, None]
        titlesplit = title.split(' ')
        readID = titlesplit[0]
        orient = titlesplit[1]
        if orient.startswith('2:'):
            seq = RevComp(seq)
        if seq in mapDict:
            BC = mapDict.get(seq)
            hit = [BC,seq, 0]
        else:
            for k,v in list(mapDict.items()):
                alignment = edlib.align(k, seq, mode="NW", k=bcmismatch)
                if alignment["editDistance"] < 0:
                    continue
                if hit[0]:
                    oldhit = hit[2]
                    if alignment["editDistance"] < oldhit:
                        hit = [v, k, alignment["editDistance"]]
                else:
                    hit = [v, k, alignment["editDistance"]]

            if not hit[0]: #no match, discard read
                NoMatch.append(readID)
                continue
        if not readID in Results:
            Results[readID] = (hit[0], hit[1], hit[2])
    return Results, NoMatch

def RevComp(s):
    rev_comp_lib = {'A':'T','C':'G','G':'C','T':'A','U':'A','M':'K','R':'Y','W':'W','S':'S','Y':'R','K':'M','V':'B','H':'D','D':'H','B':'V','X':'X','N':'N'}
    cseq = ''
    n = len(s)
    for i in range(0,n):
        c = s[n-i-1]
        cseq += rev_comp_lib[c.upper()]
    return cseq

def mapping2dict(input):
    #parse a qiime mapping file pull out seqs and ID into dictionary
    MapDict = {}
    IDs = []
    with open(input, 'r') as inputfile:
        for line in inputfile:
            if line.startswith('#'):
                continue
            cols = line.split('\t')
            ID = cols[0]
            Seq = cols[1]
            if not Seq in MapDict:
                MapDict[Seq] = ID
            else:
                log.error("duplicate BC seq found {}: {}".format(Seq, ID))
            if not ID in IDs:
                IDs.append(ID)
            else:
                log.error("duplicate ID in mapping file: {}, exiting".format(ID))
                sys.exit(1)
    return MapDict

def runMultiProgress(function, inputList, cpus, args=False):
    #setup pool
    #multiprocessing.set_start_method('fork')
    ctx = multiprocessing.get_context('fork')
    p = ctx.Pool(cpus)
    #setup results and split over cpus
    tasks = len(inputList)
    results = []
    for i in inputList:
        results.append(p.apply_async(function, args=([i]), kwds={'args':args}))
    #refresh pbar every 5 seconds
    while True:
        incomplete_count = sum(1 for x in results if not x.ready())
        if incomplete_count == 0:
            break
        sys.stdout.write("     Progress: %.2f%% \r" % (float(tasks - incomplete_count) / tasks * 100))
        sys.stdout.flush()
        time.sleep(1)
    p.close()
    p.join()


def batch_iterator(iterator, batch_size):
    entry = True #Make sure we loop once
    while entry :
        batch = []
        while len(batch) < batch_size :
            try :
                entry = next(iterator)
            except StopIteration :
                entry = None
            if entry is None :
                #End of file
                break
            batch.append(entry)
        if batch :
            yield batch

def setupLogging(LOGNAME):
    global log
    if 'darwin' in sys.platform:
        stdoutformat = logging.Formatter(colr.GRN+'%(asctime)s'+colr.END+': %(message)s', datefmt='[%b %d %I:%M %p]')
    else:
        stdoutformat = logging.Formatter('%(asctime)s: %(message)s', datefmt='[%I:%M:%S %p]')
    fileformat = logging.Formatter('%(asctime)s: %(message)s', datefmt='[%x %H:%M:%S]')
    log = logging.getLogger(__name__)
    log.setLevel(logging.DEBUG)
    sth = logging.StreamHandler()
    sth.setLevel(logging.INFO)
    sth.setFormatter(stdoutformat)
    log.addHandler(sth)
    fhnd = logging.FileHandler(LOGNAME)
    fhnd.setLevel(logging.DEBUG)
    fhnd.setFormatter(fileformat)
    log.addHandler(fhnd)

def FastMaxEEFilter(input, trunclen, maxee, output):
    from Bio.SeqIO.QualityIO import FastqGeneralIterator
    with open(output, 'w') as out:
        with open(input, 'r') as file:
            for title, seq, qual in FastqGeneralIterator(file):
                trunclen = int(trunclen)
                Seq = seq[:trunclen]
                Qual = qual[:trunclen]
                ee = 0
                for bp, Q in enumerate(Qual):
                    q = int(ASCII.get(Q))
                    P = 10**(float(-q)/10)
                    ee += P
                if ee <= float(maxee):
                    out.write("@%s\n%s\n+\n%s\n" % (title, Seq, Qual))

def MaxEEFilter(input, maxee):
    from Bio import SeqIO
    with open(input, 'r') as f:
        for rec in SeqIO.parse(f, "fastq"):
            ee = 0
            for bp, Q in enumerate(rec.letter_annotations["phred_quality"]):
                P = 10**(float(-Q)/10)
                ee += P
            if ee <= float(maxee):
                rec.name = ""
                rec.description = ""
                yield rec

def dereplicate(input, output):
    from Bio.SeqIO.QualityIO import FastqGeneralIterator
    seqs = {}
    with open(input, 'r') as file:
        for title, sequence, qual in FastqGeneralIterator(file):
            if sequence not in seqs:
                if title.endswith(';'):
                    seqs[sequence]=title+'size=1;'
                else:
                    seqs[sequence]=title+';size=1;'
            else:
                count = int(seqs[sequence].split('=')[-1].rstrip(';')) + 1
                formated_string = seqs[sequence].rsplit('=', 1)[0]+'='+str(count)+';'
                seqs[sequence] = formated_string
    with open(output, 'w') as out:
        for sequence in seqs:
            out.write('>'+seqs[sequence]+'\n'+sequence+'\n')

def convertSize(num, suffix='B'):
    for unit in ['','K','M','G','T','P','E','Z']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)

def faqual2fastq(fasta, qual, fastq):
    global skipCount
    from Bio.SeqIO.QualityIO import PairedFastaQualIterator
    with open(fastq, 'w') as output:
        records = PairedFastaQualIterator(open(fasta), open(qual))
        for rec in records:
            try:
                SeqIO.write(rec, output, 'fastq')
            except ValueError:
                skipCount +1
    return skipCount

def checkfastqsize(input):
    filesize = os.path.getsize(input)
    return filesize

def fastqreindex(input, output):
    from Bio.SeqIO.QualityIO import FastqGeneralIterator
    count = 1
    with open(output, 'w') as out:
        with open(input, 'r') as fastq:
            for title, sequence, qual in FastqGeneralIterator(fastq):
                cols = title.split(';')
                header = 'R_'+str(count)+';'+cols[1]+';'
                count += 1
                out.write("@%s\n%s\n+\n%s\n" % (header, sequence, qual))

def which(name):
    try:
        with open(os.devnull) as devnull:
            diff = ['tbl2asn', 'dustmasker', 'mafft']
            if not any(name in x for x in diff):
                subprocess.Popen([name], stdout=devnull, stderr=devnull).communicate()
            else:
                subprocess.Popen([name, '--version'], stdout=devnull, stderr=devnull).communicate()
    except OSError as e:
        if e.errno == os.errno.ENOENT:
            return False
    return True

def CheckDependencies(input):
    missing = []
    for p in input:
        if which(p) == False:
            missing.append(p)
    if missing != []:
        error = ", ".join(missing)
        log.error("Missing Dependencies: %s.  Please install missing dependencies and re-run script" % (error))
        sys.exit(1)

def fastarename(input, relabel, output):
    from Bio.SeqIO.FastaIO import FastaIterator
    with open(output, 'w') as outfile:
        counter = 1
        for record in FastaIterator(open(input)):
            newName = relabel+str(counter)
            outfile.write(">%s\n%s\n" % (newName, record.seq))
            counter += 1

def fasta_strip_padding(file, output, stripsize=False):
    from Bio.SeqIO.FastaIO import FastaIterator
    with open(output, 'w') as outputfile:
        for record in FastaIterator(gzopen(file)):
            Seq = record.seq.rstrip('N')
            if ';size=' in record.id:
                record.id = record.id.split(';size=')[0]
            outputfile.write(">%s\n%s\n" % (record.id, Seq))

def fastq_strip_padding(file, output):
    from Bio.SeqIO.QualityIO import FastqGeneralIterator
    with open(output, 'w') as outputfile:
        for title, seq, qual in FastqGeneralIterator(open(file)):
            Seq = seq.rstrip('N')
            Qual = qual[:len(Seq)]
            assert len(Seq) == len(Qual)
            outputfile.write("@%s\n%s\n+\n%s\n" % (title, Seq, Qual))

def ReverseComp(input, output):
    with open(output, 'w') as revcomp:
        with open(input, 'r') as fasta:
            for rec in SeqIO.parse(fasta, 'fasta'):
                revcomp.write(">%s\n%s\n" % (rec.id, rec.seq.reverse_complement()))

def guess_csv_dialect(header):
    """ completely arbitrary fn to detect the delimiter
    :type header: str
    :raise ValueError:
    :rtype: csv.Dialect
    """
    possible_delims = "\t,"
    lines = header.split("\n")
    if len(lines) < 2:
        raise ValueError("CSV header must contain at least 1 line")

    dialect = csv.Sniffer().sniff(header, delimiters=possible_delims)
    return dialect


def fasta2list(input):
    seqlist = []
    with open(input, 'r') as inseq:
        for rec in SeqIO.parse(inseq, 'fasta'):
            if not rec.description in seqlist:
                seqlist.append(rec.description)
    return seqlist

def updateMappingFile(mapfile, barcode_count, output):
    with open(output, 'w') as outfile:
        with open(mapfile, 'r') as infile:
            for line in infile:
                line = line.rstrip()
                cols = line.split('\t')
                if line.startswith('#Sample'): #header row, look for DemuxReads
                    if 'DemuxReads' in cols:
                        loc = cols.index('DemuxReads')
                    elif 'phinchID' in cols:
                        loc = cols.index('phinchID')+1
                        cols.insert(loc,'DemuxReads')
                    else:
                        cols.append('DemuxReads')
                        loc = cols.index('DemuxReads')
                    outfile.write('{:}\n'.format('\t'.join(cols)))
                else:
                    if cols[0] in barcode_count:
                        outfile.write('{:}\t{:}\t{:}\n'.format('\t'.join(cols[:loc]), str(barcode_count[cols[0]]), '\t'.join(cols[loc+1:])))

def CreateGenericMappingFile(barcode_dict, revbarcode_dict, fwd_primer, rev_primer, output, barcodes_found):
    with open(output, 'w') as outfile:
        outfile.write('#SampleID\tBarcodeSequence\tLinkerPrimerSequence\tRevBarcodeSequence\tReversePrimer\tphinchID\tDemuxReads\tTreatment\n')
        for k,v in natsorted(barcodes_found.items()):
            sample = k
            Fsample,Rsample = (None,)*2
            if ':-:' in sample:
                Fsample, Rsample = sample.split(':-:')
            count = v
            forbarcode, revbarcode = ('no_data',)*2
            if Fsample:
                if Fsample in barcode_dict:
                    forbarcode = barcode_dict[Fsample]
            else:
                if sample in barcode_dict:
                    forbarcode = barcode_dict[sample]
            if Rsample:
                if Rsample in revbarcode_dict:
                    revbarcode = revbarcode_dict[Rsample]
            else:
                if sample in revbarcode_dict:
                    revbarcode = revbarcode_dict[sample]
            outfile.write('%s\t%s\t%s\t%s\t%s\t%s\t%i\t%s\n' % (sample, forbarcode, fwd_primer, revbarcode, rev_primer, sample, count, 'no_data'))


def CreateGenericMappingFileIllumina(samples, fwd_primer, rev_primer, output, barcodes):
    with open(output, 'w') as outfile:
        outfile.write('#SampleID\tBarcodeSequence\tLinkerPrimerSequence\tRevBarcodeSequence\tReversePrimer\tphinchID\tDemuxReads\tTreatment\n')
        for k,v in natsorted(list(samples.items())):
            count = barcodes.get(k, 0)
            if count > 0:
                if '-' in v:
                    forbarcode,revbarcode = v.split('-')
                else:
                    forbarcode = v
                    revbarcode = 'no_data'
                outfile.write('%s\t%s\t%s\t%s\t%s\t%s\t%i\t%s\n' % (k, forbarcode, fwd_primer, revbarcode, rev_primer, k, int(count), "no_data"))

def parseMappingFile(input, output):
    '''
    function to parse mapping file pull out primers and barcode sequences
    '''
    fwdprimer = ''
    revprimer = ''
    with open(output, 'w') as outfile:
        with open(input, 'r') as inputfile:
            for line in inputfile:
                line = line.replace('\n', '')
                if line.startswith('#'):
                    continue
                cols = line.split('\t')
                outfile.write('>%s\n%s\n' % (cols[0], cols[1]))
                match = edlib.align(cols[1], cols[2], mode="HW", k=0)
                if match["editDistance"] == 0:
                    Trim = match["locations"][0][1]+1
                    if fwdprimer == '':
                        fwdprimer = cols[2][Trim:]
                        revprimer = cols[3]
    return (fwdprimer, revprimer)

def getMappingHeaderIndexes(input):
    IDx,FBCx,RBCx,FPx,RPx = (None,)*5
    with open(input, 'r') as infile:
        for line in infile:
            line = line.rstrip('\n')
            if line.startswith('#'):
                cols = line.split('\t')
                if '#SampleID' in cols:
                    IDx = cols.index('#SampleID')
                if 'BarcodeSequence' in cols:
                    FBCx = cols.index('BarcodeSequence')
                if 'LinkerPrimerSequence' in cols:
                    FPx = cols.index('LinkerPrimerSequence')
                if 'RevBarcodeSequence' in cols:
                    RBCx = cols.index('RevBarcodeSequence')
                if 'ReversePrimer' in cols:
                    RPx = cols.index('ReversePrimer')
    return IDx, FBCx, FPx, RBCx, RPx

exampleMapFile='#SampleID\tBarcodeSequence\tLinkerPrimerSequence\tRevBarcodeSequence\tReversePrimer\tphinchID\tTreatment\n'

def parseMappingFileNEW(input):
    '''
    function to parse mapping file pull out primers and barcode sequences
    '''
    results = {}
    ForBCDict = {}
    RevBCDict = {}
    IDx, FBCx, FPx, RBCx, RPx = getMappingHeaderIndexes(input)
    if not any([IDx, FBCx, FPx, RPx]):
        log.error('Mapping file incorrectly formatted, headers should be (RevBarcodeSequence is optional):\n{:}'.format(exampleMapFile))
        sys.exit(1)
    if not RBCx:
        log.debug('Mapping file missing header: "RevBarcodeSequence", skipping reverse barcodes')
    with open(input, 'r') as inputfile:
        for line in inputfile:
            line = line.rstrip()
            if line.startswith('#'):
                continue
            cols = line.split('\t')
            if len(cols) < 4:
                continue
            ID = cols[IDx]
            FBC = cols[FBCx]
            FP = cols[FPx]
            if FBC in FP: #barcode nested in primer_db
                loc = FP.index(FBC) + len(FBC)
                FP = FP[loc:]
            RP = cols[RPx]
            if RBCx:
                RBC = cols[RBCx]
                if RBC != '':
                    if RBC in RP:
                        loc = RP.index(RBC) + len(RBC)
                        RP = RP[loc:]
                else:
                    RBC = None
            else:
                RBC = None
            if not ID in results:
                results[ID] = {'ForBarcode': FBC, 'ForPrimer': FP, 'RevBarcode': RBC, 'RevPrimer': RP}
                ForBCDict[ID] = FBC
                if RBC:
                    RevBCDict[ID] = RBC
            else:
                log.error('Please fix duplicate SampleID detected in mapping file: {:}'.format(ID))
                sys.exit(1)
    if len(results) < 1:
        log.error('Mapping file has no parsable lines: {}'.format(input))
        sys.exit(1)
    return results, ForBCDict, RevBCDict, FP, RP

def parseMappingFileIllumina(input):
    fwdprimer = ''
    revprimer = ''
    samples = []
    with open(input, 'r') as inputfile:
        for line in inputfile:
            line = line.replace('\n', '')
            if line.startswith('#'):
                continue
            cols = line.split('\t')
            if not cols[0] in samples:
                samples.append(cols[0])
            match = edlib.align(cols[1], cols[2], mode="HW", k=0)
            if match["editDistance"] == 0:
                Trim = match["locations"][0][1]+1
                if fwdprimer == '':
                    fwdprimer = cols[2][Trim:]
                    revprimer = cols[3]
            else:
                fwdprimer = cols[2]
                revprimer = cols[3]
        return (samples, fwdprimer, revprimer)

def removefile(input):
    if os.path.isfile(input):
        os.remove(input)
