import os

import time as tt

import zipfile as zf
import collections as cx
import random as rn

import numpy  as np
import pandas as pd
import edlib  as ed

from ShareDB import ShareDB

from . import utils as ut
from . import phiX  as px


# Failure Categories for Read Counting

ACOUNT_FAILURE_CATEGORIES = (
    'phix_match',
    'low_complexity',
    'anchor_missing',
    'barcode_prefix_missing',
    'barcode_suffix_missing',
    'barcode_absent',
    'barcode_ambiguous',
    'associate_prefix_missing',
    'associate_suffix_missing',
    'associate_mismatch',
    'callback_false',
    'incalculable',
)

XCOUNT_FAILURE_CATEGORIES = (
    'phix_match',
    'low_complexity',
    'anchor_missing',
    'barcode_prefix_missing',
    'barcode_suffix_missing',
    'barcode_absent',
    'barcode_ambiguous',
    'callback_false',
    'incalculable',
)


# Parser and Setup Functions

def get_random_DNA(length):
    '''
    Return a random DNA sequence
    of required length.
    Internal use only.

    :: length
       type - integer
       desc - length of required
              DNA sequence
    '''

    return ''.join(np.random.choice(
        ('A', 'T', 'G', 'C')) for _ in range(length))

def get_meta_info(packfile):
    '''
    Compute the number of reads to
    stream for callback parsing, and
    extract the meta read pack.

    :: packfile
       type - string
       desc - path to packfile storing
              read packs
    '''

    # Open Pack File
    packfile = ut.get_archive(
        arcfile=packfile)

    # List all packs
    allpacknames = [pkname for pkname in packfile.namelist() \
        if pkname != 'packing.stat']

    # Load Header Pack
    packzero = ut.loadpack(
        archive=packfile,
            pfile=allpacknames[0])

    # Close Packfile
    packfile.close()

    # Determine Stream Count
    count = min(1000, len(packzero))

    # Return Results
    return (packzero,
        count)

def stream_random_reads(count):
    '''
    Stream several random read tuples.
    Internal use only.

    :: count
       type - integer
       desc - total number of reads
              to be streamed
    '''

    for _ in range(count):
        r1 = get_random_DNA(
            length=np.random.randint(10, 1000))
        # 50% chance of having r2 (simulating concat vs merged)
        if np.random.randint(2) == 0:
            r2 = get_random_DNA(
                length=np.random.randint(10, 1000))
        else:
            r2 = None
        yield r1, r2

def stream_packed_reads(packzero, count):
    '''
    Stream several experiment read tuples.
    Internal use only.

    :: packfile
       type - string
       desc - meta read pack containing
              most frequent reads
    :: count
       type - integer
       desc - total number of reads
              to be streamed
    '''

    # Build Streaming Index Vector
    indexvec = np.arange(len(packzero))
    np.random.shuffle(indexvec)
    indexvec = indexvec[:count]

    # Read Streaming Loop
    for readidx in indexvec:
        # Pack format: [[r1, r2], count]
        read_tuple = packzero[readidx][0]
        r1 = read_tuple[0]
        r2 = read_tuple[1]  # None for merged reads
        yield r1, r2

def stream_IDs(indexfiles, count):
    '''
    Stream random ID combination.
    Internal use only.

    :: indexfiles
       type - tuple
       desc - tuple of file paths
              to applied indices
    :: count
       type - integer
       desc - total number of reads
              to be streamed
    '''

    # Open Index Files
    indexfiles = list(
        ut.get_archive(idxfile) for idxfile in indexfiles)

    # Load IDdics
    IDdicts = [ut.loaddict(
        archive=indexfile,
        dfile='ID.map') for indexfile in indexfiles]

    # Close Index Files
    for indexfile in indexfiles:
        indexfile.close()

    # Stream Random ID Combination
    for _ in range(count):

        # Build ID Tuple
        IDtuple = []
        for IDdict in IDdicts:
            if len(indexfiles) == 1:
                entry = IDdict[
                    np.random.randint(len(IDdict))]
            else:
                toss = np.random.randint(10)
                if toss == 0:
                    entry = '-'
                else:
                    entry = IDdict[
                        np.random.randint(len(IDdict))]
            IDtuple.append(entry)

        # Stream ID Tuple
        yield tuple(IDtuple)

def get_parsed_callback(
    indexfiles,
    callback,
    packfile,
    ncores,
    liner):
    '''
    Determine if given callback
    function is valid.
    Internal use only.

    :: indexfiles
       type - tuple
       desc - tuple of file paths
              to applied indices
    :: callback
       type - function
       desc - callback function
              specified
    :: packfile
       type - string
       desc - path to packfile storing
              read packs
    :: ncores
       type - integer
       desc - total number of cores to
              be used in counting
    :: liner
       type - coroutine
       desc - dynamic printing
    '''

    # Start Timer
    t0 = tt.time()

    # Show Update
    liner.send(' Loading Read Generators ...')

    # Get Meta Information
    (packzero,
    count) = get_meta_info(
        packfile=packfile)

    # Setup Streamers
    randstreamer = stream_random_reads(
        count=count)
    packstreamer = stream_packed_reads(
        packzero=packzero,
        count=count)
    streamers  = [randstreamer, packstreamer]
    IDstreamer = stream_IDs(
        indexfiles=indexfiles,
        count=count)

    # Parsing Loop
    pass_count   = 0
    failedinputs = []
    for readcount in range(count):

        # Get a Read Tuple!
        r1, r2 = next(streamers[np.random.randint(2)])

        # Execute Callback
        try:
            callback_input = {
                    'r1': r1,
                    'r2': r2,
                    'ID': next(IDstreamer),
                 'count': np.random.randint(1, 10001),
                'coreid': np.random.randint(ncores)}
            result = callback(**callback_input)
            if not isinstance(result,bool):
                raise
        except:
            failedinputs.append(callback_input)
            iter_valid  = False
        else:
            iter_valid  = True
            pass_count += 1

        # Show Update
        liner.send(
            ' Callback Evaluation: {:,} / 1,000 {}'.format(
                readcount,
                ('Rejected', 'Accepted')[iter_valid]))

    # Close Generators
    randstreamer.close()
    packstreamer.close()
    IDstreamer.close()

    # Final Update
    plen = ut.get_printlen(
        value=count)
    liner.send(
        ' Callback Passed: {:{},} Input(s)\n'.format(
            pass_count,
            plen))
    liner.send(
        ' Callback Failed: {:{},} Input(s)\n'.format(
            count - pass_count,
            plen))
    liner.send(
        ' Time Elapsed: {:.2f} sec\n'.format(
            tt.time() - t0))

    # Show Final Verdict
    parsestatus = len(failedinputs) == 0
    if not parsestatus:
        liner.send(
            ' Verdict: Counting Infeasible due to Callback Function\n')
    else:
        liner.send(
            ' Verdict: Counting Possibly Feasible\n')

    # Cleanup
    del packzero

    # Return results
    return (parsestatus,
        failedinputs)

def pack_loader(
    packfile,
    packqueue,
    enqueuecomplete,
    ncores,
    liner):
    '''
    Enqueue all read pack names from
    packfile. Internal use only.

    :: packfile
       type - string
       desc - filename storing read packs
    :: packqueue
       type - SafeQueue
       desc - queue storing read pack file
              paths
    :: enqueuecomplete
       type - mp.Event
       desc - multiprocessing Event set
              when all pack names stored
              in pack queue
    :: ncores
       type - integer
       desc - total number of counting
              processes engaged
    :: liner
       type - coroutine
       desc - dynamic printing
    '''

    # Start Timer
    t0 = tt.time()

    # Show Update
    liner.send(' Loading Read Packs ...')

    # Open Archive
    archive = ut.get_archive(
        arcfile=packfile)

    # Enque Read Packs
    packqueue.multiput(map(
        lambda x: ut.removestarfix(
            string=x,
            fix='.pack',
            loc=1),
        filter(
            lambda arcentry: arcentry.endswith('.pack'),
            archive.namelist())))

    # Show Final Updates
    liner.send(
        ' Pack Queue  : {:,} Read Pack(s) Loaded\n'.format(
            len(packqueue)))
    liner.send(
        ' Time Elapsed: {:.2f} sec\n'.format(
            tt.time() - t0))

    # Insert Exhaustion Tokens
    packqueue.multiput((None for _ in range(ncores)))

    # Unblock Queue
    enqueuecomplete.set()

# Engine Helper Functions

def exoneration_procedure(
    exoread,
    exofreq,
    phiXkval,
    phiXspec,
    cctrs,
    sampler=None,
    r1=None,
    r2=None,
    anchor_diagnostic=None):
    '''
    Determine exoneration for given
    read and update core counters.
    Internal use only.

    :: exoread
       type - string
       desc - read to be exonerated
    :: exofreq
       type - integer
       desc - absolute frequency of
              the read in given pack
    :: phiXkval
       type - integer
       desc - k-value for phiX matching
    :: phiXspec
       type - set
       desc - set of all possible phiX
              k-mers (includes revcomp)
    :: cctrs
       type - dict
       desc - dictionary of core counters
    :: sampler
       type - FailureSampler / None
       desc - optional failure sampler for
              collecting failed read samples
    :: r1
       type - string / None
       desc - read 1 for failure sampling
    :: r2
       type - string / None
       desc - read 2 for failure sampling
    :: anchor_diagnostic
       type - dict / None
       desc - diagnostic info for anchor_missing
    '''

    # Note: exoread is R1 extracted from (r1, r2) tuple in count engines

    # Exoneration Flag
    exonerated = False
    matched_kmer = None

    # PhiX Match?
    for kmer in ut.stream_spectrum(
        seq=exoread,
        k=phiXkval):

        if kmer in phiXspec:
            cctrs['phix_reads'] += exofreq
            exonerated = True
            matched_kmer = kmer
            break

    # Sample PhiX match
    if exonerated and sampler is not None:
        sampler.add_sample(
            category='phix_match',
            r1=r1 if r1 else exoread,
            r2=r2,
            freq=exofreq,
            diagnostic={'phix_matched_kmer': matched_kmer})
        return

    # Nucleotide Composition
    acount = exoread.count('A')
    tcount = exoread.count('T')
    gcount = exoread.count('G')
    ccount = exoread.count('C')
    rlen   = len(exoread) or 1

    # Dinucleotide Match?
    if not exonerated:
        # Composition Largely Dinucleotide?
        dinukethresh = 0.75 * len(exoread)
        if (ccount + tcount >= dinukethresh) or \
           (acount + ccount >= dinukethresh) or \
           (ccount + gcount >= dinukethresh) or \
           (acount + gcount >= dinukethresh) or \
           (acount + tcount >= dinukethresh) or \
           (tcount + gcount >= dinukethresh):
            cctrs['lowcomplexreads'] += exofreq
            exonerated = True
            if sampler is not None:
                sampler.add_sample(
                    category='low_complexity',
                    r1=r1 if r1 else exoread,
                    r2=r2,
                    freq=exofreq,
                    diagnostic={
                        'A_percent': round(100.0 * acount / rlen, 2),
                        'T_percent': round(100.0 * tcount / rlen, 2),
                        'G_percent': round(100.0 * gcount / rlen, 2),
                        'C_percent': round(100.0 * ccount / rlen, 2)})
            return

    # Mononucleotide Match?
    if not exonerated:
        # Composition Large Mononucleotide?
        mononukethresh = 0.50 * len(exoread)
        if ccount >= mononukethresh or \
           acount >= mononukethresh or \
           gcount >= mononukethresh or \
           tcount >= mononukethresh:
            cctrs['lowcomplexreads'] += exofreq
            exonerated = True
            if sampler is not None:
                sampler.add_sample(
                    category='low_complexity',
                    r1=r1 if r1 else exoread,
                    r2=r2,
                    freq=exofreq,
                    diagnostic={
                        'A_percent': round(100.0 * acount / rlen, 2),
                        'T_percent': round(100.0 * tcount / rlen, 2),
                        'G_percent': round(100.0 * gcount / rlen, 2),
                        'C_percent': round(100.0 * ccount / rlen, 2)})
            return

    # Trinucleotide Match?
    if not exonerated:
        # Composition Largely Dinucleotide?
        trinukethresh = 1.00 * len(exoread)
        if (ccount + acount + tcount >= trinukethresh) or \
           (ccount + acount + gcount >= trinukethresh) or \
           (acount + tcount + gcount >= trinukethresh) or \
           (tcount + ccount + gcount >= trinukethresh):
            cctrs['lowcomplexreads'] += exofreq
            exonerated = True
            if sampler is not None:
                sampler.add_sample(
                    category='low_complexity',
                    r1=r1 if r1 else exoread,
                    r2=r2,
                    freq=exofreq,
                    diagnostic={
                        'A_percent': round(100.0 * acount / rlen, 2),
                        'T_percent': round(100.0 * tcount / rlen, 2),
                        'G_percent': round(100.0 * gcount / rlen, 2),
                        'C_percent': round(100.0 * ccount / rlen, 2)})
            return

    # Uncategorized Discard
    if not exonerated:
        cctrs['incalcreads'] += exofreq
        if sampler is not None:
            sampler.add_sample(
                category='anchor_missing',
                r1=r1 if r1 else exoread,
                r2=r2,
                freq=exofreq,
                diagnostic=anchor_diagnostic)

def get_anchored_read_candidates(
    read,
    metamap):
    '''
    Return anchored read segment candidates
    sorted by edit distance (best first).
    Internal use only.

    :: read
       type - string
       desc - read to be anchored
    :: metamap
       type - dict
       desc - dictionary containing index
              meta information
    '''

    cands = []

    # Too Short of a Read!
    if len(read) < (len(metamap['anchor']) - metamap['anchortval']):
        return cands

    # Helper: split a read into per-anchor segments so each candidate is evaluated
    # against exactly one anchor occurrence (avoids rfind/find always picking the
    # same copy when reads contain multiple concatenated molecules).
    def _segments_from_spans(oriented_read, spans, policy):
        if not spans:
            return []

        spans = sorted(spans, key=lambda x: x[0])

        # Anchor is suffix-only (barcodeprefix is None): barcode is upstream of anchor.
        if policy == 2:
            segments = []
            prev_end = 0
            for _, end in spans:
                segments.append(oriented_read[prev_end:])
                prev_end = end + 1
            return segments

        # Anchor is prefix (policy 1 or 1.5): barcode is downstream of anchor.
        segments = []
        starts = [s for s, _ in spans]
        for idx, start in enumerate(starts):
            stop = starts[idx + 1] if idx + 1 < len(starts) else len(oriented_read)
            segments.append(oriented_read[start:stop])
        return segments

    # Quick Anchor Check!
    fcount = read.count(metamap['anchor'])
    rcount = read.count(metamap['revanchor'])
    trimpolicy = metamap.get('trimpolicy')

    # Fast Path: Single Exact Match
    if fcount == 1 and rcount == 0:
        spans = [(read.find(metamap['anchor']),
                  read.find(metamap['anchor']) + len(metamap['anchor']) - 1)]
        segs = _segments_from_spans(read, spans, trimpolicy)
        return [(segs[0], 0, 0)] if segs else []
    if rcount == 1 and fcount == 0:
        rcread = ut.get_revcomp(seq=read)
        pos = rcread.find(metamap['anchor'])
        spans = [(pos, pos + len(metamap['anchor']) - 1)] if pos >= 0 else []
        segs = _segments_from_spans(rcread, spans, trimpolicy)
        return [(segs[0], 0, 1)] if segs else []

    # Ambiguous Exact Match (Both Orientations)
    if fcount > 0 and rcount > 0:
        return []

    # Fast Path: Multiple Exact Matches (Same Orientation)
    if fcount > 1:
        spans = []
        start = 0
        while True:
            idx = read.find(metamap['anchor'], start)
            if idx < 0:
                break
            spans.append((idx, idx + len(metamap['anchor']) - 1))
            start = idx + 1
        segs = _segments_from_spans(read, spans, trimpolicy)
        return [(seg, 0, 0) for seg in segs]
    if rcount > 1:
        rcread = ut.get_revcomp(seq=read)
        spans = []
        start = 0
        while True:
            idx = rcread.find(metamap['anchor'], start)
            if idx < 0:
                break
            spans.append((idx, idx + len(metamap['anchor']) - 1))
            start = idx + 1
        segs = _segments_from_spans(rcread, spans, trimpolicy)
        return [(seg, 0, 1) for seg in segs]

    # Fuzzy Matching via edlib (No Exact Matches)
    # Compute Forward Anchor Alignment
    alnf = ed.align(
        query=metamap['anchor'],
        target=read,
        mode='HW',
        task='locations',
        k=metamap['anchortval'])

    # Store Forward Candidates
    if alnf['editDistance'] >= 0 and alnf['locations']:
        segs = _segments_from_spans(read, alnf['locations'], trimpolicy)
        for seg in segs:
            cands.append((seg, alnf['editDistance'], 0))

    # Compute Reverse Anchor Alignment
    rcread = ut.get_revcomp(seq=read)
    alnr = ed.align(
        query=metamap['anchor'],
        target=rcread,
        mode='HW',
        task='locations',
        k=metamap['anchortval'])

    # Store Reverse Candidates
    if alnr['editDistance'] >= 0 and alnr['locations']:
        segs = _segments_from_spans(rcread, alnr['locations'], trimpolicy)
        for seg in segs:
            cands.append((seg, alnr['editDistance'], 1))

    # Check for Orientation Ambiguity
    if cands:
        fwd_scores = [c[1] for c in cands if c[2] == 0]
        rev_scores = [c[1] for c in cands if c[2] == 1]
        if fwd_scores and rev_scores:
            # Both orientations found - ambiguous if equal best scores
            if min(fwd_scores) == min(rev_scores):
                return []
            # Keep only better orientation
            if min(fwd_scores) < min(rev_scores):
                cands = [c for c in cands if c[2] == 0]
            else:
                cands = [c for c in cands if c[2] == 1]

    # Sort by Edit Distance
    cands.sort(key=lambda x: x[1])

    return cands

def get_barcode_candidate(
    ancread,
    metamap):
    '''
    Extract and return the barcode candidate window
    from an anchored read segment without classifying
    it. Returns None on trim failure or empty window.
    Internal use only.

    :: ancread
       type - string
       desc - anchored read segment to extract
              barcode candidate from
    :: metamap
       type - dict
       desc - dictionary containing index meta
              information (trimpolicy, barcodeprefix,
              barcodesuffix, barcodelen, etc.)
    '''

    window, status = get_barcode_index(
        barcoderead=ancread,
        metamap=metamap,
        model=None)
    if status != 'success':
        return None
    return window or None

def get_best_barcode_from_candidates(
    read,
    metamap,
    model):
    '''
    Return best barcode from multiple
    anchor positions in given read.
    Internal use only.

    :: read
       type - string
       desc - read to process
    :: metamap
       type - dict
       desc - dictionary containing index
              meta information
    :: model
       type - Scry
       desc - Scry model for classifying
              barcode
    '''

    # Get All Anchor Candidates
    cands = get_anchored_read_candidates(read, metamap)

    # No Anchors Found!
    if not cands:
        return None, 'anchor_missing', None

    # Filter to Best Anchor Score
    bestscore = min(c[1] for c in cands)
    bestcands = [c for c in cands if c[1] == bestscore]

    # Classify Barcode at Each Position
    bcresults     = []
    fail_statuses = []
    for ancread, _, __ in bestcands:
        barcode_result, get_status = get_barcode_index(
            barcoderead=ancread,
            metamap=metamap,
            model=model)

        # Trim or gap failure
        if get_status != 'success':
            fail_statuses.append(get_status)
            continue

        # Extract Index and Confidence
        if isinstance(barcode_result, tuple):
            bcidx, conf = barcode_result
        else:
            bcidx = barcode_result
            conf  = 1.0

        # Store Valid Barcode
        if bcidx is not None:
            if conf is None:
                conf = 1.0
            bcresults.append((bcidx, conf))
        else:
            fail_statuses.append('barcode_unclassifiable')

    # No Barcodes Found!
    if not bcresults:
        best_ancread  = bestcands[0][0] if bestcands else None
        candidate     = get_barcode_candidate(best_ancread, metamap) if best_ancread else None
        trim_failures = [s for s in fail_statuses
                         if s in ('prefix_missing', 'suffix_missing')]
        primary_status = trim_failures[0] if trim_failures else 'barcode_absent'
        return None, primary_status, {
            'anchor_hit_count':            len(bestcands),
            'anchor_best_edit_distance':   bestscore,
            'extracted_barcode_candidate': candidate}

    # Find Best Scry Score
    bestconf = max(r[1] for r in bcresults)
    bestbcs  = [r for r in bcresults if r[1] == bestconf]

    # Check for Ambiguity
    uniqueids = set(r[0] for r in bestbcs)
    if len(uniqueids) > 1:
        best_ancread = bestcands[0][0] if bestcands else None
        candidate    = get_barcode_candidate(best_ancread, metamap) if best_ancread else None
        return None, 'barcode_ambiguous', {
            'anchor_hit_count':            len(bestcands),
            'tied_indices':                list(uniqueids),
            'extracted_barcode_candidate': candidate}

    # Unique Best Barcode Found!
    return bestbcs[0][0], 'success', None

def get_trimmed_read(
    read,
    const,
    constype,
    constval):
    '''
    Return read after trimming
    constant sequence.
    Internal use only.

    :: read
       type - string
       desc - read to be trimmed
    :: const
       type - string / None
       desc - constant to trim
    :: constype
       type - integer
       desc - constant type identifier
              0 = prefix constant
              1 = suffix constant
    :: constval
       type - integer / None
       desc - constant t-value to be
              tolerated for matching
    '''

    # Nothing to Trim!
    if const is None:
        return read, True
    if len(read) == 0:
        return read, False

    # Too Short of a Read!
    if len(read) < (len(const) - constval):
        return read, False

    # Quick Find!
    if constype <= 0:
        start = read.rfind(const)
    else:
        start = read.find(const)

    # Match Found!
    if start > -1:

        # Compute Extraction Coordinates
        if constype <= 0:
            start = start + len(const)
            stop  = len(read)
        else:
            stop  = start
            start = 0

    # No Direct Match ..
    else:

        # Compute Constant Alignment
        aln = ed.align(
            query=const,
            target=read,
            mode='HW',
            task='locations',
            k=constval)

        # Constant Absent
        if aln['editDistance'] == -1:
            return read, False

        # Extract Locations
        locs = aln['locations']

        # Compute Extraction Coordinates
        if constype <= 0:
            start = locs[-1][-1]+1
            stop  = len(read)
        else:
            stop  = locs[+0][+0]+0
            start = 0

    # Compute and Return Trimmed Read
    return read[start:stop], True

def get_barcode_index(
    barcoderead,
    metamap,
    model):
    '''
    Determine and return barcode identifier
    index in given anchored read.
    Internal use only.

    :: barcoderead
       type - string
       desc - read containing barcode
              information
    :: metamap
       type - dict
       desc - ditionary containing index
              meta information
    :: model
       type - Scry
       desc - Scry model for classifying
              barcode
    '''

    # Book-keeping
    trimpolicy   = metamap['trimpolicy']
    pxtrimstatus = True
    sxtrimstatus = True
    trimstatus   = True

    # Prefix Trim
    if trimpolicy <= 1.5:
        # Prefix Trim Read
        barcoderead, pxtrimstatus = get_trimmed_read(
            read=barcoderead,
            const=metamap['barcodeprefix'],
            constype=0,
            constval=metamap['bpxtval'])
        # Trim was Unsuccessful
        if (trimpolicy == 1) and (pxtrimstatus is False):
            return None, 'prefix_missing'

    # Suffix Trim
    if trimpolicy >= 1.5:
        # Suffix Trim Read
        barcoderead, sxtrimstatus = get_trimmed_read(
            read=barcoderead,
            const=metamap['barcodesuffix'],
            constype=1,
            constval=metamap['bsxtval'])
        # Trim was Unsuccessful
        if (trimpolicy == 2) and (sxtrimstatus is False):
            return None, 'suffix_missing'

    # Policy Trim
    if trimpolicy == 1:
        barcoderead = barcoderead[:+(metamap['barcodelen'] + metamap['barcodetval']) ]
    if trimpolicy == 2:
        barcoderead = barcoderead[ -(metamap['barcodelen'] + metamap['barcodetval']):]
    if trimpolicy == 1.5:
        trimstatus = pxtrimstatus and sxtrimstatus
        if not trimstatus:
            if not pxtrimstatus:
                return None, 'prefix_missing'
            return None, 'suffix_missing'

    # Gap Trim
    if metamap['barcodegapped']:
        # Localise Gap Lengths
        pregap, postgap = (metamap['barcodepregap'],
            metamap['barcodepostgap'])
        # Pregap Trim
        if pregap:
            barcoderead = barcoderead[ +pregap:]
        # Postgap Trim
        if postgap:
            barcoderead = barcoderead[:-postgap]
        # Nothing Remains after Gap Trim
        if not barcoderead:
            return None, 'barcode_absent'

    # Return Trimmed Window or Classify
    if model is None:
        return barcoderead, 'success'
    return model.predict(x=barcoderead)[0], 'success'

def is_associate_match(
    associate,
    associateread,
    associatetval):
    '''
    Determine if associate exists
    in given anchored read.
    Internal use only.

    :: associate
       type - string
       desc - full associate sequence
              to match
    :: associateread
       type - string
       desc - read containing associate
              information
    :: associatetval
       type - integer
       desc - associate t-value to be
              tolerated for matching
    '''

    # Query-Reference Adjustment
    if len(associateread) <= len(associate):
        query  = associateread
        target = associate
    else:
        query  = associate
        target = associateread

    # Quick Match!
    if query in target:
        return True

    # Compute Associate Alignment
    aln = ed.align(
        query=query,
        target=target,
        mode='HW',
        task='distance',
        k=associatetval)

    # Return Results
    if aln['editDistance'] == -1:
        return False
    return True

def get_associate_match(
    associateread,
    associateerrors,
    associatedict,
    index,
    metamap):
    '''
    Process associate read and determine if
    it contains required associate.
    Internal use only.

    :: associateread
       type - string
       desc - read containing associate
              information
    :: associaterrors
       type - integer
       desc - maximum number of mismatches
              between reference and read
              associate
    :: associatedict
       type - dict
       desc - dictionary containing associate
              information from index
    :: index
       type - integer
       desc - associate index identifier
    :: metamap
       type - dict
       desc - ditionary containing index
              meta information
    '''

    # Trim Associate Prefix
    associateread, trimstatus = get_trimmed_read(
        read=associateread,
        const=metamap['associateprefix'],
        constype=0,
        constval=metamap['apxtval'])

    # Associate Prefix Absent
    if not trimstatus:
        return False, 0, 'prefix'

    # Trim Associate Suffix
    associateread, trimstatus = get_trimmed_read(
        read=associateread,
        const=metamap['associatesuffix'],
        constype=1,
        constval=metamap['asxtval'])

    # Associate Suffix Absent
    if not trimstatus:
        return False, 0, 'suffix'

    # Gap Trim
    if metamap.get('associategapped'):
        # Localise Gap Lengths
        pregap  = metamap.get('associatepregap', 0)
        postgap = metamap.get('associatepostgap', 0)
        # Pregap Trim
        if pregap:
            associateread = associateread[+pregap:]
        # Postgap Trim
        if postgap:
            associateread = associateread[:-postgap]
        # Nothing Remains after Gap Trim
        if not associateread:
            return False, 0, 'suffix'

    # Match Associate
    associate, associatetval = associatedict[index]
    if (associateerrors > -1) and \
       (associateerrors < associatetval):
        associatetval = associateerrors

    # Return Results
    return (is_associate_match(
        associate=associate,
        associateread=associateread,
        associatetval=associatetval),
        1, None)

def get_failed_inputs(
    packqueue,
    countdir,
    liner):
    '''
    Empty packqueue and return failed inputs
    for callback. Internal use only.

    :: packqueue
       type - SafeQueue
       desc - queue storing read pack file
              paths
    :: countdir
       type - string
       desc - filepath to counting workspace
              directory
    :: liner
       type - coroutine
       desc - dynamic printing
    '''

    # Show Update
    liner.send(
        ' Emptying Pack Queue ...')

    # Consume Pack Queue
    for item in packqueue.multiget():
        pass

    # Show Update
    liner.send(
        ' Extracting Failed Input(s) ...')

    # Aggregate Callback Dumps
    failedinputs = []
    for entry in os.listdir(countdir):
        if entry.endswith('.callback.dump'):
            failedinputs.append(
                ut.loaddump(dfile='{}/{}'.format(
                    countdir,
                    entry)))

    # Return Results
    return failedinputs

def count_aggregator(
    countqueue,
    countdir,
    prodcount,
    prodactive,
    assoc,
    liner):
    '''
    Aggregate count dictionaries in
    countqueue into a count database.
    Internal use only.

    :: countqueue
       type - SafeQueue
       desc - count queue storing
              count dictionaries
    :: countdir
       type - string
       desc - path to work space
              count directory
    :: prodcount
       type - integer
       desc - total number of producers
              scheduled to add to
              countqueue
    :: prodactive
       type - SafeCounter
       desc - total number of producers
              actively adding to
              countqueue
    :: assoc
       type - boolean
       desc - if True, we are aggregating
              association counts of tuples
    :: liner
       type - coroutine
       desc - dynamic printing and
              logging
    '''

    # Define CountDB file
    countdb = '{}/countdb'.format(countdir)

    # Open CountDB Instance
    countdb = ShareDB(path=countdb, map_size=None)

    # Waiting on Count Matrices
    while countqueue.empty():
        tt.sleep(0)
        continue

    # Count Dictionary Aggregation Loop
    while prodcount:

        # Fetch Count Path / Token
        cqtoken = countqueue.get()

        # Exhaustion Token
        if cqtoken is None:
            prodcount -= 1
            continue

        # Build Count Path
        cpath = '{}/{}'.format(countdir, cqtoken)

        # Load Count List
        fname = cqtoken
        countlist = ut.loadcount(cfile=cpath)

        # Remove Count List
        ut.remove_file(filepath=cpath)

        # Show Updates
        if prodactive and \
           prodactive.value() == 0:
            liner.send(' Aggregating: {}'.format(fname))

        # Update Batch
        while countlist:
            k,v = countlist.pop()
            w = countdb.get(k)
            match assoc:
                case True:
                    if w is None:
                        w = [0, 0]
                    v[0] += w[0]
                    v[1] += w[1]
                case False:
                    if w is None:
                        w = 0
                    v += w
            countdb[k] = v

        # Update CountDB
        countdb.sync()

        # Release Control
        tt.sleep(0)

def write_count(
    indexfiles,
    countdir,
    countfile,
    assoc,
    liner):
    '''
    Write entries in count database
    to a final count file / matrix.
    Internal use only.

    :: indexfiles
       type - tuple
       desc - tuple of index file
              paths
    :: countdir
       type - string
       desc - path to workspace
              count directory
    :: countfile
       type - string
       desc - path to CSV file storing
              read counts for discovered
              ID combinations
    :: assoc
       type - boolean
       desc - if True, we are writing
              association counts
    :: liner
       type - coroutine
       desc - dynamic printing
    '''

    # Book-keeping
    IDdicts    = []
    indexnames = []
    t0 = tt.time()

    # Show Update
    liner.send(' Loading IDs ...')

    # Load IDdicts
    for indexfile in indexfiles:

        # Update Index Names
        indexnames.append(ut.removestarfix(
            string=indexfile.split('/')[-1],
            fix='.oligopool.index',
            loc=1))

        # Open indexfile
        indexfile = zf.ZipFile(
            file=indexfile)

        # Update IDdict
        IDdicts.append(ut.loaddict(
            archive=indexfile,
            dfile='ID.map'))

        # Close indexfile
        indexfile.close()

    # Show Update
    liner.send(' Writing Count Matrix ...')

    # Define CountDB file
    countdb = '{}/countdb'.format(
        countdir)

    # Open CountDB Instance
    countdb = ShareDB(path=countdb, map_size=None)

    # Count Matrix Loop
    with open(countfile, 'w') as outfile:

        # Write Header
        match assoc:
            case True:
                outfile.write(','.join('{}.ID'.format(
                    idxname) for idxname in indexnames) + ',BarcodeCount,AssociationCount\n')
            case False:
                outfile.write(','.join('{}.ID'.format(
                    idxname) for idxname in indexnames) + ',CombinatorialCount\n')

        # Book-keeping
        entrycount = 0
        batchsize  = 10**4
        batchreach = 0

        # Loop through CountDB Entries
        for indextuple,count in countdb.items():

            # Update Book-keeping
            entrycount += 1
            batchreach += 1

            # Build Entry
            rows = []
            for IDx,index in enumerate(indextuple):
                if index == '-':
                    rows.append('-')
                else:
                    rows.append(IDdicts[IDx][int(index)])
            match assoc:
                case True:
                    rows.append(','.join(str(x) for x in count))
                case False:
                    rows.append(count)

            # Write Entry to Count Matrix
            outfile.write(','.join(map(str,rows)) + '\n')

            # Show Update
            if batchreach == batchsize:
                liner.send(
                    ' Rows Written: {:,} Unique ID / Combination(s)'.format(
                        entrycount))
                batchreach = 0

    # Final Updates
    liner.send(
        ' Rows Written: {:,} Unique ID / Combination(s)\n'.format(
            entrycount))

    # Read Back the CSV
    liner.send(' Reading Count Matrix ...')
    df = pd.read_csv(countfile)

    # Show Time Elapsed
    liner.send(
        ' Time Elapsed: {:.2f} sec\n'.format(
            tt.time() - t0))

    # Return DataFrame
    return df

def acount_engine(
    indexfile,
    packfile,
    packqueue,
    countdir,
    countqueue,
    maptype,
    barcodeerrors,
    associateerrors,
    previousreads,
    analyzedreads,
    phiXreads,
    lowcomplexreads,
    misassocreads,
    falsereads,
    incalcreads,
    experimentreads,
    callback,
    callbackerror,
    coreid,
    batchid,
    ncores,
    nactive,
    memlimit,
    restart,
    shutdown,
    launchtime,
    liner,
    failuresamplequeue=None,
    failed_reads_sample_size=1000):
    '''
    Association count read packs stored in packfile
    and enqueue the final read count matrix in to
    countqueue. Internal use only.

    :: indexfile
       type - string
       desc - archive of indexed objects storing barcode
              and associate data and models
    :: packfile
       type - string
       desc - path to compressed zipfile
              storing read packs
    :: packqueue
       type - SimpleQueue
       desc - queue storing path to read pack
              files stored in packfile
    :: countdir
       type - string
       desc - filepath to counting workspace
              directory
    :: countqueue
       type - SimpleQueue
       desc - queue storing count matrices
              aggregating one or more read
              packs processed by each core
    :: maptype
       type - integer
       desc - mapping type identifier
              0 = fast mapping
              1 = exact mapping
    :: barcodeerrors
       type - Real / None
       desc - maximum number of mutations
              tolerated in barcodes before
              discarding reads from counting
    :: associateerrors
       type - Real / None
       desc - maximum number of mutations
              tolerated in associates before
              discarding reads from counting
    :: previousreads
       type - SafeCounter
       desc - total number of reads processed
              previously
    :: analyzedreads
       type - SafeCounter
       desc - total number of reads processed
              so far during counting
    :: phiXreads
       type - SafeCounter
       desc - total number of reads processed
              attributed to PhiX contamination
    :: lowcomplexreads
       type - SafeCounter
       desc - total number of reads processed
              attributed to low complexity products
    :: misassocreads
       type - SafeCounter
       desc - total number of reads processed
              attributed to mis-association
    :: falsereads
       type - SafeCounter
       desc - total number of reads processed
              that were rejected by callback function
    :: incalcreads
       type - SafeCounter
       desc - total number of invalid reads or ones
              not usable for quantification
    :: experimentreads
       type - SafeCounter
       desc - total number of valid reads used for
              quantification
    :: callback
       type - function
       desc - callback function to invoke during
              counting for concurrent processing
    :: callbackerror
       type - mp.Event
       desc - If True, signals error in callback
              execution
    :: coreid
       type - integer
       desc - current core integer id
    :: batchid
       type - integer
       desc - current batch integer id
    :: ncores
       type - integer
       desc - total number of packers
              concurrently initiated
    :: nactive
       type - SafeCounter
       desc - total number of packers
              concurrently active
    :: memlimit
       type - nu.Real
       desc - total amount of memory
              allowed per core
    :: restart
       type - mp.Event
       desc - multiprocessing event when
              process needs to restart
    :: shutdown
       type - mp.Event
       desc - multiprocessing event when
              process needs to shutdown
    :: launchtime
       type - time
       desc - initial launch timestamp
    :: liner
       type - coroutine
       desc - dynamic printing
    :: failuresamplequeue
       type - SafeQueue / None
       desc - queue for storing failure sample
              file paths (None = disabled)
    :: failed_reads_sample_size
       type - integer
       desc - maximum samples per failure category
    '''

    # Open indexfile
    indexfile = ut.get_archive(
        arcfile=indexfile)

    # Load Barcode Model
    model = ut.loadmodel(
        archive=indexfile,
        mfile='barcode.model')

    # Prime Barcode Model
    model.prime(
        t=barcodeerrors,
        mode=maptype)

    # Load Maps
    IDdict = ut.loaddict(
        archive=indexfile,
        dfile='ID.map')
    metamap = ut.loaddict(
        archive=indexfile,
        dfile='meta.map')
    associatedict = ut.loaddict(
        archive=indexfile,
        dfile='associate.map')

    # Close indexfile
    indexfile.close()

    # Define PhiX Spectrum
    phiXkval = 30
    phiXspec = px.get_phiX_spectrum(
        k=phiXkval)

    # Open packfile
    packfile = ut.get_archive(
        arcfile=packfile)

    # Load packing.stat
    packstat = ut.loaddict(
        archive=packfile,
        dfile='packing.stat')

    # Book-keeping Variables
    cctrs = {
        'previousreads'  : previousreads.value(),
        'analyzedreads'  : 0,
        'phix_reads'     : 0,
        'lowcomplexreads': 0,
        'misassocreads'  : 0,
        'falsereads'     : 0,
        'incalcreads'    : 0,
        'experimentreads': 0}
    callbackabort = False

    # Initialize failure sampler if enabled
    sampler = None
    if failuresamplequeue is not None:
        sampler = ut.FailureSampler(
            sample_size=failed_reads_sample_size,
            categories=ACOUNT_FAILURE_CATEGORIES)

    # Compute Printing Lengths
    clen = ut.get_printlen(value=ncores)
    slen = plen = ut.get_printlen(
        value=packstat['survived_reads'])

    # Read Count Storage
    countdict = cx.defaultdict(lambda: [0, 0])
    countpath = '{}/{}.{}.count'.format(
        countdir, coreid, batchid)

    # Pack Counting Loop
    while not packqueue.empty():

        # Callback Error Somewhere?
        if not callback is None:
            if callbackerror.is_set():
                callbackabort = True
                shutdown.set()
                break

        # Fetch Pack Name / Token
        packname = packqueue.get()

        # Exhaustion Token
        if packname is None:
            shutdown.set()
            break

        # Fetch Read Pack
        cpack = ut.loadpack(
            archive=packfile,
            pfile='{}.pack'.format(
                packname))

        # Book-keeping Variables
        exoread       = None
        exofreq       = None
        exo_r1        = None  # Track r1 for failure sampling
        exo_r2        = None  # Track r2 for failure sampling
        exo_anchor_diag = None  # Anchor diagnostic for exoneration
        readcount = 0

        verbiagereach  = 0
        verbiagetarget = rn.randint(
            *map(round, (len(cpack) * 0.080,
                         len(cpack) * 0.120)))

        # Start Timer
        t0 = tt.time()

        # Read Counting Loop
        while True:

            # Callback Error Somewhere?
            if not callback is None:
                if callbackerror.is_set():
                    callbackabort = True
                    break

            # Exoneration Block
            if not exoread is None:

                # Run Exoneration Procedure
                exoneration_procedure(
                    exoread=exoread,
                    exofreq=exofreq,
                    phiXkval=phiXkval,
                    phiXspec=phiXspec,
                    cctrs=cctrs,
                    sampler=sampler,
                    r1=exo_r1,
                    r2=exo_r2,
                    anchor_diagnostic=exo_anchor_diag)

                # Clear for Next Exoneration
                exoread         = None
                exofreq         = None
                exo_r1          = None
                exo_r2          = None
                exo_anchor_diag = None

            # Time to Show Update?
            if verbiagereach >= verbiagetarget:

                # Show Update
                liner.send(
                    ' Core {:{},d}: Analyzed {:{},d} Reads in {:.2f} sec'.format(
                        coreid,
                        clen,
                        cctrs['analyzedreads'] + cctrs['previousreads'],
                        slen,
                        tt.time()-launchtime))

                # Update Book-keeping
                verbiagereach = 0

            # Continue processing pack?
            if not cpack:
                break

            # Fetch Read Tuple and Frequency
            # Read tuple format: (r1, r2) for concat, (merged, None) for merged
            (read_tuple,
            freq) = cpack.pop()

            # Extract both reads from tuple
            r1 = read_tuple[0]
            r2 = read_tuple[1]  # None for merged reads

            # Update Book-keeping
            cctrs['analyzedreads'] += freq
            verbiagereach += 1
            readcount     += 1

            # Try multi-anchor barcode detection on R1 first
            (index, barcode_status, diag_info) = get_best_barcode_from_candidates(
                read=r1,
                metamap=metamap,
                model=model)

            # If R1 failed and R2 exists, try R2
            if index is None and r2 is not None:
                (index, barcode_status, diag_info) = get_best_barcode_from_candidates(
                    read=r2,
                    metamap=metamap,
                    model=model)

            # Handle failure cases
            if index is None:
                if barcode_status == 'anchor_missing':
                    # Anchor missing - set up exoneration
                    exoread = r1
                    exofreq = freq
                    exo_r1  = r1
                    exo_r2  = r2
                    exo_anchor_diag = {
                        'expected_anchor':           metamap['anchor'],
                        'expected_anchor_tolerance': metamap['anchortval'],
                        'anchor_role':               'prefix' if metamap['trimpolicy'] <= 1.5 else 'suffix'}
                elif barcode_status == 'prefix_missing':
                    cctrs['incalcreads'] += freq
                    if sampler is not None:
                        sampler.add_sample(
                            category='barcode_prefix_missing',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'expected_barcode_prefix':           metamap['barcodeprefix'],
                                'expected_barcode_prefix_tolerance': metamap['bpxtval']})
                elif barcode_status == 'suffix_missing':
                    cctrs['incalcreads'] += freq
                    if sampler is not None:
                        sampler.add_sample(
                            category='barcode_suffix_missing',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'expected_barcode_suffix':           metamap['barcodesuffix'],
                                'expected_barcode_suffix_tolerance': metamap['bsxtval']})
                elif barcode_status == 'barcode_absent':
                    # Barcode not found at any anchor position
                    cctrs['incalcreads'] += freq
                    if sampler is not None:
                        sampler.add_sample(
                            category='barcode_absent',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'anchor_hit_count':            diag_info['anchor_hit_count'] if diag_info else None,
                                'anchor_best_edit_distance':   diag_info['anchor_best_edit_distance'] if diag_info else None,
                                'extracted_barcode_candidate': diag_info['extracted_barcode_candidate'] if diag_info else None})
                elif barcode_status == 'barcode_ambiguous':
                    # Multiple anchors yielded different barcodes with same confidence
                    cctrs['incalcreads'] += freq
                    if sampler is not None:
                        sampler.add_sample(
                            category='barcode_ambiguous',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'anchor_hit_count':            diag_info['anchor_hit_count'] if diag_info else None,
                                'tied_barcode_ids':            [IDdict[i] for i in diag_info['tied_indices']] if diag_info else None,
                                'extracted_barcode_candidate': diag_info['extracted_barcode_candidate'] if diag_info else None})
                continue

            # Compute Associate Match - try both reads
            associatematch, basalmatch, basal_reason = get_associate_match(
                associateread=r1,
                associateerrors=associateerrors,
                associatedict=associatedict,
                index=index,
                metamap=metamap)

            # If R1 failed and R2 exists, try R2
            if not associatematch and r2 is not None:
                associatematch_r2, basalmatch_r2, reason_r2 = get_associate_match(
                    associateread=r2,
                    associateerrors=associateerrors,
                    associatedict=associatedict,
                    index=index,
                    metamap=metamap)
                if associatematch_r2:
                    associatematch = True
                if basalmatch_r2:
                    basalmatch     = True
                    basal_reason   = reason_r2

            # Associate Absent / Incorrect
            if not associatematch:

                # Associate Constants Missing
                if not basalmatch:
                    cctrs['incalcreads'] += freq
                    if sampler is not None:
                        if basal_reason == 'prefix':
                            sampler.add_sample(
                                category='associate_prefix_missing',
                                r1=r1, r2=r2, freq=freq,
                                diagnostic={
                                    'matched_barcode_id':        IDdict[index],
                                    'expected_associate_prefix': metamap['associateprefix']})
                        else:
                            sampler.add_sample(
                                category='associate_suffix_missing',
                                r1=r1, r2=r2, freq=freq,
                                diagnostic={
                                    'matched_barcode_id':        IDdict[index],
                                    'expected_associate_suffix': metamap['associatesuffix']})
                    continue
                # Associate Mismatches with Reference
                else:
                    cctrs['misassocreads'] += freq
                    if sampler is not None:
                        expected_assoc = associatedict[index][0]
                        sampler.add_sample(
                            category='associate_mismatch',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'matched_barcode_id':          IDdict[index],
                                'expected_associate_sequence': expected_assoc[:30] if len(expected_assoc) > 30 else expected_assoc})

            # Compute Callback Evaluation
            if associatematch and (not callback is None):
                try:
                    # Execute Callback with both reads
                    evaluation = callback(
                        r1=r1,
                        r2=r2,
                        ID=(IDdict[index],),
                        count=freq,
                        coreid=coreid)

                    # Uh oh ... Non-boolean Output
                    if not ((evaluation is True) or \
                            (evaluation is False)):
                        raise
                except:
                    # We have a failed evaluation!
                    callbackerror.set()
                    callbackabort = True

                    # Dump input for Stats
                    ut.savedump(dobj={
                            'r1': r1,
                            'r2': r2,
                            'ID': (index,),
                         'count': freq,
                        'coreid': coreid},
                        filepath='{}/{}.callback.dump'.format(
                            countdir,
                            coreid))

                    # Break out!
                    break

                else:
                    # Callback Evaluation is False
                    if not evaluation:
                        cctrs['falsereads'] += freq
                        if sampler is not None:
                            sampler.add_sample(
                                category='callback_false',
                                r1=r1, r2=r2, freq=freq,
                                diagnostic={'matched_barcode_id': IDdict[index]})
                        continue

            # Tally Read Counts
            if associatematch:
                cctrs['experimentreads'] += freq
                countdict[((index),)][1] += freq # Association Count
            countdict[((index),)][0] += freq     # Barcode Count

        # Show Final Updates
        liner.send(
            ' Core {:{},d}: Counted Pack {} w/ {:{},d} Reads in {:05.2f} sec\n'.format(
                coreid,
                clen,
                packname,
                readcount,
                plen,
                tt.time()-t0))

        # Free Memory
        ut.free_mem()

        # Release Control
        tt.sleep(0)

        # Did we Abort due to Callback?
        if callbackabort:
            shutdown.set()
            break

        # Need to Restart?
        if ut.needs_restart(
            memlimit=memlimit):
            restart.set() # Enable Restart
            break # Release, your Memory Real Estate

    # Pack Queue is Empty
    else:
        shutdown.set()


    # Close packfile
    packfile.close()

    # Shutdown!
    if shutdown.is_set():
        # Show Updates
        liner.send(' Core {:{},d}: Shutting Down\n'.format(
            coreid,
            clen))
    # Restart, We Must!
    elif restart.is_set():
        # Show Updates
        liner.send(' Core {:{},d}: Restarting ...\n'.format(
            coreid,
            clen))

    # We didn't Abort right?
    if not callbackabort:

        # Do we have counts?
        if countdict:

            # Save Count Dictionary
            ut.savecount(
                cobj=countdict,
                filepath=countpath)

            # Release Control
            tt.sleep(0)

            # Queue Count Dicionary Path
            countqueue.put(
                countpath.split('/')[-1])

            # Release Control
            tt.sleep(0)

    # Update Read Counting Book-keeping
    previousreads.increment(incr=cctrs['analyzedreads'])
    analyzedreads.increment(incr=cctrs['analyzedreads'])
    phiXreads.increment(incr=cctrs['phix_reads'])
    lowcomplexreads.increment(incr=cctrs['lowcomplexreads'])
    misassocreads.increment(incr=cctrs['misassocreads'])
    falsereads.increment(incr=cctrs['falsereads'])
    incalcreads.increment(incr=cctrs['incalcreads'])
    experimentreads.increment(incr=cctrs['experimentreads'])

    # Save and queue failure samples if enabled
    if sampler is not None and failuresamplequeue is not None:
        samplepath = '{}/{}.{}.failures'.format(countdir, coreid, batchid)
        save_failure_samples(sampler=sampler, filepath=samplepath)
        failuresamplequeue.put(samplepath.split('/')[-1])

    # Counting Completed
    nactive.decrement()
    if shutdown.is_set():
        countqueue.put(None)
        if failuresamplequeue is not None:
            failuresamplequeue.put(None)

    # Release Control
    tt.sleep(0)

def xcount_engine(
    indexfiles,
    packfile,
    packqueue,
    countdir,
    countqueue,
    maptype,
    barcodeerrors,
    previousreads,
    analyzedreads,
    phiXreads,
    lowcomplexreads,
    falsereads,
    incalcreads,
    experimentreads,
    callback,
    callbackerror,
    coreid,
    batchid,
    ncores,
    nactive,
    memlimit,
    restart,
    shutdown,
    launchtime,
    liner,
    failuresamplequeue=None,
    failed_reads_sample_size=1000):
    '''
    Combinatorial count barcodes from multiple
    indexes in packed reads and enque the count
    entries to countqueue. Internal use only.

    :: indexfiles
       type - string / list
       desc - one or more archive of indexed objects
              storing barcode data and models
    :: packfile
       type - string
       desc - path to compressed zipfile
              storing read packs
    :: packqueue
       type - SimpleQueue
       desc - queue storing path to read pack
              files stored in packfile
    :: countdir
       type - string
       desc - filepath to counting workspace
              directory
    :: countqueue
       type - SimpleQueue
       desc - queue storing count matrices
              aggregating one or more read
              packs processed by each core
    :: maptype
       type - integer
       desc - mapping type identifier
              0 = fast mapping
              1 = exact mapping
    :: barcodeerrors
       type - Real / None
       desc - maximum number of mutations
              tolerated in barcodes before
              discarding reads from counting
    :: previousreads
       type - SafeCounter
       desc - total number of reads processed
              previously
    :: analyzedreads
       type - SafeCounter
       desc - total number of reads processed
              so far during counting
    :: phiXreads
       type - SafeCounter
       desc - total number of reads processed
              attributed to PhiX contamination
    :: lowcomplexreads
       type - SafeCounter
       desc - total number of reads processed
              attributed to low complexity products
    :: falsereads
       type - SafeCounter
       desc - total number of reads processed
              that were rejected by callback function
    :: incalcreads
       type - SafeCounter
       desc - total number of invalid reads or ones
              not usable for quantification
    :: experimentreads
       type - SafeCounter
       desc - total number of valid reads used for
              quantification
    :: callback
       type - function
       desc - callback function to invoke during
              counting for concurrent processing
    :: callbackerror
       type - mp.Event
       desc - If True, signals error in callback
              execution
    :: coreid
       type - integer
       desc - current core integer id
    :: batchid
       type - integer
       desc - current batch integer id
    :: ncores
       type - integer
       desc - total number of packers
              concurrently initiated
    :: nactive
       type - SafeCounter
       desc - total number of packers
              concurrently active
    :: memlimit
       type - nu.Real
       desc - total amount of memory
              allowed per core
    :: restart
       type - mp.Event
       desc - multiprocessing event when
              process needs to restart
    :: shutdown
       type - mp.Event
       desc - multiprocessing event when
              process needs to shutdown
    :: launchtime
       type - time
       desc - initial launch timestamp
    :: liner
       type - coroutine
       desc - dynamic printing
    :: failuresamplequeue
       type - SafeQueue / None
       desc - queue for storing failure sample
              file paths (None = disabled)
    :: failed_reads_sample_size
       type - integer
       desc - maximum samples per failure category
    '''

    # Aggregate Index Files
    models   = []
    metamaps = []
    IDdicts  = []
    numindex = len(indexfiles)
    for indexfile in indexfiles:

        # Open indexfile
        indexfile = ut.get_archive(
            arcfile=indexfile)

        # Load Barcode Model
        model = ut.loadmodel(
            archive=indexfile,
            mfile='barcode.model')

        # Prime Barcode Model
        model.prime(
            t=barcodeerrors,
            mode=maptype)

        # Load Maps
        IDdict = ut.loaddict(
            archive=indexfile,
            dfile='ID.map')
        metamap = ut.loaddict(
            archive=indexfile,
            dfile='meta.map')

        # Close indexfile
        indexfile.close()

        # Append Index Objects
        models.append(model)
        metamaps.append(metamap)
        IDdicts.append(IDdict)

    # Define PhiX Spectrum
    phiXkval = 30
    phiXspec = px.get_phiX_spectrum(
        k=phiXkval)

    # Open packfile
    packfile = ut.get_archive(
        arcfile=packfile)

    # Load packing.stat
    packstat = ut.loaddict(
        archive=packfile,
        dfile='packing.stat')

    # Book-keeping Variables
    cctrs = {
        'previousreads'  : previousreads.value(),
        'analyzedreads'  : 0,
        'phix_reads'     : 0,
        'lowcomplexreads': 0,
        'falsereads'     : 0,
        'incalcreads'    : 0,
        'experimentreads': 0}
    callbackabort = False

    # Initialize failure sampler if enabled
    sampler = None
    if failuresamplequeue is not None:
        sampler = ut.FailureSampler(
            sample_size=failed_reads_sample_size,
            categories=XCOUNT_FAILURE_CATEGORIES)

    # Compute Printing Lengths
    clen = ut.get_printlen(value=ncores)
    slen = plen = ut.get_printlen(
        value=packstat['survived_reads'])

    # Read Count Storage
    countdict = cx.Counter()
    countpath = '{}/{}.{}.count'.format(
        countdir, coreid, batchid)

    # Pack Counting Loop
    while not packqueue.empty():

        # Callback Error Somewhere?
        if not callback is None:
            if callbackerror.is_set():
                callbackabort = True
                shutdown.set()
                break

        # Fetch Pack Name / Token
        packname = packqueue.get()

        # Exhaustion Token
        if packname is None:
            shutdown.set()
            break

        # Fetch Read Pack
        cpack = ut.loadpack(
            archive=packfile,
            pfile='{}.pack'.format(
                packname))

        # Book-keeping Variables
        exoread         = None
        exofreq         = None
        exo_r1          = None  # Track r1 for failure sampling
        exo_r2          = None  # Track r2 for failure sampling
        exo_anchor_diag = None   # Anchor diagnostic for exoneration
        readcount = 0

        verbiagereach  = 0
        verbiagetarget = rn.randint(
            *map(round, (len(cpack) * 0.080,
                         len(cpack) * 0.120)))

        # Start Timer
        t0 = tt.time()

        # Read Counting Loop
        while True:

            # Callback Error Somewhere?
            if not callback is None:
                if callbackerror.is_set():
                    callbackabort = True
                    break

            # Exoneration Block
            if not exoread is None:

                # Run Exoneration Procedure
                exoneration_procedure(
                    exoread=exoread,
                    exofreq=exofreq,
                    phiXkval=phiXkval,
                    phiXspec=phiXspec,
                    cctrs=cctrs,
                    sampler=sampler,
                    r1=exo_r1,
                    r2=exo_r2,
                    anchor_diagnostic=exo_anchor_diag)

                # Clear for Next Exoneration
                exoread         = None
                exofreq         = None
                exo_r1          = None
                exo_r2          = None
                exo_anchor_diag = None

            # Time to Show Update?
            if verbiagereach >= verbiagetarget:

                # Show Update
                liner.send(
                    ' Core {:{},d}: Analyzed {:{},d} Reads in {:.2f} sec'.format(
                        coreid,
                        clen,
                        cctrs['analyzedreads'] + cctrs['previousreads'],
                        slen,
                        tt.time()-launchtime))

                # Update Book-keeping
                verbiagereach = 0

            # Continue processing pack?
            if not cpack:
                break

            # Fetch Read Tuple and Frequency
            # Read tuple format: (r1, r2) for concat, (merged, None) for merged
            (read_tuple,
            freq) = cpack.pop()

            # Extract both reads from tuple
            r1 = read_tuple[0]
            r2 = read_tuple[1]  # None for merged reads

            # Update Book-keeping
            cctrs['analyzedreads'] += freq
            verbiagereach += 1
            readcount    += 1

            # Barcode Mapping Loop
            indextuple = []
            partialanc = False
            partialmap = False
            has_ambiguous = False
            per_index_status                         = []
            per_index_anchor_hit_count               = []
            per_index_anchor_best_edit_distance      = []
            per_index_extracted_barcode_candidate    = []
            per_index_tied_barcode_ids               = []
            first_trim_failure_status  = None
            first_trim_failure_idx     = None
            first_trim_failure_diag    = None
            for idx in range(numindex):

                # Try multi-anchor barcode detection on R1 first
                (index, barcode_status, diag_info) = get_best_barcode_from_candidates(
                    read=r1,
                    metamap=metamaps[idx],
                    model=models[idx])

                # If R1 failed and R2 exists, try R2
                if index is None and r2 is not None:
                    (index, barcode_status, diag_info) = get_best_barcode_from_candidates(
                        read=r2,
                        metamap=metamaps[idx],
                        model=models[idx])

                # Collect per-index barcode candidate and anchor quality
                bc_candidate = diag_info.get('extracted_barcode_candidate') if diag_info else None
                per_index_extracted_barcode_candidate.append(bc_candidate)
                per_index_anchor_hit_count.append(
                    diag_info.get('anchor_hit_count') if diag_info else None)
                per_index_anchor_best_edit_distance.append(
                    diag_info.get('anchor_best_edit_distance') if diag_info else None)

                # Collect per-index tied barcode IDs (only for barcode_ambiguous)
                if barcode_status == 'barcode_ambiguous' and diag_info and diag_info.get('tied_indices'):
                    tied_ids = [IDdicts[idx][i] for i in diag_info['tied_indices']]
                else:
                    tied_ids = None
                per_index_tied_barcode_ids.append(tied_ids)

                # Handle result
                if index is None:
                    if barcode_status == 'anchor_missing':
                        indextuple.append('-')
                        per_index_status.append('anchor_missing')
                    elif barcode_status in ('prefix_missing', 'suffix_missing'):
                        partialanc = True
                        if first_trim_failure_status is None:
                            first_trim_failure_status = barcode_status
                            first_trim_failure_idx    = idx
                            first_trim_failure_diag   = diag_info
                        indextuple.append('-')
                        per_index_status.append(barcode_status)
                    elif barcode_status == 'barcode_absent':
                        partialanc = True  # Anchor was found, just no valid barcode
                        indextuple.append('-')
                        per_index_status.append('barcode_absent')
                    elif barcode_status == 'barcode_ambiguous':
                        partialanc = True  # Anchor was found
                        has_ambiguous = True
                        indextuple.append('-')
                        per_index_status.append('barcode_ambiguous')
                    continue

                # Found a Barcode!
                partialanc = True
                partialmap = True
                indextuple.append(index)
                per_index_status.append('ok')

            # All Anchors Absent
            if not partialanc:
                exoread = r1  # Use r1 for exoneration
                exofreq = freq
                exo_r1  = r1  # Track for failure sampling
                exo_r2  = r2
                exo_anchor_diag = {
                    'per_index_anchor':           [m['anchor'] for m in metamaps],
                    'per_index_anchor_tolerance': [m['anchortval'] for m in metamaps],
                    'per_index_anchor_role':      ['prefix' if m['trimpolicy'] <= 1.5 else 'suffix' for m in metamaps]}
                continue

            # All Barcodes Absent (but anchors were found)
            if not partialmap:
                cctrs['incalcreads'] += freq
                if sampler is not None:
                    if first_trim_failure_status == 'prefix_missing':
                        fail_mm = metamaps[first_trim_failure_idx]
                        sampler.add_sample(
                            category='barcode_prefix_missing',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'expected_barcode_prefix':           fail_mm['barcodeprefix'],
                                'expected_barcode_prefix_tolerance': fail_mm['bpxtval']})
                    elif first_trim_failure_status == 'suffix_missing':
                        fail_mm = metamaps[first_trim_failure_idx]
                        sampler.add_sample(
                            category='barcode_suffix_missing',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'expected_barcode_suffix':           fail_mm['barcodesuffix'],
                                'expected_barcode_suffix_tolerance': fail_mm['bsxtval']})
                    elif has_ambiguous:
                        sampler.add_sample(
                            category='barcode_ambiguous',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'per_index_status':                      per_index_status,
                                'per_index_anchor_hit_count':            per_index_anchor_hit_count,
                                'per_index_anchor_best_edit_distance':   per_index_anchor_best_edit_distance,
                                'per_index_extracted_barcode_candidate': per_index_extracted_barcode_candidate,
                                'per_index_tied_barcode_ids':            per_index_tied_barcode_ids})
                    else:
                        sampler.add_sample(
                            category='barcode_absent',
                            r1=r1, r2=r2, freq=freq,
                            diagnostic={
                                'per_index_status':                      per_index_status,
                                'per_index_anchor_hit_count':            per_index_anchor_hit_count,
                                'per_index_anchor_best_edit_distance':   per_index_anchor_best_edit_distance,
                                'per_index_extracted_barcode_candidate': per_index_extracted_barcode_candidate})
                continue

            # Convert Tuples to Indexes
            indextuple = tuple(indextuple)

            # Compute Callback Evaluation
            if not callback is None:
                try:
                    # Execute Callback with both reads
                    evaluation = callback(
                        r1=r1,
                        r2=r2,
                        ID=tuple(IDdicts[k][it] if not it == '-' else None
                                 for k, it in enumerate(indextuple)),
                        count=freq,
                        coreid=coreid)

                    # Uh oh ... Non-boolean Output
                    if not ((evaluation is True) or \
                            (evaluation is False)):
                        raise
                except:
                    # We have a failed evaluation!
                    callbackerror.set()
                    callbackabort = True

                    # Dump input for Stats
                    ut.savedump(dobj={
                            'r1': r1,
                            'r2': r2,
                            'ID': indextuple,
                         'count': freq,
                        'coreid': coreid},
                        filepath='{}/{}.callback.dump'.format(
                            countdir,
                            coreid))

                    # Break out!
                    break

                else:
                    # Callback Evaluation is False
                    if not evaluation:
                        cctrs['falsereads'] += freq
                        if sampler is not None:
                            sampler.add_sample(
                                category='callback_false',
                                r1=r1, r2=r2, freq=freq,
                                diagnostic={'per_index_barcode_id': [IDdicts[k][it] if it != '-' else None
                                    for k, it in enumerate(indextuple)]})
                        continue

            # All Components Valid
            countdict[indextuple]    += freq
            cctrs['experimentreads'] += freq

        # Show Final Updates
        liner.send(
            ' Core {:{},d}: Counted Pack {} w/ {:{},d} Reads in {:05.2f} sec\n'.format(
                coreid,
                clen,
                packname,
                readcount,
                plen,
                tt.time()-t0))

        # Free Memory
        ut.free_mem()

        # Release Control
        tt.sleep(0)

        # Did we Abort due to Callback?
        if callbackabort:
            shutdown.set()
            break

        # Need to Restart?
        if ut.needs_restart(
            memlimit=memlimit):
            restart.set() # Enable Restart
            break # Release, your Memory Real Estate

    # Pack Queue is Empty
    else:
        shutdown.set()


    # Close packfile
    packfile.close()

    # Shutdown!
    if shutdown.is_set():
        # Show Updates
        liner.send(' Core {:{},d}: Shutting Down\n'.format(
            coreid,
            clen))
    # Restart, We Must!
    elif restart.is_set():
        # Show Updates
        liner.send(' Core {:{},d}: Restarting ...\n'.format(
            coreid,
            clen))

    # We didn't Abort right?
    if not callbackabort:

        # Do we have counts?
        if countdict:

            # Save Count Dictionary
            ut.savecount(
                cobj=countdict,
                filepath=countpath)

            # Release Control
            tt.sleep(0)

            # Queue Count Dicionary Path
            countqueue.put(
                countpath.split('/')[-1])

            # Release Control
            tt.sleep(0)

    # Update Read Counting Book-keeping
    previousreads.increment(incr=cctrs['analyzedreads'])
    analyzedreads.increment(incr=cctrs['analyzedreads'])
    phiXreads.increment(incr=cctrs['phix_reads'])
    lowcomplexreads.increment(incr=cctrs['lowcomplexreads'])
    falsereads.increment(incr=cctrs['falsereads'])
    incalcreads.increment(incr=cctrs['incalcreads'])
    experimentreads.increment(incr=cctrs['experimentreads'])

    # Save and queue failure samples if enabled
    if sampler is not None and failuresamplequeue is not None:
        samplepath = '{}/{}.{}.failures'.format(countdir, coreid, batchid)
        save_failure_samples(sampler=sampler, filepath=samplepath)
        failuresamplequeue.put(samplepath.split('/')[-1])

    # Counting Completed
    nactive.decrement()
    if shutdown.is_set():
        countqueue.put(None)
        if failuresamplequeue is not None:
            failuresamplequeue.put(None)

    # Release Control
    tt.sleep(0)

# Failure Sampling Helper Functions

def save_failure_samples(sampler, filepath):
    '''
    Save failure samples to a file.
    Internal use only.

    :: sampler
       type - ut.FailureSampler
       desc - sampler containing failure samples
    :: filepath
       type - string
       desc - path to save samples
    '''
    data = {
        'samples': sampler.get_samples(),
        'seen_counts': sampler.get_seen_counts(),
        'sample_size': sampler.sample_size
    }
    ut.savedump(dobj=data, filepath=filepath)

def load_failure_samples(filepath):
    '''
    Load failure samples from a file.
    Internal use only.

    :: filepath
       type - string
       desc - path to load samples from
    '''
    return ut.loaddump(dfile=filepath)

def merge_failure_samples(sample_files, sample_size):
    '''
    Merge failure samples from multiple workers.
    Produces a uniform sample (up to sample_size per category) from the
    union of all worker failure streams, using each worker's per-category
    reservoir sample and seen_counts.
    Internal use only.

    :: sample_files
       type - list
       desc - list of paths to sample files
    :: sample_size
       type - integer
       desc - maximum samples to keep per category
    '''
    # Load all sample files
    worker_data = [load_failure_samples(fp) for fp in sample_files]
    if not worker_data:
        return {}, {}

    # If sample_size differs across worker files (shouldn't happen within a
    # single run), merge conservatively to avoid over-drawing from any worker
    # reservoir.
    sample_size = min(
        int(sample_size),
        *[int(d.get('sample_size', sample_size)) for d in worker_data])

    # Preserve category ordering from the first worker file, then append any
    # additional categories seen in other files (should be identical).
    category_order = list(worker_data[0].get('samples', {}).keys())
    for data in worker_data[1:]:
        for cat in data.get('samples', {}).keys():
            if cat not in category_order:
                category_order.append(cat)

    merged_samples = {}
    merged_seen_counts = {}

    for category in category_order:
        # Total stream length for this failure category
        worker_seen = [
            int(data.get('seen_counts', {}).get(category, 0))
            for data in worker_data
        ]
        total_seen = sum(worker_seen)
        merged_seen_counts[category] = total_seen

        # Nothing to merge
        if total_seen <= 0:
            merged_samples[category] = []
            continue

        # Number of samples to keep overall for this category
        target = min(sample_size, total_seen)

        # Draw how many samples to take from each worker stream using a
        # multivariate hypergeometric decomposition.
        remaining_total = total_seen
        remaining_draws = target
        take_counts = []
        for idx, seen_i in enumerate(worker_seen):
            if remaining_draws <= 0 or seen_i <= 0:
                take_counts.append(0)
                remaining_total -= seen_i
                continue

            # Last worker gets whatever is left (guaranteed <= seen_i)
            if idx == (len(worker_seen) - 1):
                take = remaining_draws
            else:
                take = np.random.hypergeometric(
                    ngood=seen_i,
                    nbad=remaining_total - seen_i,
                    nsample=remaining_draws)
                take = int(take)

            take_counts.append(take)
            remaining_total -= seen_i
            remaining_draws -= take

        # Select the requested number of samples from each worker's reservoir.
        merged = []
        for data, take in zip(worker_data, take_counts):
            if take <= 0:
                continue
            reservoir = data.get('samples', {}).get(category, [])
            if not reservoir:
                continue
            # Note: reservoir size is min(sample_size, seen_i) so take <= len(reservoir)
            merged.extend(rn.sample(reservoir, take))

        merged_samples[category] = merged

    return merged_samples, merged_seen_counts

def write_failed_reads(samples, seen_counts, filepath, liner):
    '''
    Write failed reads samples to CSV file.
    Internal use only.

    :: samples
       type - dict
       desc - dictionary of category -> list of (r1, r2, freq, diagnostic) tuples;
              diagnostic is dict / list / None, JSON-serialized at write time
    :: seen_counts
       type - dict
       desc - dictionary of category -> total samples seen
    :: filepath
       type - string
       desc - output CSV file path
    :: liner
       type - coroutine
       desc - dynamic printing
    '''
    # Start Timer
    t0 = tt.time()

    # Show Update
    liner.send(' Writing Failed Reads ...')

    # Count total rows
    total_rows = sum(len(cat_samples) for cat_samples in samples.values())

    # Write CSV
    # Note: use csv.writer to correctly quote fields that may contain delimiter
    # characters/newlines/quotes; diagnostics are JSON-serialized dicts.
    import csv
    import json
    with open(filepath, 'w', newline='') as outfile:
        writer = csv.writer(outfile)
        writer.writerow(
            ('FailureReason', 'R1', 'R2', 'ReadCount', 'DiagnosticDetails'))

        # Write samples grouped by failure category
        for category, cat_samples in samples.items():
            for (r1, r2, freq, diagnostic) in cat_samples:
                writer.writerow((
                    category,
                    r1 if r1 else '',
                    r2 if r2 else '',
                    freq,
                    json.dumps(diagnostic) if diagnostic else ''))

    # Final Updates
    liner.send(
        ' Rows Written: {:,} Failed Read Sample(s)\n'.format(total_rows))
    liner.send(
        ' Time Elapsed: {:.2f} sec\n'.format(tt.time() - t0))

def aggregate_failure_samples(
    failuresamplequeue,
    countdir,
    failed_reads_file,
    failed_reads_sample_size,
    prodcount,
    liner):
    '''
    Aggregate failure samples from worker processes and write to CSV.
    Internal use only.

    :: failuresamplequeue
       type - SafeQueue
       desc - queue storing paths to failure sample files
    :: countdir
       type - string
       desc - path to workspace count directory
    :: failed_reads_file
       type - string
       desc - output CSV file path
    :: failed_reads_sample_size
       type - integer
       desc - maximum samples per category
    :: prodcount
       type - integer
       desc - total number of producers
    :: liner
       type - coroutine
       desc - dynamic printing
    '''
    # Collect sample file paths
    sample_files = []

    # Wait for all producers to finish
    finished_count = 0
    while finished_count < prodcount:
        token = failuresamplequeue.get()
        if token is None:
            finished_count += 1
        else:
            sample_path = '{}/{}'.format(countdir, token)
            if os.path.exists(sample_path):
                sample_files.append(sample_path)
        tt.sleep(0)

    # Merge samples if any exist
    if sample_files:
        liner.send('\n[Step 5: Writing Failed Read Samples]\n')

        samples, seen_counts = merge_failure_samples(
            sample_files=sample_files,
            sample_size=failed_reads_sample_size)

        # Write to CSV
        write_failed_reads(
            samples=samples,
            seen_counts=seen_counts,
            filepath=failed_reads_file,
            liner=liner)

        # Cleanup temp files
        for sample_path in sample_files:
            ut.remove_file(filepath=sample_path)
