# core Python imports
import datetime
import logging
import operator


# django imports
from django.core.exceptions import ObjectDoesNotExist
from django.utils.timezone import utc
from django.db.models import Q


# local app imports
from djhcup_staging.models import StagingBatch, StagingQueue, StagingTable, ImportBatch, File
from djhcup_staging.utils import misc


# start a logger
default_logger = logging.getLogger('default')


# constants
STAGING_CONVERSIONS = [
    #dict(cat_out='CHARGES', req_in=['CHARGES_SOURCE']),
    #dict(cat_out='PR', req_in=['CORE_SOURCE']),
    #dict(cat_out='DX', req_in=['CORE_SOURCE']),
    #dict(cat_out='UFLAGS',
    #     req_in=[
    #        'CORE_SOURCE',
    #        'CHARGES',
    #        'PR',
    #    ]),
    dict(cat_out='CORE_PROCESSED',
         req_in=[
            'CORE_SOURCE',
        ],
         opt_in=[
            'AHALINK_SOURCE',
            'UFLAGS',
            'DX_PR_GRPS_SOURCE',
            'SEVERITY_SOURCE',
            'CORE_SOURCE',
            'DAYSTOEVENT_SOURCE',
        ]),
]

CAT_IN_TO_ATTR = {
    'AHALINK_SOURCE': 'aha_input',
    'CHARGES_SOURCE': 'stg_input',
    'DX_PR_GRPS_SOURCE': 'dx_pr_grps_input',
    'SEVERITY_SOURCE': 'severity_input',
    'CORE_SOURCE': 'stg_input',
    'DAYSTOEVENT_SOURCE': 'dte_input',
    'CHARGES': 'charges_input',
    'PR': 'pr_input',
    'UFLAGS': 'uflag_input',
}



# TODO: Modify this so that all the related DataSource objects have the same parent
# (prevents SID CORE from being used with SASD CHGS, etc.)
def unprocessed_batches(conversions=STAGING_CONVERSIONS, cat_to_attr=CAT_IN_TO_ATTR):
    """
    Creates batches for unprocessed files, including batches with
    changed input availability.
    """
    batches = []
    now = datetime.datetime.utcnow().replace(tzinfo=utc).strftime("%Y%m%d%H%M%S")

    states = misc.states()
    years = misc.years()

    for con in conversions:
        req_in = con['req_in']
        cat_out = con['cat_out']
        name = '{inputs} to {cat_out} {now}' \
            .format(inputs=req_in, cat_out=cat_out, now=now)
        description = 'Batch of all incomplete {inputs} to {cat_out} StagingTables as of {now}.' \
            .format(inputs=req_in, cat_out=cat_out, now=now)
        b = StagingBatch(name=name, description=description)
        b.save()
        for st in ['NE']:#states:
            for yr in [2007]:# years:
                # instantiate, but do not save/add to batch til end
                stg_q = StagingQueue(category=cat_out)
                
                reqs = []
                for reqd in req_in:
                    attr = cat_to_attr[reqd]
                    
                    print "reqd looking for {st} {yr} {cat}".format(st=st, yr=yr, cat=reqd)
                        
                    reqd_qs = StagingTable.objects \
                        .filter(category=reqd, wip=False) \
                        .order_by('-created_at')
                    #reqd_qs = reqd_qs \
                    #    .extra(select={'complete_null': 'complete is null'}) \
                    #    .extra(order_by=['complete_null'])
                
                    print "reqd_qs is {qs}".format(qs=reqd_qs)
                    
                    reqd_lst = filter(lambda x: x.state==st and x.year==yr, reqd_qs)
                    
                    if len(reqd_lst) > 0:
                        print "reqd found some matching {attr}".format(attr=attr)
                        i = reqd_lst[0]
                        try:
                            if getattr(stg_q, attr) != None:
                                # unable to add a second time if already not None
                                # should emit a logger error about this
                                print "{stg_q}.{attr} value is not None ({val})" \
                                    .format(stg_q=stg_q, attr=attr, val=getattr(stg_q, attr))
                                reqs.append(False)
                            else:
                                print "{stg_q}.{attr} value is None and didn't freak out" \
                                    .format(stg_q=stg_q, attr=attr)
                                setattr(stg_q, attr, i)
                                print "{stg_q}.{attr} set to {i}" \
                                    .format(stg_q=stg_q, attr=attr, i=i)
                                reqs.append(reqd)
                        except ObjectDoesNotExist:
                            if reqd == req_in[0]:
                                print "reqd pole position always assigned to stg_input"
                                stg_q.stg_input = i
                                
                            else:
                                print "Setting {stg}.{attr} to {i}".format(stg=stg_q, attr=attr, i=i)
                                setattr(stg_q, attr, i)
                            reqs.append(reqd)
                    else:
                        reqs.append(False)
                
                opts = []
                if 'opt_in' in con:
                    print "there's an opt_in key in con"
                    opt_in = con['opt_in']
                    for opt in opt_in:
                        attr = cat_to_attr[opt]
                        
                        print "opt looking for {st} {yr} {cat}".format(st=st, yr=yr, cat=opt)
                        
                        opt_qs = StagingTable.objects \
                            .filter(category=opt, wip=False) \
                            .order_by('-created_at')
                        
                        opt_lst = filter(lambda x: x.state==st and x.year==yr, opt_qs)
                        
                        if len(opt_lst) > 0:
                            print "opt found some matching {attr}".format(attr=attr)
                            i = opt_lst[0]
                            if getattr(stg_q, attr) == None:
                                setattr(stg_q, attr, i)
                                opts.append(opt)
                            else:
                                # unable to add a second time if already not None
                                # should emit a logger error about this
                                opts.append(False)
                        
                        else:
                            opts.append(False)
                
                print "reqs is {reqs}".format(reqs=reqs)
                if any(x==False for x in reqs):
                    b.log("One or more required components of this conversion ({state} {year} {cat}) are unavailable." \
                        .format(state=st, year=yr, cat=cat_out))
                
                else:
                    # check for completed StagingQueue built matching cat_out and state-year already
                    st_tbl_qs = StagingQueue.objects \
                        .filter(category=cat_out, complete__isnull=False)
                    
                    reqd_predicates = [('%s__isnull' % CAT_IN_TO_ATTR[x], False) for x in reqs if x is not False]
                    opt_predicates = [('%s__isnull' % CAT_IN_TO_ATTR[x], False) for x in opts if x is not False]
                    equiv_q_lst = [Q(x) for x in reqd_predicates + opt_predicates]
                    equivalent_qs = st_tbl_qs.filter(reduce(operator.or_, equiv_q_lst))
                    
                    equivalent_lst = filter(lambda x: x.stg_input.state==st and x.stg_input.year==yr, equivalent_qs)
                    equivalent_ct = len(equivalent_lst)
                    print "equivalent_ct is {ct}".format(ct=equivalent_ct)
                    
                    # while we're here, make sure it isn't in the queryset for this batch as well
                    inb_st_tbl_qs = StagingQueue.objects \
                        .filter(category=cat_out, batch=b)
                    
                    reqd_predicates = [('%s__isnull' % CAT_IN_TO_ATTR[x], False) for x in reqs if x is not False]
                    opt_predicates = [('%s__isnull' % CAT_IN_TO_ATTR[x], False) for x in opts if x is not False]
                    inb_q_lst = [Q(x) for x in reqd_predicates + opt_predicates]
                    inb_qs = inb_st_tbl_qs.filter(reduce(operator.or_, inb_q_lst))
                    
                    print inb_qs
                    inb_lst = filter(lambda x: x.stg_input.state==st and x.stg_input.year==yr, inb_qs)
                    inb_ct = len(inb_lst)
                    print "inb_ct is {ct}".format(ct=inb_ct)
                    print inb_lst
                    
                    if equivalent_ct == 0 and inb_ct == 0:
                        # TODO: save the thing
                        msg = "There are no StagingTable or StagingQueue objects for {state} {year} {category} with an equivalent constellation of inputs compared to those now available."
                        b.log(msg.format(state=st, year=yr, category=cat_out))
                        #print "%s should be saved now" % (stg_q)
                        stg_q.batch = b
                        stg_q.save()
                        b.log("Saved StagingQueue {pk} for {state} {year} {category}."\
                            .format(pk=stg_q.pk, state=st, year=yr, category=cat_out))
                    
                    elif inb_ct > 0:
                        msg = "There are no StagingTable objects for {state} {year} {category} with the constellation of inputs now available, but one is in the batch already."
                        b.log(msg.format(state=st, year=yr, category=cat_out))
                    
                    elif equivalent_ct > 0:
                        msg = "There are already one or more StagingTable objects for {state} {year} {category} with the constellation of inputs now available."
                        b.log(msg.format(state=st, year=yr, category=cat_out))
                        # if replacing, delete old ones?
                        # OR!
                        # Set up a model to hold deprecation queues. Would have attributes like
                        # st_q, superceded_by, a timestamp of when that happened/was noticed,
                        # and a boolean for whether the old one has been dropped.
                        # that could be pruned manually on a periodic basis, or whatevs.
                        # would probably need to review dependencies/parent-child tbl relations
                    
        set_length = len(b.stagingqueue_set.all())
        if set_length == 0:
            b.log('Found no out-of-date conversions for StagingBatch {pk}. The batch will be marked as void.')
            b.void = True
            b.save()
        else:
            b.log('Created StagingBatch {pk} with {length} files'.format(pk=b.pk, length=set_length))
        
        batches.append(b)
    
    return batches


def unimported_batch():
    """
    Creates an ImportBatch object for discovered files that have not
    yet been imported to StagingTable objects.
    """
    now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")

    name = 'Non-imported {now}' \
        .format(now=now)

    description = 'Non-imported files as of {now}' \
        .format(now=now)

    b = ImportBatch(name=name, description=description)
    b.save()

    fqs = File.objects.filter(
        Q(file_type__content_type='CONTENTWH')|Q(file_type__content_type='CONTENTNH',loadfile__isnull=False)
        ).exclude(state__abbreviation='TX') # these are special

    fqs = fqs.order_by('state__abbreviation', 'year', '-file_type__compression')

    # add each if one matching state-year is not already in there
    for f in fqs:
        # look for other matches that cover this already
        st = f.state
        yr = f.year
        ds = f.file_type.source
        
        # either in the database
        st_tbl_qs = StagingTable.objects.filter(
            imported_via__file__state=st,
            imported_via__file__year=yr,
            imported_via__file__file_type__source=ds
        )
        
        
        # or in the queue set itself
        q_qs = File.objects.filter(
            state=st,
            year=yr,
            file_type__source=ds,
            importqueue__batch=b
        )
        
        if len(st_tbl_qs) == 0 and len(q_qs) == 0:
            b.importqueue_set.create(file=f)
    
    return b

    