#!/usr/bin/env python
"""
_WMBSHelper_

Use WMSpecParser to extract information for creating workflow, fileset, and subscription
"""

from builtins import next, str as newstr, bytes, zip, range
from future.utils import viewitems

import logging
import threading
from collections import defaultdict

from Utils.PythonVersion import PY3
from Utils.Utilities import encodeUnicodeToBytesConditional
from WMComponent.DBS3Buffer.DBSBufferDataset import DBSBufferDataset
from WMComponent.DBS3Buffer.DBSBufferFile import DBSBufferFile
from WMCore.BossAir.BossAirAPI import BossAirAPI, BossAirException
from WMCore.DAOFactory import DAOFactory
from WMCore.DataStructs.File import File as DatastructFile
from WMCore.DataStructs.LumiList import LumiList
from WMCore.DataStructs.Run import Run
from WMCore.JobStateMachine.ChangeState import ChangeState
from WMCore.ResourceControl.ResourceControl import ResourceControl
from WMCore.WMBS.File import File
from WMCore.WMBS.Fileset import Fileset
from WMCore.WMBS.Job import Job
from WMCore.WMBS.Subscription import Subscription
from WMCore.WMBS.Workflow import Workflow
from WMCore.WMConnectionBase import WMConnectionBase
from WMCore.WMException import WMException
from WMCore.WMRuntime.SandboxCreator import SandboxCreator


def wmbsSubscriptionStatus(logger, dbi, conn, transaction):
    """
    Function to return status of wmbs subscriptions
    """
    daoFactory = DAOFactory(package='WMCore.WMBS', logger=logger, dbinterface=dbi)
    action = daoFactory(classname='Monitoring.SubscriptionStatus')
    return action.execute(conn=conn, transaction=transaction)


class WorkQueueWMBSException(WMException):
    """
    Dummy exception class for exceptions raised
    in WMBS Helper.

    TODO: Do something useful

    """

    pass


def killWorkflow(workflowName, jobCouchConfig, bossAirConfig=None):
    """
    _killWorkflow_

    Kill a workflow that is already executing inside the agent.  This will
    mark all incomplete jobs as failed and files that belong to all
    non-cleanup and non-logcollect subscriptions as failed.  The name of the
    JSM couch database and the URL to the database must be passed in as well
    so the state transitions are logged.
    """
    myThread = threading.currentThread()
    daoFactory = DAOFactory(package="WMCore.WMBS",
                            logger=myThread.logger,
                            dbinterface=myThread.dbi)
    killFilesAction = daoFactory(classname="Subscriptions.KillWorkflow")
    killJobsAction = daoFactory(classname="Jobs.KillWorkflow")

    killFilesAction.execute(workflowName=workflowName,
                            conn=myThread.transaction.conn)

    liveJobs = killJobsAction.execute(workflowName=workflowName,
                                      conn=myThread.transaction.conn)

    changeState = ChangeState(jobCouchConfig)

    # Deal with any jobs that are running in the batch system
    # only works if we can start the API
    if bossAirConfig:
        bossAir = BossAirAPI(config=bossAirConfig)
        killableJobs = []
        for liveJob in liveJobs:
            if liveJob["state"].lower() == 'executing':
                # Then we need to kill this on the batch system
                liveWMBSJob = Job(id=liveJob["id"])
                liveWMBSJob.update(liveJob)
                killableJobs.append(liveJob)
        # Now kill them
        try:
            logging.info("Killing %d jobs for workflow: %s", len(killableJobs), workflowName)
            bossAir.kill(jobs=killableJobs, workflowName=workflowName)
        except BossAirException as ex:
            errMsg = "Error while trying to kill running jobs in workflow. Error: %s" % str(ex)
            logging.exception(errMsg)
            # But continue; we need to kill the jobs in the master
            # the batch system will have to take care of itself.

    liveWMBSJobs = defaultdict(list)
    for liveJob in liveJobs:
        if liveJob["state"] == "killed":
            # Then we've killed it already
            continue
        liveWMBSJob = Job(id=liveJob["id"])
        liveWMBSJob.update(liveJob)
        liveWMBSJobs[liveJob["state"]].append(liveWMBSJob)

    for state, jobsByState in viewitems(liveWMBSJobs):
        if len(jobsByState) > 100 and state != "executing":
            # if there are to many jobs skip the couch and dashboard update
            # TODO: couch and dashboard need to be updated or parallel.
            changeState.check("killed", state)
            changeState.persist(jobsByState, "killed", state)
        else:
            changeState.propagate(jobsByState, "killed", state)
    return


def freeSlots(multiplier=1.0, minusRunning=False, allowedStates=None, knownCmsSites=None):
    """
    Get free resources from wmbs.

    Specify multiplier to apply a ratio to the actual numbers.
    minusRunning control if running jobs should be counted
    """
    allowedStates = allowedStates or ['Normal']
    rc_sites = ResourceControl().listThresholdsForCreate()
    thresholds = defaultdict(lambda: 0)
    jobCounts = defaultdict(dict)
    for name, site in viewitems(rc_sites):
        if not site.get('cms_name'):
            logging.warning("Not fetching work for %s, cms_name not defined", name)
            continue
        if knownCmsSites and site['cms_name'] not in knownCmsSites:
            logging.warning(
                "%s doesn't appear to be a known cms site, work may fail to be acquired for it", site['cms_name'])
        if site['state'] not in allowedStates:
            continue
        slots = site['total_slots']
        thresholds[site['cms_name']] += (slots * multiplier)
        if minusRunning:
            jobCounts[site['cms_name']] = dict(
                (k, jobCounts[site['cms_name']].get(k, 0) + site['pending_jobs'].get(k, 0))
                for k in site['pending_jobs'])

    return dict(thresholds), dict(jobCounts)


class WMBSHelper(WMConnectionBase):
    """
    _WMBSHelper_

    Interface between the WorkQueue and WMBS.
    """

    def __init__(self, wmSpec, taskName, blockName=None, mask=None,
                 cachepath='.', commonLocation=None):
        """
        _init_

        Initialize DAOs and other things needed.
        """
        self.block = blockName
        self.mask = mask
        self.wmSpec = wmSpec
        self.topLevelTask = wmSpec.getTask(taskName)
        self.cachepath = cachepath
        self.isDBS = True

        self.topLevelFileset = None
        self.topLevelSubscription = None
        self.topLevelTaskDBSBufferId = None
        self.commonLocation = commonLocation or []

        self.mergeOutputMapping = {}

        # Initiate the pieces you need to run your own DAOs
        WMConnectionBase.__init__(self, "WMCore.WMBS")
        myThread = threading.currentThread()
        self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
                                        logger=myThread.logger,
                                        dbinterface=myThread.dbi)

        # DAOs from WMBS for file commit
        self.setParentage = self.daofactory(classname="Files.SetParentage")
        self.setFileLocation = self.daofactory(classname="Files.SetLocationForWorkQueue")
        self.setFileAddChecksum = self.daofactory(classname="Files.AddChecksumByLFN")
        self.addFileAction = self.daofactory(classname="Files.Add")
        self.addToFileset = self.daofactory(classname="Files.AddDupsToFileset")
        self.getLocationInfo = self.daofactory(classname="Locations.GetSiteInfo")

        # DAOs from DBSBuffer
        self.dbsCreateFiles = self.dbsDaoFactory(classname="DBSBufferFiles.Add")
        self.dbsSetLocation = self.dbsDaoFactory(classname="DBSBufferFiles.SetLocationByLFN")
        self.dbsInsertLocation = self.dbsDaoFactory(classname="DBSBufferFiles.AddLocation")
        self.dbsSetChecksum = self.dbsDaoFactory(classname="DBSBufferFiles.AddChecksumByLFN")
        self.dbsInsertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow")

        # Added for file creation bookkeeping
        self.dbsFilesToCreate = set()
        self.wmbsFilesToCreate = set()
        self.insertedBogusDataset = -1

        return

    def createSandbox(self):
        """Create the runtime sandbox"""
        sandboxCreator = SandboxCreator()
        sandboxCreator.makeSandbox(self.cachepath, self.wmSpec)

    def createTopLevelFileset(self, topLevelFilesetName=None):
        """
        _createTopLevelFileset_

        Create the top level fileset for the workflow.  If the name of the top
        level fileset is not given create one.
        """
        if topLevelFilesetName is None:
            filesetName = ("%s-%s" % (self.wmSpec.name(),
                                      self.wmSpec.getTopLevelTask()[0].name()))
            if self.block:
                filesetName += "-%s" % self.block
            if self.mask:
                from hashlib import md5
                mask_string = ",".join(["%s=%s" % (x, self.mask[x]) for x in sorted(self.mask)])
                mask_string = encodeUnicodeToBytesConditional(mask_string, condition=PY3)
                filesetName += "-%s" % md5(mask_string).hexdigest()
        else:
            filesetName = topLevelFilesetName

        self.topLevelFileset = Fileset(filesetName)
        self.topLevelFileset.create()
        return

    def outputFilesetName(self, task, outputModuleName, dataTier=''):
        """
        _outputFilesetName_

        Generate an output fileset name for the given task, output module and data tier.
        """
        if task.taskType() == "Merge":
            outputFilesetName = "%s/merged-%s%s" % (task.getPathName(),
                                                    outputModuleName, dataTier)
        else:
            outputFilesetName = "%s/unmerged-%s%s" % (task.getPathName(),
                                                      outputModuleName, dataTier)

        return outputFilesetName

    def createSubscription(self, task, fileset, alternativeFilesetClose=False):
        """
        _createSubscription_

        Create subscriptions in the database.
        This includes workflows in WMBS and DBSBuffer, output maps, datasets
        and phedex subscriptions, and filesets for each task below and including
        the given task.
        """
        self._createSubscriptionsInWMBS(task, fileset, alternativeFilesetClose)

        self._createWorkflowsInDBSBuffer()
        self._createDatasetSubscriptionsInDBSBuffer()

        return

    def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False,
                                   createSandbox=True):
        """
        __createSubscriptionsInWMBS_

        Create subscriptions in WMBS for all the tasks in the spec.  This
        includes filesets, workflows and the output map for each task.
        :param createSandbox: boolean flag to skip (re-)creation of the workload sandbox
        """
        # create runtime sandbox for workflow
        if createSandbox:
            self.createSandbox()

        # FIXME: Let workflow put in values if spec is missing them
        workflow = Workflow(spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"],
                            dn=self.wmSpec.getOwner().get("dn", "unknown"),
                            group=self.wmSpec.getOwner().get("group", "unknown"),
                            owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"),
                            owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"),
                            name=self.wmSpec.name(), task=task.getPathName(),
                            wfType=self.wmSpec.getDashboardActivity(),
                            alternativeFilesetClose=alternativeFilesetClose,
                            priority=self.wmSpec.priority())
        workflow.create()

        subscription = Subscription(fileset=fileset, workflow=workflow,
                                    split_algo=task.jobSplittingAlgorithm(),
                                    type=task.getPrimarySubType())
        subscription.create()

        ### FIXME: I'm pretty sure we can improve how we handle this site white/black list
        for site in task.siteWhitelist():
            subscription.addWhiteBlackList([{"site_name": site, "valid": True}])

        for site in task.siteBlacklist():
            subscription.addWhiteBlackList([{"site_name": site, "valid": False}])

        if self.topLevelSubscription is None:
            self.topLevelSubscription = subscription
            logging.info("Top level subscription %s created for %s", subscription["id"], self.wmSpec.name())
        else:
            logging.info("Child subscription %s created for %s", subscription["id"], self.wmSpec.name())

        outputModules = task.getOutputModulesForTask()
        ignoredOutputModules = task.getIgnoredOutputModulesForTask()
        for outputModule in outputModules:
            for outputModuleName in sorted(outputModule.listSections_()):
                if outputModuleName in ignoredOutputModules:
                    msg = "%s has %s as IgnoredOutputModule, skipping fileset creation."
                    logging.info(msg, task.getPathName(), outputModuleName)
                    continue
                dataTier = getattr(getattr(outputModule, outputModuleName), "dataTier", '')
                filesetName = self.outputFilesetName(task, outputModuleName, dataTier)
                outputFileset = Fileset(filesetName)
                outputFileset.create()
                outputFileset.markOpen(True)
                mergedOutputFileset = None

                for childTask in task.childTaskIterator():
                    if childTask.data.input.outputModule == outputModuleName:
                        childDatatier = getattr(childTask.data.input, 'dataTier', '')
                        if childTask.taskType() in ["Cleanup", "Merge"] and childDatatier != dataTier:
                            continue
                        elif childTask.taskType() == "Merge" and childDatatier == dataTier:
                            filesetName = self.outputFilesetName(childTask, "Merged", dataTier)
                            mergedOutputFileset = Fileset(filesetName)
                            mergedOutputFileset.create()
                            mergedOutputFileset.markOpen(True)

                            primaryDataset = getattr(getattr(outputModule, outputModuleName), "primaryDataset", None)
                            if primaryDataset is not None:
                                self.mergeOutputMapping[mergedOutputFileset.id] = primaryDataset

                        self._createSubscriptionsInWMBS(childTask, outputFileset,
                                                        alternativeFilesetClose, createSandbox=False)

                if mergedOutputFileset is None:
                    workflow.addOutput(outputModuleName + dataTier, outputFileset,
                                       outputFileset)
                else:
                    workflow.addOutput(outputModuleName + dataTier, outputFileset,
                                       mergedOutputFileset)

        return

    def addMCFakeFile(self):
        """Add a fake file for wmbs to run production over"""
        needed = ['FirstEvent', 'FirstLumi', 'FirstRun', 'LastEvent', 'LastLumi', 'LastRun']
        for key in needed:
            if self.mask and self.mask.get(key) is None:
                msg = 'Invalid value "%s" for %s' % (self.mask.get(key), key)
                raise WorkQueueWMBSException(msg)

        locations = set()
        siteInfo = self.getLocationInfo.execute(conn=self.getDBConn(),
                                                transaction=self.existingTransaction())
        for site in siteInfo:
            if site['pnn'] in self.commonLocation:
                locations.add(site['pnn'])

        if not locations:
            msg = 'No locations to inject Monte Carlo work to, unable to proceed'
            raise WorkQueueWMBSException(msg)
        mcFakeFileName = "MCFakeFile-%s" % self.topLevelFileset.name
        wmbsFile = File(lfn=mcFakeFileName,
                        first_event=self.mask['FirstEvent'],
                        last_event=self.mask['LastEvent'],
                        events=self.mask['LastEvent'] - self.mask['FirstEvent'] + 1,  # inclusive range
                        locations=locations,
                        merged=False,  # merged causes dbs parentage relation
                        )

        if self.mask:
            lumis = list(range(self.mask['FirstLumi'], self.mask['LastLumi'] + 1))  # inclusive range
            wmbsFile.addRun(Run(self.mask['FirstRun'], *lumis))  # assume run number static
        else:
            wmbsFile.addRun(Run(1, 1))

        wmbsFile['inFileset'] = True  # file is not a parent

        logging.debug("WMBS MC Fake File: %s on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations'])

        self.wmbsFilesToCreate.add(wmbsFile)

        totalFiles = self.topLevelFileset.addFilesToWMBSInBulk(self.wmbsFilesToCreate,
                                                               self.wmSpec.name(),
                                                               isDBS=self.isDBS)

        self.topLevelFileset.markOpen(False)
        return totalFiles

    def createSubscriptionAndAddFiles(self, block):
        """
        _createSubscriptionAndAddFiles_

        Create the subscription and add files at one time to
        put everything in one transaction.

        """
        existingTransaction = self.beginTransaction()

        self.createTopLevelFileset()
        try:
            self.createSubscription(self.topLevelTask, self.topLevelFileset)
        except Exception as ex:
            myThread = threading.currentThread()
            myThread.transaction.rollback()
            logging.exception("Failed to create subscription. Error: %s", str(ex))
            raise ex

        if block:
            logging.info('"%s" Injecting block %s (%d files) into wmbs.',
                         self.wmSpec.name(), self.block, len(block['Files']))
            addedFiles = self.addFiles(block)
        else:
            # For MC case
            logging.info('"%s" Injecting production %s:%s:%s - %s:%s:%s (run:lumi:event) into wmbs',
                         self.wmSpec.name(),
                         self.mask['FirstRun'], self.mask['FirstLumi'], self.mask['FirstEvent'],
                         self.mask['LastRun'], self.mask['LastLumi'], self.mask['LastEvent'])
            addedFiles = self.addMCFakeFile()

        self.commitTransaction(existingTransaction)
        logging.info("Transaction committed: %s, for %s", not existingTransaction, self.wmSpec.name())

        # Now that we've created those files, clear the list
        self.dbsFilesToCreate = set()
        self.wmbsFilesToCreate = set()

        return self.topLevelSubscription, addedFiles

    def addFiles(self, block):
        """
        _addFiles_

        create wmbs files from given dbs block.
        as well as run lumi update
        """
        if self.topLevelTask.getInputACDC():
            self.isDBS = False
            logging.info('Adding ACDC files into WMBS for %s', self.wmSpec.name())
            for acdcFile in self.validFiles(block['Files']):
                self._addACDCFileToWMBSFile(acdcFile)
        else:
            self.isDBS = True
            blockPNNs = block['PhEDExNodeNames']
            logging.info('Adding files into WMBS for %s with PNNs: %s', self.wmSpec.name(), blockPNNs)
            for dbsFile in self.validFiles(block['Files']):
                self._addDBSFileToWMBSFile(dbsFile, blockPNNs)

        # Add files to WMBS
        logging.info('Inserting %d files in bulk into WMBS for %s', len(self.wmbsFilesToCreate),
                     self.wmSpec.name())
        totalFiles = self.topLevelFileset.addFilesToWMBSInBulk(self.wmbsFilesToCreate,
                                                               self.wmSpec.name(),
                                                               isDBS=self.isDBS)
        # Add files to DBSBuffer
        logging.info('Inserting %d files in bulk into DBSBuffer for %s', len(self.dbsFilesToCreate),
                     self.wmSpec.name())
        self._createFilesInDBSBuffer()

        # DBS blocks are always closed, so mark fileset as closed as well
        self.topLevelFileset.markOpen(False)
        return totalFiles

    def getMergeOutputMapping(self):
        """
        _getMergeOutputMapping_

        retrieves the relationship between primary
        dataset and merge output fileset ids for
        all merge tasks created
        """
        return self.mergeOutputMapping

    def _createWorkflowsInDBSBuffer(self):
        """
        _createWorkflowsInDBSBuffer_

        Register workflow information and settings in dbsbuffer for all
        tasks that will potentially produce any output in this spec.
        """
        existingTransaction = self.beginTransaction()

        for task in self.wmSpec.listOutputProducingTasks():
            workflow_id = self.dbsInsertWorkflow.execute(self.wmSpec.name(), task,
                                                         self.wmSpec.getBlockCloseMaxWaitTime(),
                                                         self.wmSpec.getBlockCloseMaxFiles(),
                                                         self.wmSpec.getBlockCloseMaxEvents(),
                                                         self.wmSpec.getBlockCloseMaxSize(),
                                                         conn=self.getDBConn(), transaction=self.existingTransaction())
            if task == self.topLevelTask.getPathName():
                self.topLevelTaskDBSBufferId = workflow_id

        self.commitTransaction(existingTransaction)

    def _createDatasetSubscriptionsInDBSBuffer(self):
        """
        _createDatasetSubscriptionsInDBSBuffer_

        Insert the subscriptions defined in the workload for the output
        datasets with the different options.
        """
        existingTransaction = self.beginTransaction()

        subInfo = self.wmSpec.getSubscriptionInformation()
        for dataset in subInfo:
            dbsDataset = DBSBufferDataset(path=dataset)
            dbsDataset.create()
            dbsDataset.addSubscription(subInfo[dataset])
        self.commitTransaction(existingTransaction)
        return

    def _createFilesInDBSBuffer(self):
        """
        _createFilesInDBSBuffer_

        It does the actual job of creating things in DBSBuffer

        """
        if not self.dbsFilesToCreate:
            # Whoops, nothing to do!
            return

        dbsFileTuples = set()
        dbsFileLoc = []
        dbsCksumBinds = []
        locationsToAdd = set()

        # The first thing we need to do is add the datasetAlgo
        # Assume all files in a pass come from one datasetAlgo?
        if self.insertedBogusDataset == -1:
            self.insertedBogusDataset = next(iter(self.dbsFilesToCreate)).insertDatasetAlgo()

        for dbsFile in self.dbsFilesToCreate:
            # Append a tuple in the format specified by DBSBufferFiles.Add
            # Also run insertDatasetAlgo

            lfn = dbsFile['lfn']
            selfChecksums = dbsFile['checksums']

            newTuple = (lfn, dbsFile['size'], dbsFile['events'],
                        self.insertedBogusDataset, dbsFile['status'],
                        self.topLevelTaskDBSBufferId, dbsFile['in_phedex'])

            dbsFileTuples.add(newTuple)

            if len(dbsFile['newlocations']) < 1:
                msg = ''
                msg += "File created without any locations!\n"
                msg += "File lfn: %s\n" % (lfn)
                msg += "Rejecting this group of files in DBS!\n"
                logging.error(msg)
                raise WorkQueueWMBSException(msg)

            for jobLocation in dbsFile['newlocations']:
                locationsToAdd.add(jobLocation)
                dbsFileLoc.append({'lfn': lfn, 'pnn': jobLocation})

            if selfChecksums:
                # If we have checksums we have to create a bind
                # For each different checksum
                for entry in selfChecksums:
                    dbsCksumBinds.append({'lfn': lfn, 'cksum': selfChecksums[entry],
                                          'cktype': entry})

        self.dbsInsertLocation.execute(siteName=locationsToAdd,
                                       conn=self.getDBConn(),
                                       transaction=self.existingTransaction())

        self.dbsCreateFiles.execute(files=dbsFileTuples,
                                    conn=self.getDBConn(),
                                    transaction=self.existingTransaction())

        self.dbsSetLocation.execute(binds=dbsFileLoc,
                                    conn=self.getDBConn(),
                                    transaction=self.existingTransaction())

        if len(dbsCksumBinds) > 0:
            self.dbsSetChecksum.execute(bulkList=dbsCksumBinds,
                                        conn=self.getDBConn(),
                                        transaction=self.existingTransaction())

        return

    def _addToDBSBuffer(self, dbsFile, checksums, locations):
        """
        This step is just for increase the performance for
        Accountant doesn't neccessary to check the parentage
        """
        dbsBuffer = DBSBufferFile(lfn=dbsFile["LogicalFileName"],
                                  size=dbsFile["FileSize"],
                                  events=dbsFile["NumberOfEvents"],
                                  checksums=checksums,
                                  locations=locations,
                                  status="GLOBAL",
                                  inPhedex=1)
        dbsBuffer.setDatasetPath('bogus')
        dbsBuffer.setAlgorithm(appName="cmsRun", appVer="Unknown",
                               appFam="Unknown", psetHash="Unknown",
                               configContent="Unknown")

        self.dbsFilesToCreate.add(dbsBuffer)
        return

    def _addDBSFileToWMBSFile(self, dbsFile, storageElements, inFileset=True):
        """
        There are two assumptions made to make this method behave properly,
        1. DBS returns only one level of ParentList.
           If DBS returns multiple level of parentage, it will be still get handled.
           However that might not be what we wanted. In that case, restrict to one level.
        2. Assumes parents files are in the same location as child files.
           This is not True in general case, but workquue should only select work only
           where child and parent files are in the same location
        """
        # TODO get dbsFile with lumi event information
        wmbsParents = []
        dbsFile.setdefault("ParentList", [])
        for parent in dbsFile["ParentList"]:
            wmbsParents.append(self._addDBSFileToWMBSFile(parent,
                                                          storageElements, inFileset=False))

        checksums = {}
        if dbsFile.get('Checksum'):
            checksums['cksum'] = dbsFile['Checksum']
        if dbsFile.get('Adler32'):
            checksums['adler32'] = dbsFile['Adler32']

        wmbsFile = File(lfn=dbsFile["LogicalFileName"],
                        size=dbsFile["FileSize"],
                        events=dbsFile["NumberOfEvents"],
                        checksums=checksums,
                        # TODO: need to get list of parent lfn
                        parents=wmbsParents,
                        locations=set(storageElements))

        for lumi in dbsFile['LumiList']:
            if isinstance(lumi['LumiSectionNumber'], list):
                lumiSecList = (list(zip(lumi['LumiSectionNumber'], lumi['EventCount']))
                               if 'EventCount' in lumi else lumi['LumiSectionNumber'])
                run = Run(lumi['RunNumber'], lumiSecList)
            else:
                lumiSecTuple = ((lumi['LumiSectionNumber'], lumi['EventCount'])
                                if 'EventCount' in lumi else lumi['LumiSectionNumber'])
                run = Run(lumi['RunNumber'], lumiSecTuple)
            wmbsFile.addRun(run)

        self._addToDBSBuffer(dbsFile, checksums, storageElements)

        logging.debug("WMBS File: %s on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations'])

        wmbsFile['inFileset'] = bool(inFileset)
        self.wmbsFilesToCreate.add(wmbsFile)

        return wmbsFile

    def _convertACDCFileToDBSFile(self, acdcFile):
        """
        convert ACDCFiles to dbs file format
        """
        dbsFile = {}
        dbsFile["LogicalFileName"] = acdcFile["lfn"]
        dbsFile["FileSize"] = acdcFile["size"]
        dbsFile["NumberOfEvents"] = acdcFile["events"]
        return dbsFile

    def _addACDCFileToWMBSFile(self, acdcFile, inFileset=True):
        """
        adds the ACDC files into WMBS database
        """
        wmbsParents = []
        # If file is merged, then it will be the parent of whatever output that
        # process this job (it twists my mind!). Meaning, block below can be skipped
        if int(acdcFile.get("merged", 0)) == 0 and acdcFile["parents"]:
            # set the parentage for all the unmerged parents
            for parent in acdcFile["parents"]:
                if parent.startswith("/store/unmerged/") or parent.startswith("MCFakeFile"):
                    logging.warning("WMBS ACDC skipped parent invalid file: %s", parent)
                    continue
                logging.debug("WMBS ACDC Parent File: %s", parent)
                parent = self._addACDCFileToWMBSFile(DatastructFile(lfn=parent,
                                                                    locations=acdcFile["locations"],
                                                                    merged=True),
                                                     inFileset=False)
                wmbsParents.append(parent)

        # pass empty check sum since it won't be updated to dbs anyway
        checksums = {}
        wmbsFile = File(lfn=str(acdcFile["lfn"]),
                        size=acdcFile["size"],
                        events=acdcFile["events"],
                        first_event=acdcFile.get('first_event', 0),
                        last_event=acdcFile.get('last_event', 0),
                        checksums=checksums,
                        parents=wmbsParents,
                        locations=acdcFile["locations"],
                        merged=acdcFile.get('merged', True))

        ## TODO need to get the lumi lists
        for run in acdcFile['runs']:
            wmbsFile.addRun(run)

        if not acdcFile["lfn"].startswith("/store/unmerged") or wmbsParents:
            # only add to DBSBuffer if is not unmerged file or it has parents.
            dbsFile = self._convertACDCFileToDBSFile(acdcFile)
            self._addToDBSBuffer(dbsFile, checksums, acdcFile["locations"])

        logging.debug("WMBS ACDC File: %s on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations'])

        wmbsFile['inFileset'] = bool(inFileset)

        self.wmbsFilesToCreate.add(wmbsFile)

        return wmbsFile

    def validFiles(self, files):
        """
        Apply lumi mask and or run white/black list and return files which have
        one or more of the requested lumis
        """
        runWhiteList = self.topLevelTask.inputRunWhitelist()
        runBlackList = self.topLevelTask.inputRunBlacklist()
        lumiMask = self.topLevelTask.getLumiMask()

        blackMask = None
        if lumiMask:  # We have a lumiMask, so use it and modify with run white/black list
            if runWhiteList:
                lumiMask.selectRuns(runWhiteList)
            if runBlackList:
                lumiMask.removeRuns(runBlackList)
        elif runWhiteList:  # We have a run whitelist, subtract off blacklist
            lumiMask = LumiList(runs=runWhiteList)
            if runBlackList:  # We only have a blacklist, so make a black mask out of it instead
                lumiMask.removeRuns(runBlackList)
        else:
            lumiMask = None
            if runBlackList:
                blackMask = LumiList(runs=runBlackList)

        results = []
        for f in files:
            if isinstance(f, (newstr, bytes)) or "LumiList" not in f:
                results.append(f)
                continue

            # Create a LumiList from the WMBS info
            runLumis = {}
            for x in f['LumiList']:
                if x['RunNumber'] in runLumis:
                    runLumis[x['RunNumber']].extend(x['LumiSectionNumber'])
                else:
                    runLumis[x['RunNumber']] = x['LumiSectionNumber']
            fileLumiList = LumiList(runsAndLumis=runLumis)

            if lumiMask:
                if fileLumiList & lumiMask:  # At least one lumi from file is in lumiMask
                    results.append(f)
            elif blackMask:
                if fileLumiList - blackMask:  # At least one lumi from file is not in blackMask
                    results.append(f)
            else:  # There is effectively no mask
                results.append(f)

        return results
