#!/usr/bin/env python
"""
WorkQueueBackend

Interface to WorkQueue persistent storage
"""

from builtins import object
from math import ceil

from future.utils import viewitems

import json
import random
import time

from WMCore.Database.CMSCouch import CouchServer, CouchNotFoundError, Document
from WMCore.Lexicon import sanitizeURL
from WMCore.WMSpec.WMWorkload import WMWorkloadHelper
from WMCore.WorkQueue.DataStructs.CouchWorkQueueElement import CouchWorkQueueElement, fixElementConflicts
from WMCore.WorkQueue.DataStructs.WorkQueueElement import possibleSites
from WMCore.WorkQueue.WorkQueueExceptions import WorkQueueNoMatchingElements, WorkQueueError


def formatReply(answer, *items):
    """Take reply from couch bulk api and format labeling errors etc
    """
    result, errors = [], []
    for ans in answer:
        if 'error' in ans:
            errors.append(ans)
            continue
        for item in items:
            if item.id == ans['id']:
                item.rev = ans['rev']
                result.append(item)
                break
    return result, errors


def sortAvailableElements(elementsList):
    """
    Given a list of workqueue elements dictionary, this function will
    sort them in place, first by their creation time; secondly by their
    priority. In other words, higher priority and older requests will
    be first in the list
    :param elementsList: a list of elements dictionary
    :return: nothing, list is updated in place
    """
    elementsList.sort(key=lambda element: element['CreationTime'])
    elementsList.sort(key=lambda element: element['Priority'], reverse=True)


class WorkQueueBackend(object):
    """
    Represents persistent storage for WorkQueue
    """

    def __init__(self, db_url, db_name='workqueue',
                 inbox_name=None, parentQueue=None,
                 queueUrl=None, logger=None):
        if logger:
            self.logger = logger
        else:
            import logging
            self.logger = logging

        if inbox_name is None:
            inbox_name = "%s_inbox" % db_name

        self.server = CouchServer(db_url)
        self.parentCouchUrlWithAuth = parentQueue
        if parentQueue:
            self.parentCouchUrl = sanitizeURL(parentQueue)['url']
        else:
            self.parentCouchUrl = None
        self.db = self.server.connectDatabase(db_name, create=False, size=10000)
        self.hostWithAuth = db_url
        self.inbox = self.server.connectDatabase(inbox_name, create=False, size=10000)
        self.queueUrlWithAuth = queueUrl or (db_url + '/' + db_name)
        self.queueUrl = sanitizeURL(queueUrl or (db_url + '/' + db_name))['url']
        self.eleKey = 'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'

    def forceQueueSync(self):
        """Setup CouchDB replications - used only in tests"""
        self.pullFromParent(continuous=True)
        self.sendToParent(continuous=True)

    def pullFromParent(self, continuous=True, cancel=False):
        """Replicate from parent couch - blocking: used only in unit tests"""
        try:
            if self.parentCouchUrlWithAuth and self.queueUrlWithAuth:
                self.logger.info("Forcing pullFromParent from parentCouch: %s to queueUrl %s/%s",
                                 self.parentCouchUrlWithAuth, self.queueUrlWithAuth, self.inbox.name)
                self.server.replicate(source=self.parentCouchUrlWithAuth,
                                      destination="%s/%s" % (self.hostWithAuth, self.inbox.name),
                                      filter='WorkQueue/queueFilter',
                                      query_params={'childUrl': self.queueUrl,
                                                    'parentUrl': self.parentCouchUrl},
                                      continuous=continuous,
                                      cancel=cancel,
                                      sleepSecs=6)
        except Exception as ex:
            self.logger.warning('Replication from %s failed: %s' % (self.parentCouchUrlWithAuth, str(ex)))

    def sendToParent(self, continuous=True, cancel=False):
        """Replicate to parent couch - blocking: used only int test"""
        try:
            if self.parentCouchUrlWithAuth and self.queueUrlWithAuth:
                self.logger.info("Forcing sendToParent from queueUrl %s/%s to parentCouch: %s",
                                 self.queueUrlWithAuth, self.inbox.name, self.parentCouchUrlWithAuth)
                self.server.replicate(source="%s" % self.inbox.name,
                                      destination=self.parentCouchUrlWithAuth,
                                      filter='WorkQueue/queueFilter',
                                      query_params={'childUrl': self.queueUrl,
                                                    'parentUrl': self.parentCouchUrl},
                                      continuous=continuous,
                                      cancel=cancel)
        except Exception as ex:
            self.logger.warning('Replication to %s failed: %s' % (self.parentCouchUrlWithAuth, str(ex)))

    def getElementsForSplitting(self):
        """Returns the elements from the inbox that need to be split,
        if WorkflowName specified only return elements to split for that workflow"""
        elements = self.getInboxElements(status='Negotiating')
        specs = {}  # cache as may have multiple elements for same spec
        for ele in elements:
            if ele['RequestName'] not in specs:
                wmspec = WMWorkloadHelper()
                wmspec.load(self.parentCouchUrlWithAuth + "/%s/spec" % ele['RequestName'])
                specs[ele['RequestName']] = wmspec
            ele['WMSpec'] = specs[ele['RequestName']]
        del specs
        return elements

    def insertWMSpec(self, wmspec):
        """
        Insert WMSpec to backend
        """
        # Can't save spec to inbox, it needs to be visible to child queues
        # Can't save empty dict so add dummy variable
        dummyValues = {'name': wmspec.name()}
        # change specUrl in spec before saving (otherwise it points to previous url)
        wmspec.setSpecUrl(self.db['host'] + "/%s/%s/spec" % (self.db.name, wmspec.name()))
        return wmspec.saveCouch(self.hostWithAuth, self.db.name, dummyValues)

    def getWMSpec(self, name):
        """Get the spec"""
        wmspec = WMWorkloadHelper()
        wmspec.load(self.hostWithAuth + "/%s/%s/spec" % (self.db.name, name))
        return wmspec

    def insertElements(self, units, parent=None):
        """
        Insert element to database

        @param parent is the parent WorkQueueObject these element's belong to.
                                            i.e. a workflow which has been split
        """
        if not units:
            return []
        # store spec file separately - assume all elements share same spec
        self.insertWMSpec(units[0]['WMSpec'])
        newUnitsInserted = []
        for unit in units:
            # cast to couch
            if not isinstance(unit, CouchWorkQueueElement):
                unit = CouchWorkQueueElement(self.db, elementParams=dict(unit))

            if parent:
                unit['ParentQueueId'] = parent.id
                unit['TeamName'] = parent['TeamName']
                unit['WMBSUrl'] = parent['WMBSUrl']

            if unit._couch.documentExists(unit.id):
                self.logger.info('Element "%s" already exists, skip insertion.' % unit.id)
                continue

            newUnitsInserted.append(unit)
            unit.save()
            # FIXME: this is not performing bulk request, but single document commits(!)
            unit._couch.commit()

        return newUnitsInserted

    def createWork(self, spec, **kwargs):
        """Return the Inbox element for this spec.

        This does not persist it to the database.
        """
        kwargs.update({'WMSpec': spec,
                       'RequestName': spec.name(),
                       'StartPolicy': spec.startPolicyParameters(),
                       'EndPolicy': spec.endPolicyParameters(),
                       'OpenForNewData': True
                       })
        unit = CouchWorkQueueElement(self.inbox, elementParams=kwargs)
        unit.id = spec.name()
        return unit

    def getElements(self, status=None, elementIDs=None, returnIdOnly=False,
                    db=None, loadSpec=False, WorkflowName=None, **elementFilters):
        """Return elements that match requirements

        status, elementIDs & filters are 'AND'ed together to filter elements.
        returnIdOnly causes the element not to be loaded and only the id returned
        db is used to specify which database to return from
        loadSpec causes the workflow for each spec to be loaded.
        WorkflowName may be used in the place of RequestName
        """
        key = []
        if not db:
            db = self.db
        if elementFilters.get('RequestName') and not WorkflowName:
            WorkflowName = elementFilters.pop('RequestName')

        if elementIDs:
            if elementFilters or status or returnIdOnly:
                msg = "Can't specify extra filters (or return id's) when using element id's with getElements()"
                raise ValueError(msg)
            elements = [CouchWorkQueueElement(db, i).load() for i in elementIDs]
        else:
            options = {'include_docs': True, 'filter': elementFilters, 'idOnly': returnIdOnly, 'reduce': False}
            # filter on workflow or status if possible
            filterName = 'elementsByWorkflow'
            if WorkflowName:
                key.append(WorkflowName)
            elif status:
                filterName = 'elementsByStatus'
                key.append(status)
            elif elementFilters.get('SubscriptionId'):
                key.append(elementFilters['SubscriptionId'])
                filterName = 'elementsBySubscription'
            # add given params to filters
            if status:
                options['filter']['Status'] = status
            if WorkflowName:
                options['filter']['RequestName'] = WorkflowName

            view = db.loadList('WorkQueue', 'filter', filterName, options, key)
            view = json.loads(view)
            if returnIdOnly:
                return view
            elements = [CouchWorkQueueElement.fromDocument(db, row) for row in view]

        if loadSpec:
            specs = {}  # cache as may have multiple elements for same spec
            for ele in elements:
                if ele['RequestName'] not in specs:
                    wmspec = self.getWMSpec(ele['RequestName'])
                    specs[ele['RequestName']] = wmspec
                ele['WMSpec'] = specs[ele['RequestName']]
            del specs
        return elements

    def getInboxElements(self, *args, **kwargs):
        """
        Return elements from Inbox, supports same semantics as getElements()
        """
        return self.getElements(*args, db=self.inbox, **kwargs)

    def getElementsForWorkflow(self, workflow):
        """Get elements for a workflow"""
        elements = self.db.loadView('WorkQueue', 'elementsByWorkflow',
                                    {'key': workflow, 'include_docs': True, 'reduce': False})
        return [CouchWorkQueueElement.fromDocument(self.db,
                                                   x['doc'])
                for x in elements.get('rows', [])]

    def getElementsForParent(self, parent):
        """Get elements with the given parent"""
        elements = self.db.loadView('WorkQueue', 'elementsByParent', {'key': parent.id, 'include_docs': True})
        return [CouchWorkQueueElement.fromDocument(self.db,
                                                   x['doc'])
                for x in elements.get('rows', [])]

    def saveElements(self, *elements):
        """Persist elements

        Returns elements successfully saved, user must verify to catch errors
        """
        result = []
        if not elements:
            return result
        for element in elements:
            element.save()
        answer = elements[0]._couch.commit()
        result, failures = formatReply(answer, *elements)
        msg = 'Couch error saving element: "%s", error "%s", reason "%s"'
        for failed in failures:
            self.logger.error(msg % (failed['id'], failed['error'], failed['reason']))
        return result

    def _raiseConflictErrorAndLog(self, conflictIDs, updatedParams, dbName="workqueue"):
        errorMsg = "Need to update this element manually from %s\n ids:%s\n, parameters:%s\n" % (
            dbName, conflictIDs, updatedParams)
        self.logger.error(errorMsg)
        raise WorkQueueError(errorMsg)

    def updateElements(self, *elementIds, **updatedParams):
        """Update given element's (identified by id) with new parameters"""
        if not elementIds:
            return
        eleParams = {}
        eleParams[self.eleKey] = updatedParams
        conflictIDs = self.db.updateBulkDocumentsWithConflictHandle(elementIds, eleParams)
        if conflictIDs:
            self._raiseConflictErrorAndLog(conflictIDs, updatedParams)
        return

    def updateInboxElements(self, *elementIds, **updatedParams):
        """Update given inbox element's (identified by id) with new parameters"""
        if not elementIds:
            return
        eleParams = {}
        eleParams[self.eleKey] = updatedParams
        conflictIDs = self.inbox.updateBulkDocumentsWithConflictHandle(elementIds, eleParams)
        if conflictIDs:
            self._raiseConflictErrorAndLog(conflictIDs, updatedParams, "workqueue_inbox")
        return

    def deleteElements(self, *elements):
        """Delete elements"""
        if not elements:
            return
        specs = {}
        for i in elements:
            i.delete()
            specs[i['RequestName']] = None
        answer = elements[0]._couch.commit()
        _, failures = formatReply(answer, *elements)
        msg = 'Couch error deleting element: "%s", error "%s", reason "%s"'
        for failed in failures:
            # only count delete as failed if document still exists
            if elements[0]._couch.documentExists(failed['id']):
                self.logger.error(msg % (failed['id'], failed['error'], failed['reason']))
        # delete specs if no longer used
        for wf in specs:
            try:
                if not self.db.loadView('WorkQueue', 'elementsByWorkflow',
                                        {'key': wf, 'limit': 1, 'reduce': False})['rows']:
                    self.db.delete_doc(wf)
            except CouchNotFoundError:
                pass

    def calculateAvailableWork(self, thresholds, siteJobCounts):
        """
        A short version of the `availableWork` method, which is used only to calculate
        the amount of work already available at the local workqueue.
        :param thresholds: a dictionary key'ed by the site name, values representing the
            maximum number of jobs allowed at that site.
        :param siteJobCounts: a dictionary-of-dictionaries key'ed by the site name; value
            is a dictionary with the number of jobs running at a given priority.
        :return: a tuple with the elements accepted and an overview of job counts per site
        """
        # NOTE: this method can be less verbose as well
        elements = []
        # If there are no sites, punt early.
        if not thresholds:
            self.logger.error("No thresholds is set: Please check")
            return elements, siteJobCounts

        self.logger.info("Calculating available work from queue %s", self.queueUrl)

        options = {}
        options['include_docs'] = True
        options['descending'] = True
        options['resources'] = thresholds
        options['num_elem'] = 9999999  # magic number!
        result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
        result = json.loads(result)
        self.logger.info("Retrieved %d elements from workRestrictions list for: %s",
                         len(result), self.queueUrl)

        # Convert python dictionary into Couch WQE objects
        # And sort them by creation time and priority, such that highest priority and
        # oldest elements come first in the list
        sortedElements = []
        for item in result:
            element = CouchWorkQueueElement.fromDocument(self.db, item)
            sortedElements.append(element)
        sortAvailableElements(sortedElements)

        for element in sortedElements:
            commonSites = possibleSites(element)
            prio = element['Priority']
            # shuffle list of common sites all the time to give everyone the same chance
            random.shuffle(commonSites)
            possibleSite = None
            for site in commonSites:
                if site in thresholds:
                    # Count the number of jobs currently running of greater priority, if they
                    # are less than the site thresholds, then accept this element
                    curJobCount = sum([x[1] if x[0] >= prio else 0 for x in viewitems(siteJobCounts.get(site, {}))])
                    self.logger.debug("Job Count: %s, site: %s thresholds: %s", curJobCount, site, thresholds[site])
                    if curJobCount < thresholds[site]:
                        possibleSite = site
                        break

            if possibleSite:
                self.logger.debug("Meant to accept workflow: %s, with prio: %s, element id: %s, for site: %s",
                                  element['RequestName'], prio, element.id, possibleSite)
                elements.append(element)
                siteJobCounts.setdefault(possibleSite, {})
                siteJobCounts[possibleSite][prio] = siteJobCounts[possibleSite].setdefault(prio, 0) + \
                                                    element['Jobs'] * element.get('blowupFactor', 1.0)
            else:
                self.logger.debug("No available resources for %s with localdoc id %s",
                                  element['RequestName'], element.id)

        self.logger.info("And %d elements passed location and siteJobCounts restrictions for: %s",
                         len(elements), self.queueUrl)
        return elements, siteJobCounts

    def availableWork(self, thresholds, siteJobCounts, team=None, excludeWorkflows=None,
                      numElems=1000, rowsPerSlice=1000, maxRows=1000):
        """
        Get work - either from local or global queue - which is available to be run.

        :param thresholds: a dictionary key'ed by the site name, values representing the
            maximum number of jobs allowed at that site.
        :param siteJobCounts: a dictionary-of-dictionaries key'ed by the site name; value
            is a dictionary with the number of jobs running at a given priority.
        :param team: a string with the team name we want to pull work for
        :param excludeWorkflows: list of (aborted) workflows that should not be accepted
        :param numElems: integer with the maximum number of elements to be accepted.
        :param rowsPerSlice: integer defining the amount of rows for each slice (slices
             of a couchdb view request).
        :param maxRows: maximum number of available elements (rows) to be considered
            when pulling work down to the agent.
        :return: a tuple with the elements accepted and an overview of job counts per site
        """
        excludeWorkflows = excludeWorkflows or []
        acceptedElems = []
        # If there are no sites, punt early.
        if not thresholds:
            self.logger.error("No thresholds is set: Please check")
            return acceptedElems, siteJobCounts

        self.logger.info("Current siteJobCounts:")
        for site, jobsByPrio in viewitems(siteJobCounts):
            self.logger.info("    %s : %s", site, jobsByPrio)

        # Find out how many elements are in Available status
        numAvail = self.queueLength()
        self.logger.info("Current amount of WQEs in Available status: %s", numAvail)

        self.logger.info("Getting up to %d available work from %s", numElems, self.queueUrl)
        self.logger.info("  for team name: %s", team)
        self.logger.info("  with excludeWorkflows count: %s", len(excludeWorkflows))
        self.logger.debug("  with excludeWorkflows: %s", excludeWorkflows)
        self.logger.info("  for thresholds: %s", thresholds)

        options = {}
        options['include_docs'] = True
        options['descending'] = True
        options['resources'] = thresholds
        options['limit'] = rowsPerSlice
        # FIXME: num_elem option can likely be deprecated, but it needs synchronization
        # between agents and global workqueue... for now, make sure it can return the slice size
        options['num_elem'] = rowsPerSlice
        if team:
            options['team'] = team

        # Fetch workqueue elements in slices, using the CouchDB "limit" and "skip"
        # options for couch views. Conditions to stop this loop are:
        #  a) stop once total_rows is reached (exhausted all available elements)
        #  b) hit maximum allowed elements/rows to be considered for data acquisition (maxRows)
        #  c) or, once the targeted number of elements has been accepted (numElems)
        numSlices = ceil(numAvail / rowsPerSlice)
        numSlices = min(numSlices, int(maxRows / rowsPerSlice))
        for sliceNum in range(numSlices):
            # documents to skip as a function of the slice number
            options['skip'] = sliceNum * rowsPerSlice
            self.logger.info("  for slice: %s, with rows range [%s - %s]",
                             sliceNum, options['skip'], options['skip'] + options['limit'])

            result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
            # now check the remaining restrictions and priority
            wqeSlots = numElems - len(acceptedElems)
            elems = self._evalAvailableWork(json.loads(result), thresholds, siteJobCounts,
                                            excludeWorkflows, wqeSlots)
            acceptedElems.extend(elems)
            if len(acceptedElems) >= numElems:
                msg = f"Reached maximum number of elements to be accepted, "
                msg += f"configured to: {numElems}, from queue: {self.queueUrl}"
                self.logger.info(msg)
                break

        self.logger.info("Total of %d elements passed location and siteJobCounts restrictions for: %s",
                         len(acceptedElems), self.queueUrl)
        return acceptedElems, siteJobCounts

    def _evalAvailableWork(self, listElems, thresholds, siteJobCounts,
                           excludeWorkflows, numElems):
        """
        Evaluate work available in workqueue and decide whether it can be
        accepted or not.

        :param listElems: list of dictionaries that correspond to the workqueue elements.
        :param thresholds: a dictionary key'ed by the site name, values representing the
            maximum number of jobs allowed at that site.
        :param siteJobCounts: a dictionary-of-dictionaries key'ed by the site name; value
            is a dictionary with the number of jobs running at a given priority.
            NOTE that it is updated in place.
        :param excludeWorkflows: list of (aborted) workflows that should not be accepted
        :param numElems: integer with the maximum number of elements to be accepted (default
            to a very large number when pulling work from local queue, read unlimited)
        :return: a tuple with the elements accepted and an overview of job counts per site
        """
        elems = []
        self.logger.info("Retrieved %d elements from workRestrictions list for: %s",
                         len(listElems), self.queueUrl)
        # Convert python dictionary into Couch WQE objects, skipping aborted workflows
        # And sort them by creation time and priority, such that highest priority and
        # oldest elements come first in the list
        sortedElements = []
        for i in listElems:
            element = CouchWorkQueueElement.fromDocument(self.db, i)
            # make sure not to acquire work for aborted or force-completed workflows
            if element['RequestName'] in excludeWorkflows:
                msg = "Skipping aborted/force-completed workflow: %s, work id: %s"
                self.logger.info(msg, element['RequestName'], element._id)
            elif element['Status'] != 'Available':
                # Extra safety mechanism, see https://github.com/dmwm/WMCore/pull/12050
                msg = "Skipping element in unwanted status: %s, work id: %s"
                self.logger.warning(msg, element['Status'], element._id)
            else:
                sortedElements.append(element)
        sortAvailableElements(sortedElements)

        for element in sortedElements:
            if numElems <= 0:
                # it means we accepted the configured number of elements
                break
            commonSites = possibleSites(element)
            prio = element['Priority']
            # shuffle list of common sites all the time to give everyone the same chance
            random.shuffle(commonSites)
            possibleSite = None
            for site in commonSites:
                if site in thresholds:
                    # Count the number of jobs currently running of greater priority, if they
                    # are less than the site thresholds, then accept this element
                    curJobCount = sum([x[1] if x[0] >= prio else 0 for x in viewitems(siteJobCounts.get(site, {}))])
                    self.logger.debug("Job Count: %s, site: %s thresholds: %s",
                                      curJobCount, site, thresholds[site])
                    if curJobCount < thresholds[site]:
                        possibleSite = site
                        break

            if possibleSite:
                self.logger.info("Accepting workflow: %s, with prio: %s, element id: %s, for site: %s",
                                 element['RequestName'], prio, element.id, possibleSite)
                numElems -= 1
                elems.append(element)
                siteJobCounts.setdefault(possibleSite, {})
                siteJobCounts[possibleSite][prio] = siteJobCounts[possibleSite].setdefault(prio, 0) + \
                                                    element['Jobs'] * element.get('blowupFactor', 1.0)
            else:
                self.logger.debug("No available resources for %s with doc id %s",
                                  element['RequestName'], element.id)

        self.logger.info("And %d elements passed location and siteJobCounts restrictions for: %s",
                         len(elems), self.queueUrl)
        return elems

    def getActiveData(self):
        """Get data items we have work in the queue for"""
        data = self.db.loadView('WorkQueue', 'activeData', {'reduce': True, 'group': True})
        return [{'dbs_url': x['key'][0],
                 'name': x['key'][1]} for x in data.get('rows', [])]

    def getActiveParentData(self):
        """Get data items we have work in the queue for with parent"""
        data = self.db.loadView('WorkQueue', 'activeParentData', {'reduce': True, 'group': True})
        return [{'dbs_url': x['key'][0],
                 'name': x['key'][1]} for x in data.get('rows', [])]

    def getActivePileupData(self):
        """Get data items we have work in the queue for with pileup"""
        data = self.db.loadView('WorkQueue', 'activePileupData', {'reduce': True, 'group': True})
        return [{'dbs_url': x['key'][0],
                 'name': x['key'][1]} for x in data.get('rows', [])]

    def getElementsForData(self, data):
        """Get active elements for this dbs & data combo"""
        elements = self.db.loadView('WorkQueue', 'elementsByData', {'key': data, 'include_docs': True})
        return [CouchWorkQueueElement.fromDocument(self.db,
                                                   x['doc'])
                for x in elements.get('rows', [])]

    def getElementsForParentData(self, data):
        """Get active elements for this data """
        elements = self.db.loadView('WorkQueue', 'elementsByParentData', {'key': data, 'include_docs': True})
        return [CouchWorkQueueElement.fromDocument(self.db,
                                                   x['doc'])
                for x in elements.get('rows', [])]

    def getElementsForPileupData(self, data):
        """Get active elements for this data """
        elements = self.db.loadView('WorkQueue', 'elementsByPileupData', {'key': data, 'include_docs': True})
        return [CouchWorkQueueElement.fromDocument(self.db,
                                                   x['doc'])
                for x in elements.get('rows', [])]

    def isAvailable(self):
        """Is the server available, i.e. up and not compacting"""
        try:
            compacting = self.db.info()['compact_running']
            if compacting:
                self.logger.info("CouchDB compacting - try again later.")
                return False
        except Exception as ex:
            self.logger.error("CouchDB unavailable: %s" % str(ex))
            return False
        return True

    def getWorkflows(self, includeInbox=False, includeSpecs=False):
        """Returns workflows known to workqueue"""
        result = set([x['key'] for x in self.db.loadView('WorkQueue', 'elementsByWorkflow', {'group': True})['rows']])
        if includeInbox:
            result = result | set(
                    [x['key'] for x in self.inbox.loadView('WorkQueue', 'elementsByWorkflow', {'group': True})['rows']])
        if includeSpecs:
            result = result | set([x['key'] for x in self.db.loadView('WorkQueue', 'specsByWorkflow')['rows']])
        return list(result)

    def queueLength(self):
        """Return number of available elements"""
        return self.db.loadView('WorkQueue', 'availableByPriority', {'limit': 0})['total_rows']

    def fixConflicts(self):
        """Fix elements in conflict

        Each local queue runs this to resolve its conflicts with global,
        resolution propagates up to global.

        Conflicting elements are merged into one element with others deleted.

        This will fail if elements are modified during the resolution -
        if this happens rerun.
        """
        for db in [self.inbox, self.db]:
            for row in db.loadView('WorkQueue', 'conflicts')['rows']:
                elementId = row['id']
                try:
                    conflicting_elements = [CouchWorkQueueElement.fromDocument(db, db.document(elementId, rev)) \
                                            for rev in row['value']]
                    fixed_elements = fixElementConflicts(*conflicting_elements)
                    if self.saveElements(fixed_elements[0]):
                        self.saveElements(*fixed_elements[1:])  # delete others (if merged value update accepted)
                except Exception as ex:
                    self.logger.error("Error resolving conflict for %s: %s" % (elementId, str(ex)))

    def recordTaskActivity(self, taskname, comment=''):
        """Record a task for monitoring"""
        try:
            record = self.db.document('task_activity')
        except CouchNotFoundError:
            record = Document('task_activity')
        record.setdefault('tasks', {})
        record['tasks'].setdefault(taskname, {})
        record['tasks'][taskname]['timestamp'] = time.time()
        record['tasks'][taskname]['comment'] = comment
        try:
            self.db.commitOne(record)
        except Exception as ex:
            self.logger.error("Unable to update task %s freshness: %s" % (taskname, str(ex)))

    def getWMBSInjectStatus(self, request=None):
        """
        This service only provided by global queue except on draining agent
        """
        options = {'group': True, 'reduce': True}
        if request:
            options.update(key=request)
        data = self.db.loadView('WorkQueue', 'wmbsInjectStatusByRequest', options)
        if request:
            if data['rows']:
                injectionStatus = data['rows'][0]['value']
                inboxElement = self.getInboxElements(WorkflowName=request)
                requestOpen = inboxElement[0].get('OpenForNewData', False) if inboxElement else False
                return injectionStatus and not requestOpen
            else:
                raise WorkQueueNoMatchingElements("%s not found" % request)
        else:
            injectionStatus = dict((x['key'], x['value']) for x in data.get('rows', []))
            finalInjectionStatus = []
            for request in injectionStatus:
                inboxElement = self.getInboxElements(WorkflowName=request)
                requestOpen = inboxElement[0].get('OpenForNewData', False) if inboxElement else False
                finalInjectionStatus.append({request: injectionStatus[request] and not requestOpen})

            return finalInjectionStatus

    def getWorkflowNames(self, inboxFlag=False):
        """Get workflow names from workqueue db"""
        if inboxFlag:
            db = self.inbox
        else:
            db = self.db
        data = db.loadView('WorkQueue', 'elementsByWorkflow',
                           {'stale': "update_after", 'reduce': True, 'group': True})
        return [x['key'] for x in data.get('rows', [])]

    def deleteWQElementsByWorkflow(self, workflowNames):
        """
        delete workqueue elements belongs to given workflow names
        it doen't check the status of workflow so need to be careful to use this.
        Pass only workflows which has the end status
        """
        deleted = 0
        dbs = [self.db, self.inbox]
        if not isinstance(workflowNames, list):
            workflowNames = [workflowNames]

        if len(workflowNames) == 0:
            return deleted
        options = {}
        options["stale"] = "update_after"
        options["reduce"] = False

        idsByWflow = {}
        for couchdb in dbs:
            result = couchdb.loadView("WorkQueue", "elementsByWorkflow", options, workflowNames)
            for entry in result["rows"]:
                idsByWflow.setdefault(entry['key'], [])
                idsByWflow[entry['key']].append(entry['id'])
            for wflow, docIds in viewitems(idsByWflow):
                self.logger.info("Going to delete %d documents in *%s* db for workflow: %s. Doc IDs: %s",
                                 len(docIds), couchdb.name, wflow, docIds)
                try:
                    couchdb.bulkDeleteByIDs(docIds)
                except CouchNotFoundError as exc:
                    self.logger.error("Failed to find one of the documents. Error: %s", str(exc))
                deleted += len(docIds)
        # delete the workflow with spec from workqueue db
        for wf in workflowNames:
            self.db.delete_doc(wf)
        return deleted
