###############################################################################
# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
#                                                                             #
# This software is distributed under the terms of the GNU General Public      #
# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".   #
#                                                                             #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization  #
# or submit itself to any jurisdiction.                                       #
###############################################################################
"""The ProductionStatusAgent monitors productions for active requests and takes
care to update their status. Initially this is just to handle simulation
requests.

Allowed production status transitions performed by this agent include:

Idle -> ValidatingInput
Idle -> ValidatingOutput

ValidatedOutput -> Completed

ValidatingInput -> RemovingFiles

RemovedFiles -> Completed

Active -> Idle

Testing -> Idle

In addition this also updates request status from Active to Done.

To do: review usage of production API(s) and re-factor into Production Client

AZ 10.14: merged with a part from RequestTrackingAgent to avoid race conditions
"""
import time
import os
import sqlite3
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, wait

import DIRAC
from DIRAC import S_OK, S_ERROR, gLogger
from DIRAC.Core.Base.AgentModule import AgentModule
from DIRAC.Core.Utilities.TimeUtilities import timeThis
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations

from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from LHCbDIRAC.Interfaces.API.DiracProduction import DiracProduction
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
from LHCbDIRAC.ProductionManagementSystem.Client.ProductionRequestClient import ProductionRequestClient

#############################################################################
# The following is used for StandAlone debugging only (outside Agent)
gStandAlone = False  # work in command line without Agent
# gSimulate = gStandAlone and True  # real clients are replaced with simulation
gSimulate = False
gDoRealUpdate = True  # call status updates
gDoRealTracking = True  # update requests progress

KNOWN_TASK_STATES = [
    "Checking",
    "Completed",
    "Created",
    "Matched",
    "Received",
    "Reserved",
    "Rescheduled",
    "Running",
    "Submitted",
    "Waiting",
]


class ProductionStatusAgent(AgentModule):
    """Usual DIRAC agent."""

    def __init__(self, *args, **kwargs):
        """c'tor.

        :param self: self reference
        :param str agentName: name of agent
        :param str loadName: load name of agent
        :param bool baseAgentName: whatever
        :param dict properties: whatever else
        """
        if not gStandAlone:
            AgentModule.__init__(self, *args, **kwargs)
        else:
            self.log = gLogger

        self.dProd = None
        self.dirac = None
        self.prClient = None
        self.tClient = None

        self.simulationTypes = Operations().getValue(
            "Transformations/ExtendableTransfTypes",
            ["MCSimulation", "Simulation"],
        )

        self.allKnownStates = (
            "RemovedFiles",
            "RemovingFiles",
            "ValidatedOutput",
            "ValidatingInput",
            "Testing",
            "Active",
            "Idle",
        )

        self.notify = True
        self.cacheFile = os.path.join(DIRAC.rootPath, "work/ProductionManagement/cache.db")

        # For processing transformations, it can happen that there are some Unused files
        # with which no tasks can be created. The number of such files can be different depending
        # from the module and distribution between centres.
        # So we declare such transformations isIdle() once there is no jobs, no files in other
        # pending states and the number of Unused files was not changed last cyclesTillIdle times
        self.cyclesTillIdle = 1
        self.filesUnused = {}  # <tID: { 'Number': x, 'NotChanged': n }

        self.prMasters = {}  # [ prID: [<subrequests> ...] ]
        self.prSummary = {}
        self.prProds = {}  # <prID>, map production to known request, from _getProductionRequestsProgress
        self.notPrTrans = defaultdict(list)  # transformation without PR, from _getTransformationsState
        self.toUpdate = []

    #############################################################################
    def initialize(self):
        """Sets default values."""
        # shifter
        self.am_setOption("shifterProxy", "ProductionManager")

        if not gStandAlone:
            self.notify = eval(self.am_getOption("NotifyProdManager", "True"))

        # Set the clients
        self.dProd = DiracProduction()
        self.dirac = Dirac()
        if gSimulate:
            raise NotImplementedError()
        self.prClient = ProductionRequestClient()
        self.tClient = TransformationClient()

        return S_OK()

    #############################################################################
    def execute(self):
        """The execution method, track requests progress and implement a part of
        Production SM."""
        updatedT = {}  # updated transformations
        updatedPr = []  # updated production requests (excluding tracking updates)

        # Distinguish between leafs and master requests
        # Masters should not appear in the prodReqSummary and they should have no
        # associated productions.
        self.prMasters = {}  # [ prID: [<subrequests> ...] ]
        self.prSummary = {}
        # { <reqID> :
        #     'type', 'master', 'bkTotal', 'prTotal',  - from _getActiveProductionRequests()
        #     'isDone', 'prods': [ <prodIf> : { 'Used', 'Events' } ] - from __getProductionRequestsProgress
        #     'state' for each production - from _getTransformationsState()
        #     'isIdle', 'isProcIdle' for each 'Active' or 'Idle' production,
        #     'isSimulation' - from _getIdleProductionRequestProductions()
        #     'isFinished' - from _applyProductionRequestsLogic()
        #     'filesTotal' - from _getIdleProductionRequestProductions()
        #     'filesProcessed' - from _getIdleProductionRequestProductions()
        #     'filesUnused' - from _getIdleProductionRequestProductions()
        #     'filesMaxReset' - from _getIdleProductionRequestProductions()
        #     'filesNotProcessed' - from _getIdleProductionRequestProductions()
        #     'inputIDs' - from _getExtraInfo()
        #     'hasEndDate' - from _getExtraInfo()
        #     'hasActiveInput' - from _trackProductionRequests()
        # }
        self.prProds = {}  # <prID>, map production to known request, from _getProductionRequestsProgress

        self.notPrTrans = defaultdict(list)  # transformation without PR, from _getTransformationsState

        self.log.info("******************************")
        self.log.info("Collecting required information")
        self.log.info("******************************")

        result = self._getActiveProductionRequests()
        if not result["OK"]:
            self.log.error("Aborting cycle", result["Message"])
            return S_OK()
        if len(self.prProds) < len(self.prSummary):
            self.log.fatal(
                "Aborting cycle, as the number of sub-requests can't be larger than the number of productions"
            )
            return S_OK()

        self._getTransformationsState()
        result = self._getIdleProductionRequestProductions()
        if not result["OK"]:
            self.log.error("Aborting cycle", result["Message"])
            return S_OK()

        # That is IMPORTANT to do that after we have the transformation status,
        # since Validation can (really???) update BK, rendering MC incomplete
        result = self._trackProductionRequests()  # also updates PR DB
        if not result["OK"]:
            self.log.error("Aborting cycle", result["Message"])
            return S_OK()

        self.log.info("******************************")
        self.log.info("Updating Production Requests and related transformations")
        self.log.info("******************************")

        self._applyProductionRequestsLogic(updatedT, updatedPr)

        self.log.info("******************************")
        self.log.info("Updating Production Request for unrelated transformations (replication, etc.)")
        self.log.info("******************************")

        self._applyOtherTransformationsLogic(updatedT)

        self.log.info("*********")
        self.log.info("Reporting")
        self.log.info("*********")

        if updatedT:
            self.log.info("Transformations updated this cycle:")
            for name, value in updatedT.items():
                self.log.info(f"Transformations {name}: {value['from']} => {value['to']}")

        if updatedPr:
            self.log.info("Production Requests updated to Done status:", f"{', '.join(str(i) for i in updatedPr)}")

        if gDoRealUpdate and not gSimulate:
            self._mailProdManager(updatedT, updatedPr)

        self._cleanFilesUnused()

        if gSimulate:
            self.tClient.animate(self.prClient)

        return S_OK()

    #############################################################################

    @timeThis
    def __getProductionRequestsProgress(self):
        """get known progress for Active requests related productions Failures
        there are critical and can enforce wrong logic."""
        self.log.verbose("Collecting old Production Request Progress...")
        result = self.prClient.getAllProductionProgress()
        if not result["OK"]:
            return S_ERROR(f"Could not retrieve production progress summary: {result['Message']}")
        progressSummary = result["Value"]  # { <prID> : [ <prodId> : { 'Used', 'Events' } ] }

        for prID, summary in self.prSummary.items():
            # Setting it before updating will give grace period before SM ops
            summary["isDone"] = summary["bkTotal"] >= summary["prTotal"]
            summary["prods"] = progressSummary.get(prID, {})
            for tID in summary["prods"]:
                self.prProds[tID] = prID
        self.log.verbose("Done with old Production Request Progress")
        return S_OK()

    @timeThis
    def _getActiveProductionRequests(self):
        """get 'Active' requests.

        Failures there are critical and can enforce wrong logic
        Note: this method can be moved to the service
        """
        self.log.info("Collecting active production requests...")
        result = self.prClient.getProductionRequestList(0, "", "ASC", 0, 0, {"RequestState": "Active"})
        if not result["OK"]:
            return S_ERROR(f"Could not retrieve active production requests: {result['Message']}")
        activeMasters = result["Value"]["Rows"]
        for pr in activeMasters:
            prID = pr["RequestID"]
            if pr["HasSubrequest"]:
                self.prMasters[prID] = []
                result = self.prClient.getProductionRequestList(prID, "", "ASC", 0, 0, {})
                if not result["OK"]:
                    return S_ERROR(f"Could not get subrequests for production request {prID}: {result['Message']}")
                for subPr in result["Value"]["Rows"]:
                    subPrID = subPr["RequestID"]
                    self.prSummary[subPrID] = {
                        "type": pr["RequestType"],
                        "master": prID,
                        "bkTotal": subPr["bkTotal"],
                        "prTotal": subPr["rqTotal"],
                    }
                    self.prMasters[prID].append(subPrID)
            else:
                self.prSummary[prID] = {
                    "type": pr["RequestType"],
                    "master": 0,
                    "bkTotal": pr["bkTotal"],
                    "prTotal": pr["rqTotal"],
                }

        result = self.__getProductionRequestsProgress()
        if not result["OK"]:
            return result

        self.log.info(f"Will work with {len(self.prProds)} productions from {len(self.prSummary)} Active (sub)requests")
        self.log.verbose("Done with collecting Active production requests")
        return S_OK()

    @timeThis
    def __getTransformations(self, status):
        """dev function.

        Get the transformations (print info in the meanwhile)
        """
        res = self.tClient.getTransformationWithStatus(status)
        if not res["OK"]:
            self.log.error("Failed to get transformations", f"{status}: {res['Message']}")
            raise RuntimeError(f"Failed to get {status} transformations: {res['Message']}")
        if not res["Value"]:
            self.log.debug(f"No transformations in {status} status")
            return []
        if len(res["Value"]) > 20:
            self.log.verbose(f"The following number of transformations are in {status} status: {len(res['Value'])}")
        else:
            valOutStr = ", ".join(str(i) for i in res["Value"])
            self.log.verbose(f"The following transformations are in {status} status: {valOutStr}")
        return res["Value"]

    @timeThis
    def _getTransformationsState(self):
        """get Transformations state (set 'Other' for not interesting states)
        failures to get something are not critical since there is no reaction on
        'Other' state."""
        self.log.info("Collecting transformations state...")
        try:
            # We put 'Finished' for both
            tListCompleted = self.__getTransformations("Completed")
            tListArchived = self.__getTransformations("Archived")
            tListFinished = tListCompleted + tListArchived
            for tID in tListFinished:
                prID = self.prProds.get(tID, None)
                if prID:
                    self.prSummary[prID]["prods"][tID]["state"] = "Finished"

            for state in self.allKnownStates:
                tList = self.__getTransformations(state)
                for tID in tList:
                    prID = self.prProds.get(tID, None)
                    if prID:
                        self.prSummary[prID]["prods"][tID]["state"] = state
                    else:
                        self.notPrTrans[state].append(tID)
        except RuntimeError as error:
            self.log.error(error)

        for tID, prID in self.prProds.items():
            if "state" not in self.prSummary[prID]["prods"][tID]:
                self.prSummary[prID]["prods"][tID]["state"] = "Other"

        self.log.verbose("Done with collecting transformations states")

    def _getStatusCountersBulk(self, tableName, statusKey, transformationIDs):
        """Get the counter named ``statusKey`` for many transformations

        Wrapper around ``TransformationClient.getCounters`` that returns a nested
        dictionary of:
        ``{transformationID: {state1: X, state2: Y, ..., "TotalCreated": X+Y+...}}``

        :param str tableName: Table in the TransformationDB to use
        :param str statusKey: Name of the status column to consider
        :param list transformationIDs: List of transformation IDs to inspect
        :returns: ``dict`` of ``dict`` of ``int``
        """
        # This is for compatibility with TransformationClient.getTransformationStats
        # and TransformationClient.getTransformationTaskStats
        if tableName == "TransformationTasks":
            totalColumnName = "TotalCreated"
        elif tableName == "TransformationFiles":
            totalColumnName = "Total"
        else:
            raise NotImplementedError(tableName)
        self.log.verbose("Getting counters from for", f"{len(transformationIDs)} transformations")
        res = self.tClient.getCounters(
            tableName,
            ["TransformationID", statusKey],
            {"TransformationID": transformationIDs},
        )
        if not res["OK"]:
            raise RuntimeError(res)
        # Ensure totalColumnName is always included in the output
        statusDict = defaultdict(lambda: defaultdict(int, **{totalColumnName: 0}))
        for attrDict, count in res["Value"]:
            tID = attrDict["TransformationID"]
            status = attrDict[statusKey]
            statusDict[tID][status] = count
            statusDict[tID][totalColumnName] += count
        # Loop over transformationIDs to ensure all IDs are included in the output
        return {int(tID): dict(statusDict[int(tID)]) for tID in transformationIDs}

    def _isIdleCache(self, transIDs):
        """Get the cache dictionaries that need to be passed to __isIdle

        This method is used with __isIdle to minimise the number of RPC calls required::

          transformations, taskStatuses, fileStatuses = self._isIdleCache(tIDs)
          for tID in tIDs:
            isIdle, isProcIdle, isSimulation = self.__isIdle(
                tID, transformations[tID], taskStatuses[tID], fileStatuses[tID]
            )

        :param list of str tIDs: The IDs to get cached data for
        :returns: transformations, taskStatuses, fileStatuses
        """
        self.log.verbose("Filling _isIdleCache caches for", f"{len(transIDs)} transformations")
        result = self.tClient.getTransformations(condDict={"TransformationID": transIDs}, limit=1000)
        if not result["OK"]:
            self.log.error("Could not get transformations", result["Message"])
            raise RuntimeError("Could not get transformations")
        transformations = {x["TransformationID"]: x for x in result["Value"]}
        taskStatuses = self._getStatusCountersBulk("TransformationTasks", "ExternalStatus", transIDs)
        fileStatuses = self._getStatusCountersBulk("TransformationFiles", "Status", transIDs)
        return transformations, taskStatuses, fileStatuses

    def __isIdle(self, tID, tInfo, tStats, filesStats):
        """Checks if a transformation is idle, is procIdle and either the
        transformation is simulation.

        The ``tInfo``, ``tStats``, ``filesStats`` parameters are passed to improve
        performance by avoiding 3 round trips to the transformation service. See
        ``_isIdleCache`` for more details.

        :param str tID: ID of the transformation to check
        :param dict tInfo: Result of ``TransformationClient.getTransformations``
        :param dict tStats: Result of ``self._getStatusCountersBulk`` for the
                            ``ExternalStatus`` column in the ``TransformationTasks`` table
        :param dict filesStats: Result of ``self._getStatusCountersBulk`` for the
                                ``Status`` column in the ``TransformationFiles`` table
        :returns: ``dict`` of ``dict`` of ``int``
        """
        self.log.debug("Checking if transformation is idle", str(tID))
        if tInfo.get("Type", None) in self.simulationTypes:
            isSimulation = True
            # simulation : go to Idle if
            # only failed and done tasks
            # AND number of tasks created in total == number of tasks submitted
            self.log.verbose("Tasks Stats for %d: %s" % (tID, str(tStats)))
            isIdle = (tStats.get("TotalCreated", 0) > 0) and all(
                tStats.get(status, 0) == 0 for status in KNOWN_TASK_STATES
            )
            isProcIdle = isIdle
        else:
            isSimulation = False
            # other transformation type : go to Idle if
            # 0 assigned files, unused files number was not changing during the last cyclesTillIdle time
            # AND only failed and done tasks
            self.log.debug(f"Files stats: {str(filesStats)}")
            unused = filesStats.get("Unused", 0)
            unusedInherited = filesStats.get("Unused-inherited", 0)
            oldUnused = self.filesUnused.setdefault(tID, {"Number": -1, "NotChanged": 0})
            if oldUnused["Number"] == unused:
                oldUnused["NotChanged"] += 1
            else:
                oldUnused["NotChanged"] = 0
                oldUnused["Number"] = unused
            assigned = filesStats.get("Assigned", 0)
            isProcIdle = (assigned == 0) and ((unused == 0) or (oldUnused["NotChanged"] >= self.cyclesTillIdle))
            if isProcIdle:
                self.log.debug(f"Tasks Stats: {str(tStats)}")
                isProcIdle = all(tStats.get(status, 0) == 0 for status in KNOWN_TASK_STATES)
            isIdle = isProcIdle and (unused == 0) and (unusedInherited == 0)
        return (isIdle, isProcIdle, isSimulation)

    def _getIdleProductionRequestProductions(self):
        """evaluate isIdle and isProcIdle status for all productions we need.

        failures are remembered and are taken into account later
        """
        self.log.verbose("Filling caches for _getIdleProductionRequestProductions...")
        transIDs = [
            str(tID)
            for tID, prID in self.prProds.items()
            if self.prSummary[prID]["prods"][tID]["state"] in ("Active", "Idle")
        ]
        try:
            transformations, taskStatuses, fileStatuses = self._isIdleCache(transIDs)
        except RuntimeError:
            return S_ERROR("Failed to get _isIdleCache in _getIdleProductionRequestProductions")

        self.log.verbose("Checking idle productions...")
        for tID, prID in self.prProds.items():
            tInfo = self.prSummary[prID]["prods"][tID]
            if tInfo["state"] in ("Active", "Idle"):
                isIdle, isProcIdle, isSimulation = self.__isIdle(
                    tID, transformations[tID], taskStatuses[tID], fileStatuses[tID]
                )
                tInfo["isIdle"] = "Yes" if isIdle else "No"
                tInfo["isProcIdle"] = "Yes" if isProcIdle else "No"
                tInfo["isSimulation"] = isSimulation
                tInfo["filesTotal"] = fileStatuses[tID]["Total"]
                tInfo["filesProcessed"] = fileStatuses[tID].get("Processed", 0)
                tInfo["filesUnused"] = fileStatuses[tID].get("Unused", 0)
                tInfo["filesMaxReset"] = fileStatuses[tID].get("MaxReset", 0)
                tInfo["filesNotProcessed"] = fileStatuses[tID].get("NotProcessed", 0) + fileStatuses[tID].get(
                    "Removed", 0
                )
            else:
                tInfo["isIdle"] = "Unknown"
                tInfo["isProcIdle"] = "Unknown"
                tInfo["isSimulation"] = False
                tInfo["filesTotal"] = None
                tInfo["filesProcessed"] = 0
                tInfo["filesUnused"] = 0
                tInfo["filesMaxReset"] = 0
                tInfo["filesNotProcessed"] = 0
        self.log.verbose("Checking idle done")
        return S_OK()

    def _trackProductionRequests(self):
        """contact BK for the current number of processed events failures are
        critical."""
        self.log.info("Updating production requests progress...")

        # Using 10 threads, and waiting for the results before continuing
        futureThreads = []
        with ThreadPoolExecutor(10) as threadPool:
            for tID, prID in self.prProds.items():
                futureThreads.append(threadPool.submit(self._getExtraInfo, tID, prID))
            wait(futureThreads)

        # Update the Production request DB with the number of bookkeeping events
        if self.toUpdate:
            if gDoRealTracking:
                result = self.prClient.updateTrackedProductions(self.toUpdate)
            else:
                result = S_OK()
            if not result["OK"]:
                self.log.error(
                    "Could not send update to the Production Request System", result["Message"]
                )  # that is not critical
            else:
                self.log.verbose(f"The progress of {len(self.toUpdate)} Production Requests is updated")
        self.log.info("Production requests progress update is finished")

        # Get the status of the Analysis Productions input transformations
        inputTransformIDs = set()
        for summary in self.prSummary.values():
            for tID, tInfo in summary["prods"].items():
                inputTransformIDs = inputTransformIDs.union(
                    x for x in tInfo.get("inputIDs", []) if x not in summary["prods"]
                )
        inputTransformStatuses = {}
        if inputTransformIDs:
            retVal = self.tClient.getTransformations(
                condDict={"TransformationID": list(inputTransformIDs)},
                limit=10000,
                columns=["TransformationID", "Status"],
            )
            if not retVal["OK"]:
                self.log.error("Failed to call getTransformations", retVal["Message"])
                return S_ERROR("Too dangerous to continue")
            inputTransformStatuses = {d["TransformationID"]: d["Status"] for d in retVal["Value"]}
        for prID, summary in self.prSummary.items():
            for tID, tInfo in summary["prods"].items():
                if "inputIDs" not in tInfo:
                    continue
                if "hasActiveInput" in tInfo:
                    self.log.info(
                        "hasActiveInput has been overridden for",
                        f"{tID} ({prID}) to be {tInfo['hasActiveInput']}",
                    )
                    continue
                tInfo["hasActiveInput"] = False
                if tInfo["hasEndDate"]:
                    self.log.info(
                        "Skipping hasActiveInput check for",
                        f"{tID} ({prID}) as the input query has an end date",
                    )
                    continue
                for inputID in tInfo["inputIDs"]:
                    if inputID in summary["prods"]:
                        inputState = summary["prods"][inputID]["state"]
                    else:
                        inputState = inputTransformStatuses.get(inputID, "Unknown")
                    if inputState not in ["Archived", "Completed", "Finished", "Cleaned"]:
                        self.log.info(
                            "Marking hasActiveInput=True for",
                            f"{tID} ({prID}) as input {inputID} has status {inputState}",
                        )
                        tInfo["hasActiveInput"] = True

        return S_OK()

    def _getExtraInfo(self, tID, prID):
        tInfo = self.prSummary[prID]["prods"][tID]
        # Get the number of bookkeeping events
        self.log.verbose("Getting BK production progress", "Transformation ID = %d" % tID)
        result = BookkeepingClient().getProductionProducedEvents(tID)
        if result["OK"]:
            nEvents = result["Value"]
            if nEvents and nEvents != tInfo["Events"]:
                self.log.verbose("Updating production %d, with BkEvents %d" % (int(tID), int(nEvents)))
                self.toUpdate.append({"ProductionID": tID, "BkEvents": nEvents})
                tInfo["Events"] = nEvents
        else:
            self.log.error("Progress is not updated", f"{tID} : {result['Message']}")
            return S_ERROR("Too dangerous to continue")
        # Find the transformation ID for the input to each Analysis Production
        if self.prSummary[prID]["type"] == "AnalysisProduction":
            retVal = TransformationClient().getBookkeepingQuery(tID)
            if not retVal["OK"]:
                self.log.error("Failed to call getBookkeepingQuery", f"{tID} {retVal['Message']}")
                return S_ERROR("Too dangerous to continue")
            inputBkQuery = retVal["Value"]
            tInfo["hasEndDate"] = "EndDate" in inputBkQuery
            if not tInfo["hasEndDate"] and inputBkQuery.get("ConfigVersion") in Operations().getValue(
                "AnalysisProductions/ForceActiveInput", []
            ):
                tInfo["hasActiveInput"] = True
            if "ProductionID" in inputBkQuery:
                tInfo["inputIDs"] = [inputBkQuery["ProductionID"]]
            else:
                # FIXME: Just why?!
                if "DataTakingConditions" in inputBkQuery:
                    inputBkQuery["ConditionDescription"] = inputBkQuery["DataTakingConditions"]
                retVal = BookkeepingClient().getProductions(inputBkQuery)
                if not retVal["OK"]:
                    self.log.error("Failed to call getProductions", f"{tID} {retVal['Message']}")
                    return S_ERROR("Too dangerous to continue")
                tInfo["inputIDs"] = [x[0] for x in retVal["Value"]["Records"]]

    def _cleanFilesUnused(self):
        """remove old transformations from filesUnused."""
        oldIDs = []
        for tID in self.filesUnused:
            if tID in self.prProds:
                continue
            if all(tID not in IDs for _status, IDs in self.notPrTrans.items()):
                oldIDs.append(tID)
        for tID in oldIDs:
            del self.filesUnused[tID]

    def __updateTransformationStatus(self, tID, origStatus, status, updatedT):
        """This method updates the transformation status and logs the changes for
        each iteration of the agent.

        Most importantly this method only allows status transitions based on
        what the original status should be.
        """
        self.log.info(f"Changing status for transformation {tID} to {status}")

        if not gDoRealUpdate:
            updatedT[tID] = {"to": status, "from": origStatus}
            return

        result = self.tClient.setTransformationParameter(tID, "Status", status, currentStatus=origStatus)
        if not result["OK"]:
            self.log.error("Failed to update status of transformation", f"{tID} from {origStatus} to {status}")
        else:
            updatedT[tID] = {"to": status, "from": origStatus}

    def _mailProdManager(self, updatedT, updatedPr):
        """Notify the production manager of the changes as productions should be
        manually extended in some cases."""
        if not updatedT and not updatedPr:
            self.log.verbose("No changes this cycle, mail will not be sent")
            return

        if self.notify:
            with sqlite3.connect(self.cacheFile) as conn:
                try:
                    conn.execute(
                        """CREATE TABLE IF NOT EXISTS ProductionStatusAgentCache(
                        production VARCHAR(64) NOT NULL DEFAULT "",
                        from_status VARCHAR(64) NOT NULL DEFAULT "",
                        to_status VARCHAR(64) NOT NULL DEFAULT "",
                        time VARCHAR(64) NOT NULL DEFAULT ""
                       );"""
                    )

                    conn.execute(
                        """CREATE TABLE IF NOT EXISTS ProductionStatusAgentReqCache(
                        prod_requests VARCHAR(64) NOT NULL DEFAULT "",
                        time VARCHAR(64) NOT NULL DEFAULT ""
                       );"""
                    )

                except sqlite3.OperationalError:
                    self.log.error("Could not queue mail")

                for tID, val in updatedT.items():
                    conn.execute(
                        "INSERT INTO ProductionStatusAgentCache (production, from_status, to_status, time)"
                        " VALUES (?, ?, ?, ?)",
                        (tID, val["from"], val["to"], time.asctime()),
                    )

                for prod_request in updatedPr:
                    conn.execute(
                        "INSERT INTO ProductionStatusAgentReqCache (prod_requests, time) VALUES (?, ?)",
                        (prod_request, time.asctime()),
                    )

                conn.commit()

                self.log.info("Mail summary queued for sending")

    def __updateProductionRequestStatus(self, prID, status, updatedPr):
        """This method updates the production request status."""
        self.log.info(f"Marking Production Request {prID} as {status}")

        if not gDoRealUpdate:
            updatedPr.append(prID)
            return

        reqClient = ProductionRequestClient(useCertificates=False, timeout=120)
        result = reqClient.updateProductionRequest(int(prID), {"RequestState": status})
        if not result["OK"]:
            self.log.error(result)
        else:
            updatedPr.append(prID)

    def _applyOtherTransformationsLogic(self, updatedT):
        """animate not Production Requests related transformations failures are not
        critical."""
        self.log.verbose("Updating requests unrelated transformations...")

        self.log.info(
            'Processing %s requests unrelated transformations in "RemovedFiles" state'
            % len(self.notPrTrans["RemovedFiles"])
        )
        for tID in self.notPrTrans["RemovedFiles"]:
            self.__updateTransformationStatus(tID, "RemovedFiles", "Completed", updatedT)

        self.log.info(
            f"Processing {len(self.notPrTrans['Active'])} requests unrelated transformations in \"Active\" state"
        )
        try:
            transformations, taskStatuses, fileStatuses = self._isIdleCache(self.notPrTrans["Active"])
        except RuntimeError:
            self.log.error('Failed to get _isIdleCache for monitoring "Active" requests')
        else:
            for tID in self.notPrTrans["Active"]:
                isIdle, _isProcIdle, _isSimulation = self.__isIdle(
                    tID, transformations[tID], taskStatuses[tID], fileStatuses[tID]
                )
                if isIdle:
                    self.__updateTransformationStatus(tID, "Active", "Idle", updatedT)

        self.log.info(f"Processing {len(self.notPrTrans['Idle'])} requests unrelated transformations in \"Idle\" state")
        try:
            transformations, taskStatuses, fileStatuses = self._isIdleCache(self.notPrTrans["Idle"])
        except RuntimeError:
            self.log.error('Failed to get _isIdleCache for monitoring "Idle" requests')
        else:
            for tID in self.notPrTrans["Idle"]:
                isIdle, _isProcIdle, _isSimulation = self.__isIdle(
                    tID, transformations[tID], taskStatuses[tID], fileStatuses[tID]
                )
                if not isIdle:
                    self.__updateTransformationStatus(tID, "Idle", "Active", updatedT)

        self.log.verbose("Requests unrelated transformations update is finished")

    def _isReallyDone(self, summary):
        """Evaluate 'isDone' from current update cycle."""
        # Other production types rely on the number of bookkeeping events
        bkTotal = sum(t["Events"] for t in summary["prods"].values() if t["Used"])
        return bkTotal >= summary["prTotal"]

    def _producersAreIdle(self, summary):
        """Return True in case all producers (not 'Used') transformations are Idle,
        Finished or not exist."""
        for _tID, tInfo in summary["prods"].items():
            if tInfo["Used"]:
                continue
            if tInfo["isIdle"] != "Yes" and tInfo["state"] != "Finished":
                return False
        return True

    def _producersAreProcIdle(self, summary):
        """Return True in case all producers (not 'Used') transformations are
        procIdle or finished or not exist."""
        for _tID, tInfo in summary["prods"].items():
            if tInfo["Used"]:
                continue
            if tInfo["isProcIdle"] != "Yes" and tInfo["state"] != "Finished":
                return False
        return True

    def _processorsAreProcIdle(self, summary):
        """Return True in case all processors ('Used' or not Sim) transformations
        are procIdle or finished or not exist."""
        for _tID, tInfo in summary["prods"].items():
            if not tInfo["Used"] and tInfo["isSimulation"]:
                continue
            if tInfo["isProcIdle"] != "Yes" and tInfo["state"] != "Finished":
                return False
        return True

    def _mergersAreDone(self, summary):
        """Return True in case all mergers ('Used') transformations are finished or
        not exist."""
        for _tID, tInfo in summary["prods"].items():
            if not tInfo["Used"]:
                continue
            if tInfo["state"] != "Finished":
                return False
        return True

    def _mergersAreProcIdle(self, summary):
        """Return True in case all mergers ('Used') transformations are procIdle or
        finished or not exist."""
        for _tID, tInfo in summary["prods"].items():
            if not tInfo["Used"]:
                continue
            if tInfo["isProcIdle"] != "Yes" and tInfo["state"] != "Finished":
                return False
        return True

    def _requestedMoreThenProduced(self, tID, summary):
        """Check that this transformation has registered less events than it was
        requested."""
        if summary["prods"][tID]["Events"] < summary["prTotal"]:
            self.log.verbose(f" Transformation {tID} has produced less events, asking for extention ")
            return True
        return False

    def _applyProductionRequestsLogic(self, updatedT, updatedPr):
        """Apply known logic to transformations related to production requests
        NOTE: we make decision based on BK statistic collected in the PREVIOUS cycle (except isReallyDone)
              and transformation status in THIS cycle BEFORE the Logic is started
        """
        self.log.verbose("Production Requests logic...")

        for prID, summary in self.prSummary.items():
            countFinished = 0
            for tID, tInfo in summary["prods"].items():
                if tInfo["state"] == "Finished":
                    countFinished += 1
                elif tInfo["state"] == "Idle":
                    self._handleStateIdle(tID, tInfo, summary, updatedT)
                elif tInfo["state"] == "RemovedFiles":
                    self._handleStateRemovedFiles(tID, tInfo, summary, updatedT)
                elif tInfo["state"] == "Active":
                    self._handleStateActive(tID, tInfo, summary, updatedT)
                elif tInfo["state"] == "ValidatedOutput":
                    self._handleStateValidatedOutput(tID, tInfo, summary, updatedT)
                elif tInfo["state"] == "ValidatingInput":
                    self._handleStateValidatingInput(tID, tInfo, summary, updatedT)
                elif tInfo["state"] == "Testing":
                    self._handleStateTesting(tID, tInfo, summary, updatedT)

            summary["isFinished"] = countFinished == len(summary["prods"])
            if (
                summary["isFinished"]
                and not summary["master"]
                and summary["type"] in ["Simulation", "AnalysisProduction"]
            ):
                self.__updateProductionRequestStatus(prID, "Done", updatedPr)

        for masterID, prList in self.prMasters.items():
            countFinished = sum(1 for prID in prList if self.prSummary[prID]["isFinished"])
            if countFinished == len(prList):
                self.__updateProductionRequestStatus(masterID, "Done", updatedPr)

        self.log.verbose("Done with Production Requests logic")

    def _handleStateIdle(self, tID, tInfo, summary, updatedT):
        """Used by _applyProductionRequestsLogic"""
        if tInfo["isIdle"] == "No":
            # 'Idle' && !isIdle() --> 'Active'
            self.__updateTransformationStatus(tID, "Idle", "Active", updatedT)
        elif tInfo["isIdle"] != "Yes":
            return

        if summary["type"] == "Simulation":
            if self._isReallyDone(summary):
                # 'Idle' && isIdle() && isDone for MC logic
                if tInfo["Used"]:  # for standard sim requests, only the merge will go to ValidatingOutput
                    if self._producersAreIdle(summary):
                        self.__updateTransformationStatus(tID, "Idle", "ValidatingOutput", updatedT)
                    # else
                    #  it can happened that MC is !isIdle()
                else:  # for standard sim requests, all but the merge will go to ValidatingInput
                    if self._mergersAreProcIdle(summary):
                        # Note: 'isSimulation' should not be there (it should stay in 'Active')
                        self.__updateTransformationStatus(tID, "Idle", "ValidatingInput", updatedT)
                    # else:
                    #   We wait till mergers finish the job

        elif summary["type"] == "AnalysisProduction":
            if not tInfo["hasActiveInput"]:
                # The number of bookkeeping events is not well defined for Analysis Productions
                allFilesProcessed = (
                    tInfo["filesTotal"] and tInfo["filesTotal"] == tInfo["filesProcessed"] + tInfo["filesNotProcessed"]
                )
                if allFilesProcessed:
                    if tInfo["Used"]:
                        self.__updateTransformationStatus(tID, "Idle", "ValidatingOutput", updatedT)
                    else:
                        self.__updateTransformationStatus(tID, "Idle", "Completed", updatedT)
                elif (
                    tInfo["filesTotal"] == tInfo["filesProcessed"] + tInfo["filesNotProcessed"] + tInfo["filesMaxReset"]
                ):
                    self.log.warn("Transformation has files in MaxReset", f"ID={tID} count={tInfo['filesMaxReset']}")

    def _handleStateRemovedFiles(self, tID, tInfo, summary, updatedT):
        """Used by _applyProductionRequestsLogic"""
        self.__updateTransformationStatus(tID, "RemovedFiles", "Completed", updatedT)

    def _handleStateActive(self, tID, tInfo, summary, updatedT):
        """Used by _applyProductionRequestsLogic"""
        if tInfo["isIdle"] == "Yes":
            if summary["type"] == "Simulation":
                # 'Active' && isIdle() for MC logic
                if tInfo["Used"] or not tInfo["isSimulation"]:
                    # The merger will either wait for MC extention (if !isDone)
                    # or will start validation once producers are isIdle()
                    self.__updateTransformationStatus(tID, "Active", "Idle", updatedT)
                # 'Active' && isIdle() && !Used && isSimulation
                elif self._isReallyDone(summary):
                    if self._mergersAreProcIdle(summary):
                        self.__updateTransformationStatus(tID, "Active", "ValidatingInput", updatedT)
                elif self._processorsAreProcIdle(summary) or self._requestedMoreThenProduced(tID, summary):
                    # we are not done yet, extend production
                    self.__updateTransformationStatus(tID, "Active", "Idle", updatedT)
                # else:
                #  we wait till the situation with mergers is clear
            else:
                # for not MC, use reasonable default
                self.__updateTransformationStatus(tID, "Active", "Idle", updatedT)
        elif summary["type"] == "AnalysisProduction" and not tInfo["hasActiveInput"]:
            if (
                tInfo["filesUnused"]
                and tInfo["filesTotal"]
                == tInfo["filesProcessed"] + tInfo["filesNotProcessed"] + tInfo["filesUnused"] + tInfo["filesMaxReset"]
            ):
                self.__updateTransformationStatus(tID, "Active", "Flush", updatedT)

    def _handleStateValidatedOutput(self, tID, tInfo, summary, updatedT):
        """Used by _applyProductionRequestsLogic"""
        if summary["type"] == "Simulation" and summary["isDone"] and tInfo["Used"]:
            # for standard sim requests, only the merge
            self.__updateTransformationStatus(tID, "ValidatedOutput", "Completed", updatedT)
        elif summary["type"] == "AnalysisProduction" and tInfo["Used"]:
            # isDone only uses the bookkeeping and therefore isn't useful
            self.__updateTransformationStatus(tID, "ValidatedOutput", "Completed", updatedT)
        else:
            self.log.warn(f"Logical bug: transformation {tID} unexpectedly has 'ValidatedOutput'")

    def _handleStateValidatingInput(self, tID, tInfo, summary, updatedT):
        """Used by _applyProductionRequestsLogic"""
        if summary["type"] == "Simulation" and summary["isDone"] and not tInfo["Used"]:
            # for standard sim requests, all but the merge
            self.__updateTransformationStatus(tID, "ValidatingInput", "RemovingFiles", updatedT)
        else:
            self.log.warn(f"Logical bug: transformation {tID} is unexpectedly 'ValidatingInput'")

    def _handleStateTesting(self, tID, tInfo, summary, updatedT):
        """Used by _applyProductionRequestsLogic"""
        try:
            # TODO This should ideally be moved out of the loop
            transformations, taskStatuses, fileStatuses = self._isIdleCache([str(tID)])
        except RuntimeError:
            self.log.error("Failed to get _isIdleCache for", str(tID))
            return
        isIdle, isProcIdle, isSimulation = self.__isIdle(
            tID, transformations[tID], taskStatuses[tID], fileStatuses[tID]
        )
        self.log.verbose("TransID %d, %s, %s, %s" % (tID, isIdle, isProcIdle, isSimulation))
        if isIdle:
            self.__updateTransformationStatus(tID, "Testing", "Idle", updatedT)
