###############################################################################
# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
#                                                                             #
# This software is distributed under the terms of the GNU General Public      #
# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".   #
#                                                                             #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization  #
# or submit itself to any jurisdiction.                                       #
###############################################################################
"""Queries creation."""
import datetime
import re

from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.Core.Utilities.Decorators import deprecated

global default
default = "ALL"


class LegacyOracleBookkeepingDB:
    """This class provides all the methods which manipulate the database."""

    #############################################################################

    def __init__(self, *, dbW, dbR):
        """c'tor."""
        self.log = gLogger.getSubLogger("LegacyOracleBookkeepingDB")
        self.dbW_ = dbW
        self.dbR_ = dbR

    #############################################################################
    def getAvailableSteps(self, in_dict):
        """For retrieving a list of steps for a given condition.

        :param dict in_dict: contains step conditions
        :retrun: list of steps
        """
        start = 0
        maximum = 10
        paging = False
        retVal = None
        fileTypefilter = None

        condition = ""
        tables = "steps s, steps r, runtimeprojects rr "
        isMulticore = in_dict.get("isMulticore", default)
        if isMulticore.upper() != default:
            if isMulticore.upper() in ["Y", "N"]:
                condition += f" AND s.isMulticore='{isMulticore}'"
            else:
                return S_ERROR("isMulticore is not Y or N!")
        if in_dict:
            queryKwparams = {}
            infiletypes = in_dict.get("InputFileTypes", default)
            outfiletypes = in_dict.get("OutputFileTypes", default)
            matching = in_dict.get("Equal", "YES")

            if isinstance(matching, bool):
                if matching:
                    matching = "YES"
                else:
                    matching = "NO"
            elif matching.upper() not in ["YES", "NO"]:
                return S_ERROR("Wrong Equal value!")

            if infiletypes != default or outfiletypes != default:
                if isinstance(infiletypes, str):
                    infiletypes = []
                if isinstance(outfiletypes, str):
                    outfiletypes = []
                inp = "lists(" + ",".join(f"'{x}'" for x in sorted(infiletypes)) + ")"
                out = "lists(" + ",".join(f"'{x}'" for x in sorted(outfiletypes)) + ")"
                fileTypefilter = (
                    " table(BOOKKEEPINGORACLEDB.getStepsForFiletypes(%s, %s, '%s')) s \
                                   "
                    % (inp, out, matching.upper())
                )

            startDate = in_dict.get("StartDate", default)
            if startDate != default:
                condition += f" AND s.inserttimestamps >= TO_TIMESTAMP (' {startDate} ' ,'YYYY-MM-DD HH24:MI:SS')"

            stepId = in_dict.get("StepId", default)
            if stepId != default:
                if isinstance(stepId, (str, int)):
                    condition += f" AND s.stepid= {str(stepId)}"
                elif isinstance(stepId, (list, tuple)):
                    condition += f" AND s.stepid in ({','.join(str(sid) for sid in stepId)})"
                else:
                    return S_ERROR("Wrong StepId")

            stepName = in_dict.get("StepName", default)
            if stepName != default:
                if isinstance(stepName, str):
                    condition += f" AND s.stepname='{stepName}'"
                elif isinstance(stepName, list):
                    values = " AND ("
                    for i in stepName:
                        values += f" s.stepname='{i}' OR "
                    condition += values[:-3] + ")"

            appName = in_dict.get("ApplicationName", default)
            if appName != default:
                if isinstance(appName, str):
                    condition += f" AND s.applicationName='{appName}'"
                elif isinstance(appName, list):
                    values = " AND ("
                    for i in appName:
                        values += f" s.applicationName='{i}' OR "
                    condition += values[:-3] + ")"

            appVersion = in_dict.get("ApplicationVersion", default)
            if appVersion != default:
                if isinstance(appVersion, str):
                    condition += f" AND s.applicationversion='{appVersion}'"
                elif isinstance(appVersion, list):
                    values = " AND ("
                    for i in appVersion:
                        values += f" s.applicationversion='{i}' OR "
                    condition += values[:-3] + ")"

            optFile = in_dict.get("OptionFiles", default)
            if optFile != default:
                if isinstance(optFile, str):
                    condition += " AND s.optionfiles = :optionfiles"
                    queryKwparams["optionfiles"] = optFile
                elif isinstance(optFile, list):
                    bindNames = {f"optionfiles{i}": _optFile for i, _optFile in enumerate(optFile)}
                    condition += f" AND s.optionfiles in ({','.join(f':{b}' for b in bindNames)})"
                    queryKwparams.update(bindNames)

            dddb = in_dict.get("DDDB", default)
            if dddb != default:
                if isinstance(dddb, str):
                    condition += f" AND s.dddb='{dddb}'"
                elif isinstance(dddb, list):
                    values = " AND ("
                    for i in dddb:
                        values += f" s.dddb='{i}' OR "
                    condition += values[:-3] + ")"

            conddb = in_dict.get("CONDDB", default)
            if conddb != default:
                if isinstance(conddb, str):
                    condition += f" AND s.conddb='{conddb}'"
                elif isinstance(conddb, list):
                    values = " AND ("
                    for i in conddb:
                        values += f" s.conddb='{i}' OR "
                    condition += values[:-3] + ")"

            extraP = in_dict.get("ExtraPackages", default)
            if extraP != default:
                if isinstance(extraP, str):
                    condition += f" AND s.extrapackages='{extraP}'"
                elif isinstance(extraP, list):
                    values = " AND ("
                    for i in extraP:
                        values += f" s.extrapackages='{i}' OR "
                    condition += values + ")"

            visible = in_dict.get("Visible", default)
            if visible != default:
                if isinstance(visible, str):
                    condition += f" AND s.visible='{visible}'"
                elif isinstance(visible, list):
                    values = " AND ("
                    for i in visible:
                        values += f" s.visible='{i}' OR "
                    condition += values[:-3] + ")"

            procPass = in_dict.get("ProcessingPass", default)
            if procPass != default:
                if isinstance(procPass, str):
                    condition += f" AND s.processingpass LIKE'%{procPass}%'"
                elif isinstance(procPass, list):
                    values = " AND ("
                    for i in procPass:
                        values += f" s.processingpass LIKE '%{i}%' OR "
                    condition += values[:-3] + ")"

            usable = in_dict.get("Usable", default)
            if usable != default:
                if isinstance(usable, str):
                    condition += f" AND s.usable='{usable}'"
                elif isinstance(usable, list):
                    values = " AND ("
                    for i in usable:
                        values += f" s.usable='{i}' OR "
                    condition += values[:-3] + ")"

            runtimeProject = in_dict.get("RuntimeProjects", default)
            if runtimeProject != default:
                condition += " AND s.runtimeProject=%d" % (runtimeProject)

            dqtag = in_dict.get("DQTag", default)
            if dqtag != default:
                if isinstance(dqtag, str):
                    condition += f" AND s.dqtag='{dqtag}'"
                elif isinstance(dqtag, list):
                    values = " AND ("
                    for i in dqtag:
                        values += f"  s.dqtag='{i}' OR "
                    condition += values[:-3] + ")"

            optsf = in_dict.get("OptionsFormat", default)
            if optsf != default:
                if isinstance(optsf, str):
                    condition += f" AND s.optionsFormat='{optsf}'"
                elif isinstance(optsf, list):
                    values = " AND ("
                    for i in optsf:
                        values += f" s.optionsFormat='{i}' OR "
                    condition += values[:-3] + ")"

            sysconfig = in_dict.get("SystemConfig", default)
            if sysconfig != default:
                condition += f" AND s.systemconfig='{sysconfig}'"

            mcTck = in_dict.get("mcTCK", default)
            if mcTck != default:
                condition += f" AND s.mcTCK='{mcTck}'"

            start = in_dict.get("StartItem", default)
            maximum = in_dict.get("MaxItem", default)

            if start != default and maximum != default:
                paging = True

            sort = in_dict.get("Sort", default)
            if sort != default:
                condition += "Order by "
                order = sort.get("Order", "Asc")
                if order.upper() not in ["ASC", "DESC"]:
                    return S_ERROR("wrong sorting order!")
                items = sort.get("Items", default)
                if isinstance(items, list):
                    order = ""
                    for item in items:
                        order += f"s.{item},"
                    condition += f" {order[:-1]} {order}"
                elif isinstance(items, str):
                    condition += f" s.{items} {order}"
                else:
                    return S_ERROR("SortItems is not properly defined!")
            else:
                condition += " ORDER BY s.inserttimestamps DESC"
            if fileTypefilter:
                if paging:
                    command = (
                        "SELECT sstepid, sname, sapplicationname, sapplicationversion, soptionfiles, \
sdddb, sconddb, sextrapackages, svisible, sprocessingpass, susable, sdqtag, soptsf, smulti, ssysconfig, smcTck, \
rsstepid, rsname, rsapplicationname, rsapplicationversion, rsoptionfiles, rsdddb, \
rsconddb, rsextrapackages, rsvisible, rsprocessingpass, rsusable, rdqtag, roptsf, rmulti, rsysconfig, rmcTck FROM \
(SELECT ROWNUM r , sstepid, sname, sapplicationname, sapplicationversion, soptionfiles, sdddb, sconddb, \
sextrapackages, svisible, sprocessingpass, susable, sdqtag, soptsf, smulti, ssysconfig, smcTck, \
rsstepid, rsname, rsapplicationname, rsapplicationversion, rsoptionfiles, rsdddb, rsconddb, \
rsextrapackages, rsvisible, rsprocessingpass, rsusable , rdqtag, roptsf, rmulti, rsysconfig, rmcTck FROM \
(SELECT ROWNUM r, s.stepid sstepid ,s.stepname sname, s.applicationname sapplicationname, \
s.applicationversion sapplicationversion, s.optionfiles soptionfiles, \
s.DDDB sdddb,s.CONDDB sconddb, s.extrapackages sextrapackages,s.Visible svisible, \
s.ProcessingPass sprocessingpass, s.Usable susable, s.dqtag sdqtag, s.optionsFormat soptsf, \
s.isMulticore smulti, s.systemconfig ssysconfig, s.mcTCK smcTck, \
s.rstepid rsstepid ,s.rstepname rsname, s.rapplicationname rsapplicationname, \
s.rapplicationversion rsapplicationversion, s.roptionfiles rsoptionfiles, \
s.rDDDB rsdddb,s.rCONDDB rsconddb, s.rextrapackages rsextrapackages,s.rVisible rsvisible , \
s.rProcessingPass rsprocessingpass,s.rUsable rsusable, s.rdqtag rdqtag, s.roptionsFormat roptsf, \
s.risMulticore rmulti, s.rsystemconfig rsysconfig, s.mcTCK rmcTck \
FROM %s WHERE s.stepid=s.stepid %s \
) WHERE rownum <=%d ) WHERE r >%d"
                        % (fileTypefilter, condition, maximum, start)
                    )
                else:
                    command = f"SELECT * FROM {fileTypefilter} WHERE s.stepid=s.stepid {condition}"
            elif paging:
                command = (
                    "SELECT sstepid, sname, sapplicationname, sapplicationversion, soptionfiles, \
sdddb, sconddb, sextrapackages, svisible, sprocessingpass, susable, \
sdqtag, soptsf, smulti, ssysconfig, smcTck, \
rsstepid, rsname, rsapplicationname, rsapplicationversion, rsoptionfiles, rsdddb, \
rsconddb, rsextrapackages, rsvisible, rsprocessingpass, rsusable, \
rdqtag, roptsf, rmulti, rsysconfig, rmcTck FROM \
(SELECT ROWNUM r , sstepid, sname, sapplicationname, sapplicationversion, soptionfiles, sdddb, sconddb, \
sextrapackages, svisible, sprocessingpass, susable, sdqtag, soptsf, smulti, ssysconfig, smcTck, \
rsstepid, rsname, rsapplicationname, rsapplicationversion, rsoptionfiles, rsdddb, rsconddb, \
rsextrapackages, rsvisible, rsprocessingpass, rsusable , rdqtag, roptsf, rmulti, rsysconfig, rmcTck FROM \
(SELECT ROWNUM r, s.stepid sstepid ,s.stepname sname, s.applicationname sapplicationname, \
s.applicationversion sapplicationversion, s.optionfiles soptionfiles, \
s.DDDB sdddb,s.CONDDB sconddb, s.extrapackages sextrapackages,s.Visible svisible, \
s.ProcessingPass sprocessingpass, s.Usable susable, s.dqtag sdqtag, s.optionsFormat soptsf, \
s.isMulticore smulti, s.systemconfig ssysconfig, s.mcTCK smcTck, \
r.stepid rsstepid ,r.stepname rsname, r.applicationname rsapplicationname, \
r.applicationversion rsapplicationversion, r.optionfiles rsoptionfiles, \
r.DDDB rsdddb,r.CONDDB rsconddb, r.extrapackages rsextrapackages,r.Visible rsvisible, \
r.ProcessingPass rsprocessingpass,r.Usable rsusable, r.dqtag rdqtag, r.optionsFormat roptsf, \
r.isMulticore rmulti, r.systemconfig rsysconfig, r.mcTCK rmcTck \
FROM %s WHERE s.stepid=rr.stepid(+) AND r.stepid(+)=rr.runtimeprojectid %s \
) WHERE rownum <=%d) WHERE r >%d"
                    % (tables, condition, maximum, start)
                )

            else:
                command = (
                    "SELECT s.stepid,s.stepname, s.applicationname,s.applicationversion,s.optionfiles,s.DDDB,s.CONDDB, \
s.extrapackages,s.Visible, s.ProcessingPass, s.Usable, s.dqtag, s.optionsformat, s.ismulticore, \
s.systemconfig, s.mcTCK, r.stepid, r.stepname, r.applicationname,r.applicationversion,r.optionfiles, \
r.DDDB,r.CONDDB, r.extrapackages,r.Visible, r.ProcessingPass, r.Usable, r.dqtag, r.optionsformat, \
r.ismulticore, r.systemconfig, r.mcTCK FROM %s WHERE s.stepid=rr.stepid(+) AND \
r.stepid(+)=rr.runtimeprojectid  %s "
                    % (tables, condition)
                )
            retVal = self.dbR_.query(command, kwparams=queryKwparams)
        else:
            command = (
                "SELECT s.stepid, s.stepname, s.applicationname,s.applicationversion,s.optionfiles,s.DDDB,s.CONDDB, \
s.extrapackages,s.Visible, s.ProcessingPass, s.Usable, s.dqtag, s.optionsformat, s.isMulticore, s.systemconfig, \
s.mcTCK,r.stepid, r.stepname, r.applicationname,r.applicationversion,r.optionfiles,r.DDDB,r.CONDDB, \
r.extrapackages,r.Visible, r.ProcessingPass, r.Usable, r.dqtag, r.optionsformat, r.ismulticore, r.systemconfig, r.mcTCK \
FROM %s WHERE s.stepid=rr.stepid(+) AND r.stepid(+)=rr.runtimeprojectid "
                % (tables)
            )
            retVal = self.dbR_.query(command)

        if not retVal["OK"]:
            return retVal

        parameters = [
            "StepId",
            "StepName",
            "ApplicationName",
            "ApplicationVersion",
            "OptionFiles",
            "DDDB",
            "CONDDB",
            "ExtraPackages",
            "Visible",
            "ProcessingPass",
            "Usable",
            "DQTag",
            "OptionsFormat",
            "isMulticore",
            "SystemConfig",
            "mcTCK",
            "RuntimeProjects",
        ]
        rParameters = [
            "StepId",
            "StepName",
            "ApplicationName",
            "ApplicationVersion",
            "OptionFiles",
            "DDDB",
            "CONDDB",
            "ExtraPackages",
            "Visible",
            "ProcessingPass",
            "Usable",
            "DQTag",
            "OptionsFormat",
            "isMulticore",
            "SystemConfig",
            "mcTCK",
        ]
        records = []
        for record in retVal["Value"]:
            step = list(record[0:16])
            runtimeProject = []
            runtimeProject = [rec for rec in list(record[16:]) if rec is not None]
            if runtimeProject:
                runtimeProject = [runtimeProject]
            step += [
                {"ParameterNames": rParameters, "Records": runtimeProject, "TotalRecords": len(runtimeProject) + 1}
            ]
            records += [step]

        if not paging:
            return S_OK({"ParameterNames": parameters, "Records": records, "TotalRecords": len(records)})

        if fileTypefilter:
            command = f"SELECT count(*) FROM {fileTypefilter} WHERE s.stepid>0 {condition} "
        else:
            command = f"SELECT count(*) FROM steps s WHERE s.stepid>0 {condition} "

        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        totrec = retVal["Value"][0][0]
        return S_OK({"ParameterNames": parameters, "Records": records, "TotalRecords": totrec})

    #############################################################################
    def getRuntimeProjects(self, in_dict):
        """get runtime projects.

        :param dict in_dict: dictionary which contains the StepId
        :return: runtime projects if no StepId is given otherwise the runtime project
        """
        result = S_ERROR()
        condition = ""
        selection = "s.stepid,stepname, s.applicationname,s.applicationversion,s.optionfiles,s.DDDB,CONDDB,\
s.extrapackages,s.Visible, s.ProcessingPass, s.Usable, s.DQTag, s.optionsformat, s.ismulticore, s.systemconfig, s.mcTCK"
        tables = "steps s, runtimeprojects rp"
        stepId = in_dict.get("StepId", default)
        if stepId != default:
            condition += " rp.stepid=%d" % (stepId)
            command = " SELECT {} FROM {} WHERE s.stepid=rp.runtimeprojectid AND {}".format(
                selection, tables, condition
            )
            retVal = self.dbR_.query(command)
            if retVal["OK"]:
                parameters = [
                    "StepId",
                    "StepName",
                    "ApplicationName",
                    "ApplicationVersion",
                    "OptionFiles",
                    "DDDB",
                    "CONDDB",
                    "ExtraPackages",
                    "Visible",
                    "ProcessingPass",
                    "Usable",
                    "DQTag",
                    "OptionsFormat",
                    "isMulticore",
                    "SystemConfig",
                    "mcTCK",
                ]
                records = []
                for record in retVal["Value"]:
                    records += [list(record)]
                result = S_OK({"ParameterNames": parameters, "Records": records, "TotalRecords": len(records)})
            else:
                result = retVal
        else:
            result = S_ERROR("You must provide a StepId!")
        return result

    #############################################################################
    def getStepInputFiles(self, stepId):
        """input file types of a given step.

        :param int stepId: given step id.
        :return: the step input files
        """
        command = (
            "SELECT inputFiletypes.name,inputFiletypes.visible FROM steps, \
table(steps.InputFileTypes) inputFiletypes WHERE steps.stepid="
            + str(stepId)
        )
        return self.dbR_.query(command)

    #############################################################################
    def setStepInputFiles(self, stepid, fileTypes):
        """set input file types to a given step.

        :param int stepId: given step id.
        :param list fileTypes: file types
        """
        fileTypes = sorted(fileTypes, key=lambda k: k["FileType"])
        if not fileTypes:
            values = "null"
        else:
            values = "filetypesARRAY("
            for i in fileTypes:
                fileType = i.get("FileType", default)
                visible = i.get("Visible", default)
                if fileType != default and visible != default:
                    values += f"ftype('{fileType}','{visible}'),"
            values = values[:-1]
            values += ")"
        command = f"UPDATE steps SET inputfiletypes={values} WHERE stepid={str(stepid)}"
        return self.dbW_.query(command)

    #############################################################################
    def setStepOutputFiles(self, stepid, fileTypes):
        """set output file types to a given step.

        :param int stepid: given step id
        :param list fileTypes: list of file types
        """
        fileTypes = sorted(fileTypes, key=lambda k: k["FileType"])
        if not fileTypes:
            values = "null"
        else:
            values = "filetypesARRAY("
            for i in fileTypes:
                fileType = i.get("FileType", default)
                visible = i.get("Visible", default)
                if fileType != default and visible != default:
                    values += f"ftype('{fileType}','{visible}'),"
            values = values[:-1]
            values += ")"
        command = f"UPDATE steps SET Outputfiletypes={values} WHERE stepid={str(stepid)}"
        return self.dbW_.query(command)

    #############################################################################
    def getStepOutputFiles(self, stepId):
        """For retrieving the step output file types.

        :param int stepid: step id
        :return: the output file types for a given step
        """
        command = (
            "SELECT outputfiletypes.name, outputfiletypes.visible FROM "
            "steps, table(steps.outputfiletypes) outputfiletypes WHERE steps.stepid=" + str(stepId)
        )
        return self.dbR_.query(command)

    #############################################################################
    def getProductionOutputFileTypes(self, prod, stepid):
        """returns the production output file types.

        :param int prod:  production number
        :param int stepid: step id
        :return S_OK/S_ERROR: return a dictionary with file types and visibility flag.
        """
        condition = ""
        if stepid != default:
            condition = f" AND s.stepid={stepid}"

        command = (
            "SELECT DISTINCT ft.name, s.visible from productionoutputfiles s, filetypes FT WHERE "
            "s.filetypeid=ft.filetypeid AND s.production=%s %s" % (prod, condition)
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        if not retVal["Value"]:
            # this is for backward compatibility.
            # FIXME: make sure the productionoutputfiles is correctly propagated and after the method can be simpified
            command = (
                "SELECT o.name,o.visible from steps s, table(s.outputfiletypes) o, stepscontainer st \
WHERE st.stepid=s.stepid AND st.production=%d %s ORDER BY step"
                % (int(prod), condition)
            )
            retVal = self.dbR_.query(command)

        outputFiles = {}
        if retVal["OK"]:
            for filetype, visible in retVal["Value"]:
                outputFiles[filetype] = visible
        else:
            return retVal

        return S_OK(outputFiles)

    #############################################################################
    def getAvailableFileTypes(self):
        """
        For retrieving all file types.

        :return: the available file types
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getAvailableFileTypes", [])

    #############################################################################
    def insertFileTypes(self, ftype, desc, fileType):
        """inserts a given file type."""
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.insertFileTypes", int, [ftype, desc, fileType])

    #############################################################################
    def insertStep(self, in_dict):
        """
        inserts a given step for example:

        .. code-block:: python

          {'Step': {'ApplicationName': 'DaVinci',
                    'Usable': 'Yes',
                    'StepId': '',
                    'ApplicationVersion': 'v29r1',
                    'ExtraPackages': '',
                    'StepName': 'davinci prb2',
                    'ProcessingPass': 'WG-Coool',
                    'Visible': 'Y',
                    'isMulticore': 'N',
                    'OptionFiles': '',
                    'DDDB': '',
                    'CONDDB': ''},
           'OutputFileTypes': [{'Visible': 'Y', 'FileType': 'CHARM.MDST'}],
           'InputFileTypes': [{'Visible': 'Y', 'FileType': 'CHARM.DST'}],
           'RuntimeProjects': [{'StepId': 13878}]}

        :param dict in_dict: dictionary which contains step parameters
        """
        result = S_ERROR()
        values = ""
        command = "SELECT applications_index_seq.nextval from dual"
        sid = 0
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            result = retVal
        else:
            sid = retVal["Value"][0][0]

        selection = "INSERT INTO steps(stepid,stepname,applicationname,applicationversion,OptionFiles,dddb,conddb, \
extrapackages,visible, processingpass, usable, DQTag, optionsformat,isMulticore, SystemConfig, mcTCK"
        inFileTypes = in_dict.get("InputFileTypes", default)
        if inFileTypes != default:
            inFileTypes = sorted(inFileTypes, key=lambda k: k["FileType"])
            values = ",filetypesARRAY("
            selection += ",InputFileTypes"
            for i in inFileTypes:
                values += "ftype('{}', '{}'),".format(
                    (i.get("FileType", "").strip() if i.get("FileType", "") else i.get("FileType", "")),
                    (i.get("Visible", "").strip() if i.get("Visible", "") else i.get("Visible", "")),
                )
            values = values[:-1]
            values += ")"

        outFileTypes = in_dict.get("OutputFileTypes", default)
        if outFileTypes != default:
            outFileTypes = sorted(outFileTypes, key=lambda k: k["FileType"])
            values += " , filetypesARRAY("
            selection += ",OutputFileTypes"
            for i in outFileTypes:
                values += "ftype('{}', '{}'),".format(
                    (i.get("FileType", "").strip() if i.get("FileType", "") else i.get("FileType", "")),
                    (i.get("Visible", "").strip() if i.get("Visible", "") else i.get("Visible", "")),
                )
            values = values[:-1]
            values += ")"

        step = in_dict.get("Step", default)
        if step != default:
            names = {
                "StepName": "NULL",
                "ApplicationName": "NULL",
                "ApplicationVersion": "NULL",
                "OptionFiles": "NULL",
                "DDDB": "NULL",
                "CONDDB": "NULL",
                "ExtraPackages": "NULL",
                "Visible": "NULL",
                "ProcessingPass": "NULL",
                "Usable": "Not ready",
                "DQTag": "",
                "OptionsFormat": "",
                "isMulticore": "N",
                "SystemConfig": "NULL",
                "mcTCK": "NULL",
            }
            bindNames = {"sid": sid}
            bindNames.update({k.lower(): step.get(k, v) for k, v in names.items()})
            command = f"{selection}) values ({','.join(f':{b}' for b in bindNames)} {values})"
            retVal = self.dbW_.query(command, kwparams=bindNames)
            if retVal["OK"]:
                r_project = in_dict.get("RuntimeProjects", step.get("RuntimeProjects", default))
                if r_project != default:
                    for i in r_project:
                        rid = i["StepId"]
                        retVal = self.insertRuntimeProject(sid, rid)
                        if not retVal["OK"]:
                            result = retVal
                        else:
                            result = S_OK(sid)
                else:
                    result = S_OK(sid)
            else:
                result = retVal
        else:
            result = S_ERROR("The Step is not provided!")
        return result

    #############################################################################
    def deleteStep(self, stepid):
        """deletes a step.

        :param int stepid: step id to be deleted
        """
        self.log.warn("Deleting step", stepid)

        retVal = self.dbW_.query("DELETE runtimeprojects WHERE stepid=%d" % (stepid))
        if not retVal["OK"]:
            return retVal
        # now we can delete the step
        return self.dbW_.query("DELETE steps WHERE stepid=%d" % (stepid))

    #############################################################################

    def deleteStepContainer(self, prod):
        """delete a production from the step container.

        :param int prod: production number
        """
        self.log.warn("Deleting step container for prod", prod)
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.deleteStepContainer", [prod], False)

    #############################################################################

    def deleteProductionsContainer(self, prod):
        """delete a production from the productions container.

        :param int prod: the production number
        """
        self.log.warn("Deleting production container for prod", prod)
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.deleteProductionsCont", [prod], False)

    #############################################################################
    def updateStep(self, in_dict):
        """update an existing step.
        input data {'ApplicationName': 'DaVinci', 'Usable': 'Yes', 'StepId': '13860',
        'ApplicationVersion': 'v29r1', 'ExtraPackages': '', 'StepName': 'davinci prb3', 'ProcessingPass':
        'WG-Coool-new', 'InputFileTypes': [{'Visible': 'Y', 'FileType': 'CHARM.DST'}], 'Visible': 'Y',
        'DDDB': '', 'OptionFiles': '', 'CONDDB': '',
        'OutputFileTypes': [{'Visible': 'Y', 'FileType': 'CHARM.MDST'}],
        'RuntimeProjects':[{'StepId':13879}]}

        :param dict in_dict: step parameters which will be updated
        """
        result = S_ERROR()
        ok = True
        rProjects = in_dict.get("RuntimeProjects", default)
        if rProjects != default:
            if rProjects:
                for i in rProjects:
                    if "StepId" not in in_dict:
                        result = S_ERROR("The runtime project can not changed, because the StepId is missing!")
                        ok = False
                    else:
                        retVal = self.updateRuntimeProject(in_dict["StepId"], i["StepId"])
                        if not retVal["OK"]:
                            result = retVal
                            ok = False
                        else:
                            in_dict.pop("RuntimeProjects")
            else:
                retVal = self.removeRuntimeProject(in_dict["StepId"])
                if not retVal["OK"]:
                    result = retVal
                    ok = False
                else:
                    in_dict.pop("RuntimeProjects")

        if ok:
            stepid = in_dict.get("StepId", default)
            if stepid != default:
                in_dict.pop("StepId")
                condition = f" WHERE stepid={str(stepid)}"
                command = "UPDATE steps set "
                for i in in_dict:
                    if isinstance(in_dict[i], str):
                        command += f" {i}='{str(in_dict[i])}',"
                    else:
                        if in_dict[i]:
                            values = "filetypesARRAY("
                            ftypes = in_dict[i]
                            ftypes = sorted(ftypes, key=lambda k: k["FileType"])
                            for j in ftypes:
                                filetype = j.get("FileType", default)
                                if filetype != default:
                                    values += f"ftype('{filetype.strip()}',''),"
                            values = values[:-1]
                            values += ")"
                            command += i + "=" + values + ","
                        else:
                            command += i + "=null,"
                command = command[:-1]
                command += condition
                result = self.dbW_.query(command)
            else:
                result = S_ERROR("Please provide a StepId!")

        return result

    #############################################################################
    def getAvailableConfigNames(self):
        """For retrieving the list of configuration names using the materialized
        view.

        :return: the available configuration names
        """
        command = (
            "SELECT c.configname from configurations c, productionoutputfiles prod, productionscontainer cont \
WHERE cont.configurationid=c.configurationid AND prod.production=cont.production %s GROUP BY c.configname ORDER BY c.configname"
            % self.__buildVisible(visible="Y", replicaFlag="Yes")
        )
        return self.dbR_.query(command)

    ##############################################################################
    def getAvailableConfigurations(self):
        """For retrieving all available configurations even the configurations
        which are not used.

        :return: the available configurations from the configurations table
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getAvailableConfigurations", [])

    #############################################################################
    def getConfigVersions(self, configname):
        """For retrieving configuration version.

        :param str configname: the configuration name for example: MC, LHCb, etc.
        :return: the configuration version for a given configname
        """
        result = S_ERROR()
        if configname != default:
            command = (
                "SELECT c.configversion from configurations c, productionoutputfiles prod, productionscontainer cont \
WHERE cont.configurationid=c.configurationid AND c.configname='%s' AND prod.production=cont.production %s \
GROUP BY c.configversion ORDER BY c.configversion"
                % (configname, self.__buildVisible(visible="Y", replicaFlag="Yes"))
            )
            result = self.dbR_.query(command)
        else:
            result = S_ERROR("You must provide a Configuration Name!")
        return result

    #############################################################################
    def getConditions(self, configName, configVersion, evt):
        """Retrieving the data taking or simulation conditions for a given event type.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param int evt: event type id
        :return: the conditions for a given configuration name, version and event type
        """

        condition = f" AND cont.production=prod.production {self.__buildVisible(visible='Y', replicaFlag='Yes')} "
        tables = " configurations c, productionscontainer cont, productionoutputfiles prod "
        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        if evt != default:
            condition += f" AND prod.eventtypeid={str(evt)}"

        command = (
            "SELECT DISTINCT simulationConditions.SIMID,data_taking_conditions.DAQPERIODID, \
simulationConditions.SIMDESCRIPTION, simulationConditions.BEAMCOND, \
simulationConditions.BEAMENERGY, simulationConditions.GENERATOR, \
simulationConditions.MAGNETICFIELD,simulationConditions.DETECTORCOND, \
simulationConditions.LUMINOSITY, simulationconditions.G4settings, \
data_taking_conditions.DESCRIPTION,data_taking_conditions.BEAMCOND, \
data_taking_conditions.BEAMENERGY,data_taking_conditions.MAGNETICFIELD, \
data_taking_conditions.VELO,data_taking_conditions.IT, \
data_taking_conditions.TT,data_taking_conditions.OT, \
data_taking_conditions.RICH1,data_taking_conditions.RICH2, \
data_taking_conditions.SPD_PRS, data_taking_conditions.ECAL, \
data_taking_conditions.HCAL, data_taking_conditions.MUON, data_taking_conditions.L0, data_taking_conditions.HLT, \
data_taking_conditions.VeloPosition FROM simulationConditions,data_taking_conditions, %s WHERE \
cont.simid=simulationConditions.simid(+) AND cont.DAQPERIODID=data_taking_conditions.DAQPERIODID(+) %s"
            % (tables, condition)
        )

        return self.dbR_.query(command)

    #############################################################################
    def getProcessingPass(
        self, configName, configVersion, conddescription, runnumber, production, eventType=default, path="/"
    ):
        """For retrieving the processing pass for given conditions.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: data taking or simulation description
        :param int runnumber: run number
        :param int production: production number
        :param eventType: event type identifier
        :param str path: processing pass
        :return: the processing pass for a given dataset
        """
        erecords = []
        eparameters = []
        precords = []
        pparameters = []

        condition = f" AND cont.production=prod.production {self.__buildVisible(visible='Y', replicaFlag='Yes')} "
        tables = " productionscontainer cont, productionoutputfiles prod "
        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        if eventType != default:
            condition += f" AND prod.eventtypeid={str(eventType)}"

        if conddescription != default:
            retVal = self.__getConditionString(conddescription, "cont")
            if not retVal["OK"]:
                return retVal
            else:
                condition += retVal["Value"]

        if production != default:
            condition += " AND prod.production=" + str(production)

        tables = ""
        if runnumber != default:
            tables += " , prodrunview "
            condition += f" AND prodrunview.production=prod.production AND prodrunview.runnumber={str(runnumber)}"

        proc = path.split("/")[len(path.split("/")) - 1]
        if proc != "":
            command = (
                "SELECT v.id FROM (SELECT DISTINCT SYS_CONNECT_BY_PATH(name, '/') Path, id ID \
FROM processing v \
START WITH id in (SELECT DISTINCT id FROM processing WHERE name='%s') \
CONNECT BY NOCYCLE PRIOR  id=parentid) v \
WHERE v.path='%s'"
                % (path.split("/")[1], path)
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                return retVal
            pro = ""
            for i in retVal["Value"]:
                pro += f"{str(i[0])},"
            pro = pro[:-1]

            if pro == "":
                return S_ERROR("Empty Directory")
            command = (
                "SELECT DISTINCT eventTypes.EventTypeId, eventTypes.Description FROM "
                "eventtypes, productionoutputfiles prod, productionscontainer cont, configurations c, processing %s WHERE "
                "eventTypes.EventTypeId=prod.eventtypeid AND cont.processingid=processing.id AND processing.id in (%s) %s"
                % (tables, pro, condition)
            )

            retVal = self.dbR_.query(command)
            if retVal["OK"]:
                eparameters = ["EventType", "Description"]
                for record in retVal["Value"]:
                    erecords += [list(record)]
            else:
                return retVal

            command = (
                "SELECT DISTINCT name FROM processing WHERE parentid in (%s) "
                "START WITH id in "
                "(SELECT DISTINCT cont.processingid FROM "
                "productionscontainer cont, productionoutputfiles prod, configurations c %s WHERE "
                "cont.production=prod.production  %s) CONNECT BY NOCYCLE PRIOR  parentid=id ORDER BY name DESC"
                % (pro, tables, condition)
            )
        else:
            command = (
                "SELECT DISTINCT name FROM processing  WHERE parentid is null START WITH id IN "
                "(SELECT DISTINCT cont.processingid FROM "
                "productionscontainer cont, productionoutputfiles prod, configurations c %s WHERE "
                "cont.production=prod.production %s) CONNECT BY NOCYCLE PRIOR  parentid=id ORDER BY name DESC"
                % (tables, condition)
            )
        retVal = self.dbR_.query(command)
        if retVal["OK"]:
            precords = []
            pparameters = ["Name"]
            for record in retVal["Value"]:
                precords += [[record[0]]]
        else:
            return retVal

        return S_OK(
            [
                {"ParameterNames": pparameters, "Records": precords, "TotalRecords": len(precords)},
                {"ParameterNames": eparameters, "Records": erecords, "TotalRecords": len(erecords)},
            ]
        )

    #############################################################################
    def __getConditionString(self, conddescription, table="productionscontainer"):
        """builds the condition for data taking/ simulation conditions.

        :param str conddescription: data taking or simulation condition
        :param str table: table(s) will be used in the JOIN
        :return: condition used in the SQL WHERE clauses.
        """
        condition = ""
        retVal = self._getDataTakingConditionId(conddescription)
        if retVal["OK"]:
            if retVal["Value"] != -1:
                condition += " AND {}.DAQPERIODID={} AND {}.DAQPERIODID is not null ".format(
                    table,
                    str(retVal["Value"]),
                    table,
                )
            else:
                retVal = self.__getSimulationConditionId(conddescription)
                if retVal["OK"]:
                    if retVal["Value"] != -1:
                        condition += " AND {}.simid={} AND {}.simid is not null ".format(
                            table, str(retVal["Value"]), table
                        )
                    else:
                        return S_ERROR("Condition does not exist!")
                else:
                    return retVal
        else:
            return retVal
        return S_OK(condition)

    #############################################################################
    def _getDataTakingConditionId(self, desc):
        """For retrieving the data taking id for a given data taking description.

        :param str desc: data taking description
        :return: the data taking conditions identifire
        """
        command = "SELECT DAQPERIODID FROM data_taking_conditions WHERE DESCRIPTION='" + str(desc) + "'"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        if retVal["Value"]:
            return S_OK(retVal["Value"][0][0])
        return S_OK(-1)

    #############################################################################
    def __getSimulationConditionId(self, desc):
        """For retrieving the simulation condition id for a given simulation
        description.

        :param str desc: simulation condition description
        :return: the simulation condition identifier
        """
        command = f"SELECT simid FROM simulationconditions WHERE simdescription='{desc}'"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        if retVal["Value"]:
            return S_OK(retVal["Value"][0][0])
        return S_OK(-1)

    #############################################################################
    def getProductions(
        self,
        configName=default,
        configVersion=default,
        conddescription=default,
        processing=default,
        evt=default,
        visible=default,
        fileType=default,
        replicaFlag=default,
    ):
        """For retrieving the productions.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: data taking or simulation description
        :param str processing: processing pass
        :param int evt: event type identifier
        :param str visible: the file visibility flag
        :param str file type: file type
        :param str replicaFlag: replica flag
        :return: list of productions
        """

        tables = " productionoutputfiles prod, productionscontainer cont "
        condition = " AND cont.production=prod.production %s " % self.__buildVisible(
            visible=visible, replicaFlag=replicaFlag
        )

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        retVal = self._buildConditions(default, conddescription, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildEventType(evt, condition, tables, useMainTables=False)

        condition, tables = self.__buildFileTypes(fileType, condition, tables, useMainTables=False)

        retVal = self.__buildProcessingPass(processing, condition, tables, useMainTables=False)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        command = f"SELECT prod.production FROM {tables} WHERE 1=1  {condition} GROUP BY prod.production"

        return self.dbR_.query(command)

    #############################################################################
    def getFileTypes(
        self,
        configName,
        configVersion,
        conddescription=default,
        processing=default,
        evt=default,
        runnb=default,
        production=default,
        visible=default,
        replicaFlag=default,
    ):
        """For retrieving the file types

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: data taking or simulation description
        :param str processing: processing pass
        :param int evt: event type identifier
        :param int runnb: run number
        :param int production: production number
        :param str visible: the file visibility flag
        :param str file type: file type
        :param str replicaFlag: replica flag
        :return: the file types
        """

        tables = " productionoutputfiles prod, productionscontainer cont, filetypes ftypes "
        condition = " AND cont.production=prod.production %s " % self.__buildVisible(
            visible=visible, replicaFlag=replicaFlag
        )

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        retVal = self._buildConditions(default, conddescription, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildEventType(evt, condition, tables, useMainTables=False)

        condition, tables = self.__buildProduction(production, condition, tables, useMainTables=False)

        retVal = self._buildRunnumbers(runnb, None, None, condition, tables, useMainTables=False)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        proc = ""
        if processing != default:
            command = (
                "SELECT v.id FROM (SELECT DISTINCT SYS_CONNECT_BY_PATH(name, '/') Path, id ID \
FROM processing v \
START WITH id in (SELECT DISTINCT id FROM processing WHERE name='%s') \
CONNECT BY NOCYCLE PRIOR  id=parentid) v \
WHERE v.path='%s'"
                % (processing.split("/")[1], processing)
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                return retVal
            pro = "("
            for i in retVal["Value"]:
                pro += f"{str(i[0])},"
            pro = pro[:-1]
            pro += ")"
            proc = f" AND cont.processingid in {pro} "
        command = (
            "SELECT ftypes.name FROM %s \
WHERE prod.production=cont.production %s AND prod.filetypeId=ftypes.filetypeid %s GROUP BY ftypes.name"
            % (tables, condition, proc)
        )

        return self.dbR_.query(command)

    #############################################################################
    def getFilesWithMetadata(
        self,
        configName,
        configVersion,
        conddescription=default,
        processing=default,
        evt=default,
        production=default,
        filetype=default,
        quality=default,
        visible=default,
        replicaflag=default,
        startDate=None,
        endDate=None,
        runnumbers=None,
        startRunID=None,
        endRunID=None,
        tcks=default,
        jobStart=None,
        jobEnd=None,
        selection=None,
        smog2States=None,
    ):
        """For retrieving files with meta data.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: data taking or simulation description
        :param str processing: processing pass
        :param int evt: event type identifier
        :param int production: production number
        :param str filetype: file type
        :param str quality: data quality flag
        :param str visible: visibility flag
        :param str replicaflag: replica flag
        :param datetime startDate: job/run insert start time stamp
        :param datetime endDate: job/run end insert time stamp
        :param list runnumbers: run numbers
        :param int startRunID: start run
        :param int endRunID: end run
        :param str tcks: TCK number
        :param datetime jobStart: job starte date
        :param datetime jobEnd: job end date
        :return: a list of files with their metadata
        """

        if runnumbers is None:
            runnumbers = []

        if selection is None:
            selection = (
                " DISTINCT f.FileName, f.EventStat, f.FileSize, f.CreationDate, j.JobStart, j.JobEnd, "
                "j.WorkerNode, ft.Name, j.runnumber, j.fillnumber, f.fullstat, d.dataqualityflag, "
                "j.eventinputstat, j.totalluminosity, f.luminosity, f.instLuminosity, j.tck, f.guid, f.adler32, "
                "f.eventTypeid, f.md5sum,f.visibilityflag, j.jobid, f.gotreplica, f.inserttimestamp "
            )

        tables = " files f, dataquality d, jobs j, productionoutputfiles prod, productionscontainer cont, filetypes ft "
        condition = (
            " AND cont.production=prod.production AND j.production=prod.production "
            " AND j.stepid=prod.stepid AND prod.eventtypeid=f.eventtypeid %s "
            % self.__buildVisible(visible=visible, replicaFlag=replicaflag)
        )

        condition = self.__buildStartenddate(startDate, endDate, condition)

        condition = self.__buildJobsStartJobEndDate(jobStart, jobEnd, condition)

        retVal = self._buildRunnumbers(runnumbers, startRunID, endRunID, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        retVal = self.__buildTCKS(tcks, condition)
        if not retVal["OK"]:
            return retVal
        condition = retVal["Value"]

        condition, tables = self.__buildVisibilityflag(visible, condition, tables)

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        retVal = self._buildConditions(default, conddescription, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildProduction(production, condition, tables, useMainTables=False)

        condition = self.__buildReplicaflag(replicaflag, condition)

        retVal = self.__buildDataquality(quality, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        retVal = self.__buildProcessingPass(processing, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildEventType(evt, condition, tables, useMainTables=False)

        condition, tables = self.__buildFileTypes(filetype, condition, tables, useMainTables=False)

        retVal = self._buildSMOG2Conditions(smog2States, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        command = (
            "SELECT %s FROM %s WHERE j.jobid=f.jobid AND ft.filetypeid=f.filetypeid AND f.qualityid=d.qualityid %s"
            % (selection, tables, condition)
        )
        return self.dbR_.query(command)

    #############################################################################
    def getAvailableDataQuality(self):
        """For retrieving the data quality flags.

        :return: the available data quality flags
        """
        result = S_ERROR()
        command = " SELECT dataqualityflag FROM dataquality"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            result = retVal
        else:
            flags = retVal["Value"]
            values = []
            for i in flags:
                values += [i[0]]
            result = S_OK(values)
        return result

    #############################################################################
    def getAvailableProductions(self):
        """For retrieving the productions form the view.

        :return: the available productions
        """
        command = (
            "SELECT DISTINCT production FROM productionoutputfiles WHERE "
            "production > 0 AND gotreplica='Yes' AND visible='Y'"
        )
        return self.dbR_.query(command)

    #############################################################################
    def getAvailableRuns(self):
        """For retrieving the runs from the view.

        :return: aviable runs
        """
        return self.dbR_.query("SELECT DISTINCT runnumber FROM prodrunview")

    #############################################################################
    def getAvailableEventTypes(self):
        """For retrieving the event types.

        :return: all event types
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getAvailableEventTypes", [])

    #############################################################################
    def getProductionProcessingPass(self, prodid):
        """For retrieving the processing pass from a given production.

        :param int prodid: production number
        :return: processing pass
        """
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getProductionProcessingPass", str, [prodid])

    #############################################################################
    def getRunProcessingPass(self, runnumber):
        """For retrieving the processing pass for a given run number.

        :param int runnumber: run number
        :return: the processing pass for a given run
        """
        return self.dbW_.executeStoredFunctions(
            "BOOKKEEPINGORACLEDB.getProductionProcessingPass", str, [-1 * runnumber]
        )

    #############################################################################
    def getProductionProcessingPassID(self, prodid):
        """For retrieving the processing pass id.

        :param int prodid: production number
        :return: the processing pass identifier of a production
        """
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getProductionProcessingPassId", int, [prodid])

    #############################################################################
    def getMoreProductionInformations(self, prodid):
        """For retrieving the production statistics.

        :param int prodid: production number
        :return: the statistics of a production
        """

        command = (
            "SELECT c.configname, c.configversion, s.ApplicationName, s.ApplicationVersion FROM \
productionscontainer cont, configurations c, stepscontainer scont, steps s WHERE cont.production=%s AND \
cont.configurationid=c.configurationid AND cont.production=scont.production AND scont.stepid=s.stepid \
GROUP BY c.configname, c.configversion, s.ApplicationName, s.ApplicationVersion"
            % prodid
        )

        res = self.dbR_.query(command)
        if not res["OK"]:
            return res
        record = res["Value"]
        cname = record[0][0]
        cversion = record[0][1]
        pname = record[0][2]
        pversion = record[0][3]

        retVal = self.getProductionProcessingPass(prodid)
        if not retVal["OK"]:
            return retVal
        procdescription = retVal["Value"]

        simdesc = None
        daqdesc = None

        command = (
            "SELECT DISTINCT sim.simdescription, daq.description FROM simulationconditions sim, "
            "data_taking_conditions daq, productionscontainer prod WHERE "
            "sim.simid(+)=prod.simid AND daq.daqperiodid(+)=prod.daqperiodid AND prod.production=" + str(prodid)
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        value = retVal["Value"]
        if value:
            simdesc = value[0][0]
            daqdesc = value[0][1]
        else:
            return S_ERROR("Simulation condition or data taking condition not exist!")
        if simdesc is not None:
            return S_OK(
                {
                    "ConfigName": cname,
                    "ConfigVersion": cversion,
                    "ProgramName": pname,
                    "ProgramVersion": pversion,
                    "Processing pass": procdescription,
                    "Simulation conditions": simdesc,
                }
            )
        else:
            return S_OK(
                {
                    "ConfigName": cname,
                    "ConfigVersion": cversion,
                    "ProgramName": pname,
                    "ProgramVersion": pversion,
                    "Processing pass": procdescription,
                    "Data taking conditions": daqdesc,
                }
            )

    #############################################################################
    def getJobInfo(self, lfn):
        """For retrieving the job parameters for a given LFN.

        :param str lfn: logical file name
        :return: Job information for a given file
        """
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getJobInfo", [lfn])

    #############################################################################
    def bulkJobInfo(self, in_dict):
        """For retrieving jobs parameters for a list of LFNs, jobIds, or JobName.

        :param dict in_dict: dictionary which contains lfn, jobId or JobName elements
        :return: the job information for a list of files
        """

        data = []
        if "lfn" in in_dict:
            data = in_dict["lfn"]
            if not data:
                return S_ERROR("Please give at least one lfn")
            retVal = self.dbR_.executeStoredProcedure(
                packageName="BOOKKEEPINGORACLEDB.bulkJobInfo", parameters=[], output=True, array=data
            )
        elif "jobId" in in_dict:
            data = in_dict["jobId"]
            if not data:
                return S_ERROR("Please give at least one jobId")
            retVal = self.dbR_.executeStoredProcedure(
                packageName="BOOKKEEPINGORACLEDB.bulkJobInfoForJobId", parameters=[], output=True, array=data
            )

        elif "jobName" in in_dict:
            data = in_dict["jobName"]
            if not data:
                return S_ERROR("Please give at least one jobName")
            retVal = self.dbR_.executeStoredProcedure(
                packageName="BOOKKEEPINGORACLEDB.bulkJobInfoForJobName", parameters=[], output=True, array=data
            )
        else:
            return S_ERROR(
                "Wrong input parameters. You can use a dictionary with the following keys: lfn,jobId, jobName"
            )

        records = {}
        if retVal["OK"]:
            for i in retVal["Value"]:
                record = dict(
                    zip(
                        (
                            "DIRACJobId",
                            "DIRACVersion",
                            "EventInputStat",
                            "ExecTime",
                            "FirstEventNumber",
                            "Location",
                            "Name",
                            "NumberOfEvents",
                            "StatisticsRequested",
                            "WNCPUPOWER",
                            "CPUTIME",
                            "WNCACHE",
                            "WNMEMORY",
                            "WNMODEL",
                            "WORKERNODE",
                            "WNCPUHS06",
                            "JobId",
                            "TotalLumonosity",
                            "Production",
                            "ApplicationName",
                            "ApplicationVersion",
                            "WNMJFHS06",
                        ),
                        i[1:],
                    )
                )
                j = 0
                if i[0] not in records:
                    records[i[0]] = [record]
                else:
                    records[i[0]] += [record]

                j += 1

            failed = [i for i in data if i not in records]
            result = S_OK({"Successful": records, "Failed": failed})
        else:
            result = retVal

        return result

    #############################################################################
    def getJobInformation(self, params):
        """For retrieving only job information for a given production, lfn or
        DiracJobId.

        :param dict params: dictionary which contains LFN, Production, DiracJobId elements
        :return: job parameters
        """
        production = params.get("Production", default)
        lfn = params.get("LFN", default)
        condition = ""
        diracJobids = params.get("DiracJobId", default)

        tables = " jobs j, files f, configurations c"
        result = None
        if production != default:
            if isinstance(production, (str, int)):
                condition += " AND j.production=%d " % (int(production))
            elif isinstance(production, list):
                condition += " AND j.production in (" + ",".join([str(p) for p in production]) + ")"
            else:
                result = S_ERROR("The production type is invalid. It can be a list, integer or string!")
        elif lfn != default:
            if isinstance(lfn, str):
                condition += f" AND f.filename='{lfn}' "
            elif isinstance(lfn, list):
                condition += " AND (" + " or ".join([f"f.filename='{x}'" for x in lfn]) + ")"
            else:
                result = S_ERROR("You must provide an LFN or a list of LFNs!")
        elif diracJobids != default:
            if isinstance(diracJobids, (str, int)):
                condition += f" AND j.DIRACJOBID={diracJobids} "
            elif isinstance(diracJobids, list):
                condition += " AND j.DIRACJOBID in (" + ",".join([str(djobid) for djobid in diracJobids]) + ")"
            else:
                result = S_ERROR("Please provide a correct DIRAC jobid!")

        if not result:
            command = (
                "SELECT  DISTINCT j.DIRACJOBID, j.DIRACVERSION, j.EVENTINPUTSTAT, j.EXECTIME, "
                "j.FIRSTEVENTNUMBER,j.LOCATION, j.NAME, j.NUMBEROFEVENTS, j.STATISTICSREQUESTED, "
                "j.WNCPUPOWER, j.CPUTIME, j.WNCACHE, j.WNMEMORY, j.WNMODEL, "
                "j.WORKERNODE, j.WNCPUHS06, j.jobid, j.totalluminosity, j.production, j.WNMJFHS06, "
                "c.ConfigName,c.ConfigVersion, j.JobEnd, j.JobStart, j.RunNumber, j.FillNumber, j.Tck, j.stepid "
                "FROM %s WHERE f.jobid=j.jobid AND c.configurationid=j.configurationid %s" % (tables, condition)
            )
            retVal = self.dbR_.query(command)
            if retVal["OK"]:
                records = []

                parameters = [
                    "DiracJobId",
                    "DiracVersion",
                    "EventInputStat",
                    "Exectime",
                    "FirstEventNumber",
                    "Location",
                    "JobName",
                    "NumberOfEvents",
                    "StatisticsRequested",
                    "WNCPUPower",
                    "CPUTime",
                    "WNCache",
                    "WNMemory",
                    "WNModel",
                    "WorkerNode",
                    "WNCPUHS06",
                    "JobId",
                    "TotalLuminosity",
                    "Production",
                    "WNMJFHS06",
                    "ConfigName",
                    "ConfigVersion",
                    "JobEnd",
                    "JobStart",
                    "RunNumber",
                    "FillNumber",
                    "Tck",
                    "StepId",
                ]
                for i in retVal["Value"]:
                    records += [dict(zip(parameters, i))]
                result = S_OK(records)
            else:
                result = retVal

        return result

    #############################################################################
    def getRunNumber(self, lfn):
        """For retrieving the run number for a given LFN.

        :param str lfn: logical file name
        :return: the run number of a given file
        """
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getRunNumber", int, [lfn])

    #############################################################################
    def getRunNbAndTck(self, lfn):
        """For retrieving the run number and TCK for a given LFN.

        :param str lfn: logical file name
        :return: the run number and tck for a given file
        """
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getRunNbAndTck", [lfn])

    #############################################################################
    def getProductionFiles(self, prod, ftype, gotreplica=default):
        """For retrieving the list of LFNs for a given production.

        :param int prod: production number
        :param str ftype: file type
        :param str gotreplica: replica flag
        :return: the files which are belongs to a given production
        """
        result = S_ERROR()
        value = {}
        condition = ""
        if gotreplica != default:
            condition += f" AND files.gotreplica='{str(gotreplica)}'"

        if ftype != default:
            condition += f" AND filetypes.name='{ftype}'"

        command = (
            "SELECT files.filename, files.gotreplica, files.filesize,files.guid, filetypes.name, "
            "files.inserttimestamp, files.visibilityflag "
            "FROM jobs,files,filetypes WHERE "
            "jobs.jobid=files.jobid AND files.filetypeid=filetypes.filetypeid AND jobs.production=%d %s"
            % (prod, condition)
        )

        res = self.dbR_.query(command)
        if res["OK"]:
            dbResult = res["Value"]
            for record in dbResult:
                value[record[0]] = {
                    "GotReplica": record[1],
                    "FileSize": record[2],
                    "GUID": record[3],
                    "FileType": record[4],
                    "Visible": record[6],
                }
            result = S_OK(value)
        else:
            result = S_ERROR(res["Message"])
        return result

    #############################################################################
    def getProductionFilesBulk(self, prods, ftype, gotreplica=default):
        """For retrieving the list of LFNs for a given production.

        :param list prods: production numbers
        :param str ftype: file type
        :param str gotreplica: replica flag
        :return: dictionary of production number to the list of files
        """
        command = (
            "select "
            "    jobs.production, "
            "    files.filename, "
            "    files.gotreplica, "
            "    files.filesize, "
            "    files.guid, "
            "    filetypes.name, "
            "    files.visibilityflag "
            "from "
            "    jobs "
            "    INNER JOIN files on files.jobid = jobs.jobid "
            "    INNER JOIN filetypes on filetypes.filetypeid = files.filetypeid "
            "where "
            "    jobs.production in (%s)"
        )
        kwparams = {}
        if gotreplica != default:
            kwparams["gotreplica"] = str(gotreplica)
            command += " and files.gotreplica = :gotreplica"
        if ftype != default:
            kwparams["ftype"] = ftype
            command += " and filetypes.name = :ftype"

        value = {str(p): {} for p in prods}
        for prodChunk in breakListIntoChunks(prods, 100):
            res = self.dbR_.query(
                # Ensure each element is a valid int to avoid SQL injection attacks
                command % ",".join(str(int(p)) for p in prodChunk),
                kwparams=kwparams,
            )
            if not res["OK"]:
                return S_ERROR(res["Message"])
            for production, filename, gotreplica, filesize, guid, filetype, visibilityflag in res["Value"]:
                value[str(production)][filename] = {
                    "GotReplica": gotreplica,
                    "FileSize": filesize,
                    "GUID": guid,
                    "FileType": filetype,
                    "Visible": visibilityflag,
                }
        return S_OK(value)

    #############################################################################
    def getRunFiles(self, runid):
        """Retrieving list of LFNs for a given run.

        :param int runid: run number
        :return: a list of files with metadata for a given run
        """
        result = S_ERROR()
        value = {}
        res = self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getRunFiles", [runid])
        if res["OK"]:
            dbResult = res["Value"]
            for record in dbResult:
                value[record[0]] = {
                    "GotReplica": record[1],
                    "FileSize": record[2],
                    "GUID": record[3],
                    "Luminosity": record[4],
                    "InstLuminosity": record[5],
                    "EventStat": record[6],
                    "FullStat": record[7],
                }
            result = S_OK(value)
        else:
            result = res
        return result

    #############################################################################
    def updateFileMetaData(self, filename, fileAttr):
        """updates the file metadata.

        :param str filename:
        :param dict fileAttr: file attributes
        """
        utctime = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
        command = f"UPDATE files Set inserttimestamp=TO_TIMESTAMP('{str(utctime)}','YYYY-MM-DD HH24:MI:SS') ,"
        command += ",".join([f"{str(attribute)}={str(fileAttr[attribute])}" for attribute in fileAttr])
        command += f" WHERE fileName='{filename}'"
        res = self.dbW_.query(command)
        return res

    #############################################################################
    def bulkupdateFileMetaData(self, lfnswithmeta):
        """For updating the metadata a list of files:

        :param dict lfnswithmetadata: dictionary which contains LFNs and file attributes.
        """

        utctime = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
        sqls = []
        for filename in lfnswithmeta:
            command = f"UPDATE files Set inserttimestamp=TO_TIMESTAMP('{str(utctime)}','YYYY-MM-DD HH24:MI:SS'), "
            command += ",".join(
                [f"{str(attribute)}={str(lfnswithmeta[filename][attribute])}" for attribute in lfnswithmeta[filename]]
            )
            command += f" WHERE fileName='{filename}'"
            sqls += [command]

        retVal = self.dbR_.executeStoredProcedure(
            packageName="BOOKKEEPINGORACLEDB.bulkupdateFileMetaData", parameters=[], output=False, array=sqls
        )
        return retVal

    #############################################################################
    def renameFile(self, oldLFN, newLFN):
        """renames a file.

        :param str oldLFN: old logical file name
        :param str newLFN: new logical file name
        """
        utctime = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
        command = (
            " UPDATE files SET inserttimestamp=TO_TIMESTAMP('%s','YYYY-MM-DD HH24:MI:SS'), filename ='%s' WHERE filename='%s'"
            % (str(utctime), newLFN, oldLFN)
        )
        res = self.dbW_.query(command)
        return res

    #############################################################################
    def getInputFiles(self, jobid):
        """For retrieving the input files for a given job.

        :param int jobid: bookkeeping job id
        :return: the input files for a given jobid
        """
        command = (
            " SELECT files.filename FROM inputfiles,files WHERE files.fileid=inputfiles.fileid AND inputfiles.jobid="
            + str(jobid)
        )
        return self.dbR_.query(command)

    #############################################################################
    def getOutputFiles(self, jobid):
        """For retrieving the output files for a given job.

        :param int jobid: bookkeeping jobid
        :return: the outputfiles for a given jobid
        """
        return self.dbR_.query("SELECT files.filename FROM files WHERE files.jobid=" + str(jobid))

    #############################################################################
    def insertTag(self, name, tag):
        """inserts the CONDD,DDDB tags to the database.

        :param str name: tag name: CONDDB, DDDB, etc.
        :param str tag: CONDDB, DDDB tag
        """
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.insertTag", [name, tag], False)

    #############################################################################
    def existsTag(self, name, value):  # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
        """checks the tag existence in the database.

        :param str name: tag name: CONDDB, DDDB, etc.
        :param str value: CONDDB, DDDB, etc. tag
        """
        result = False
        command = f"SELECT COUNT(*) FROM tags WHERE name='{str(name)}' AND tag='{str(value)}'"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            result = retVal
        elif retVal["Value"][0][0] > 0:
            result = True
        return S_OK(result)

    #############################################################################
    def setFileDataQuality(self, lfns, flag):
        """sets the data quality for a list of lfns.

        :param list lfns: list of LFNs
        :param str flag: data quality flag
        """
        retVal = self.__getDataQualityId(flag)
        if not retVal["OK"]:
            return retVal
        qid = retVal["Value"]
        failed = []
        succ = []
        retVal = self.dbR_.executeStoredProcedure(
            packageName="BOOKKEEPINGORACLEDB.updateDataQualityFlag", parameters=[qid], output=False, array=lfns
        )
        if not retVal["OK"]:
            failed = lfns
            self.log.error(retVal["Message"])
        else:
            succ = lfns
        return S_OK({"Successful": succ, "Failed": failed})

    #############################################################################
    def __getProcessingPassId(self, root, fullpath):
        """For retrieving processing pass id.

        :param str root: root path for example /Real Data
        :param str fullpath: full processing pass for exampe: /Real Data/Reco19/Stripping20
        :return: the processing pass id
        """
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getProcessingPassId", int, [root, fullpath])

    #############################################################################
    def getProcessingPassId(self, fullpath):
        """For retrieving processing pass id.

        :param str fullpath: processing pass for example: /Real Data/Reco19
        :return: the processing pass identifier for a given path
        """
        return self.__getProcessingPassId(fullpath.split("/")[1:][0], fullpath)

    #############################################################################
    def __getDataQualityId(self, name):
        """For retrieving data quality id.

        :param str name: data quality for example OK, BAD, etc.
        :return: data quality id
        """
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getDataQualityId", int, [name])

    #############################################################################
    def setRunAndProcessingPassDataQuality(self, runNB, procpass, flag):
        """set the data quality of a run which belongs to a given processing pass.

        :param int runNB: run number
        :param str procpass: processing pass
        :param str flag: data quality flag
        """
        retVal = self.__getProcessingPassId(procpass.split("/")[1:][0], procpass)
        if not retVal["OK"]:
            self.log.error("Could not get a processing pass ID", retVal["Message"])
            return retVal
        processingid = retVal["Value"]

        retVal = self.__getDataQualityId(flag)
        if not retVal["OK"]:
            self.log.error("Could not get a data quality ID", retVal["Message"])
            return retVal
        flag = retVal["Value"]

        return self.dbW_.executeStoredProcedure(
            "BOOKKEEPINGORACLEDB.insertRunquality", [runNB, flag, processingid], False
        )

    #############################################################################
    def setRunDataQuality(self, runNb, flag):
        """sets the data quality flag for a given run.

        :param int runNb: run number
        :param flag: data quality flag
        """
        result = S_ERROR()
        command = (
            "SELECT DISTINCT j.runnumber FROM  jobs j, productionscontainer prod WHERE \
j.production=prod.production AND j.production<0 AND j.runnumber=%s"
            % (str(runNb))
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            result = retVal
        else:
            if not retVal["Value"]:
                result = S_ERROR("This " + str(runNb) + " run is missing in the BKK DB!")
            else:
                retVal = self.__getDataQualityId(flag)

                if not retVal["OK"]:
                    result = retVal
                else:
                    qid = retVal["Value"]
                    utctime = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
                    command = (
                        "UPDATE files set inserttimestamp=TO_TIMESTAMP('%s','YYYY-MM-DD HH24:MI:SS'), \
qualityId=%d WHERE fileid in (SELECT files.fileid from jobs, files where jobs.jobid=files.jobid AND jobs.runnumber=%d)"
                        % (str(utctime), qid, runNb)
                    )
                    retVal = self.dbW_.query(command)

                    if not retVal["OK"]:
                        result = retVal
                    else:
                        command = (
                            "SELECT files.filename FROM jobs, files where jobs.jobid=files.jobid AND jobs.runnumber=%s"
                            % (runNb)
                        )

                        retVal = self.dbR_.query(command)

                        if not retVal["OK"]:
                            result = retVal
                        else:
                            succ = []
                            records = retVal["Value"]
                            for record in records:
                                succ += [record[0]]
                            values = {}
                            values["Successful"] = succ
                            values["Failed"] = []
                            result = S_OK(values)
        return result

    #############################################################################
    def setProductionDataQuality(self, prod, flag):
        """sets the data quality to a production.

        :param int prod: production number
        :param str flag: data quality flag
        """
        result = S_ERROR()
        command = "SELECT DISTINCT jobs.production FROM jobs WHERE jobs.production=%d" % (prod)
        retVal = self.dbR_.query(command)

        if not retVal["OK"]:
            result = retVal
        else:
            if not retVal["Value"]:
                result = S_ERROR("This " + str(prod) + " production is missing in the BKK DB!")
            else:
                retVal = self.__getDataQualityId(flag)

                if not retVal["OK"]:
                    result = retVal
                else:
                    qid = retVal["Value"]
                    utctime = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
                    command = (
                        "UPDATE files SET inserttimestamp=TO_TIMESTAMP('%s' + str(utctime),'YYYY-MM-DD HH24:MI:SS'), \
qualityId=%d WHERE fileid IN (SELECT files.fileid FROM jobs, files WHERE jobs.jobid=files.jobid AND jobs.production=%d)"
                        % (str(utctime), qid, prod)
                    )
                    retVal = self.dbW_.query(command)

                    if not retVal["OK"]:
                        result = retVal
                    else:
                        command = (
                            "SELECT files.filename FROM jobs, files WHERE jobs.jobid=files.jobid AND jobs.production=%d"
                            % (prod)
                        )
                        retVal = self.dbR_.query(command)

                        if not retVal["OK"]:
                            result = retVal
                        else:
                            succ = []
                            records = retVal["Value"]
                            for record in records:
                                succ += [record[0]]
                            values = {}
                            values["Successful"] = succ
                            values["Failed"] = []
                            result = S_OK(values)
        return result

    #############################################################################
    def getFileAncestorHelper(self, fileName, files, depth, checkreplica):
        """Recursively retrieve the ancestors for a given file.

        :param str fileName: actual file name
        :param list files: the ancestor files list
        :param int depth: the depth of the processing pass chain(how far to go)
        :param bool checkreplica: take into account the replica flag
        :return: the ancestor of a file
        """
        failed = []

        if depth:
            depth -= 1
            result = self.dbR_.executeStoredFunctions(
                "BOOKKEEPINGORACLEDB.getJobIdWithoutReplicaCheck", int, [fileName]
            )

            if not result["OK"]:
                self.log.error("Error getting jobID", result["Message"])
            jobID = int(result.get("Value", 0))
            if jobID:
                command = (
                    "SELECT files.fileName,files.jobid, files.gotreplica, files.eventstat, \
files.eventtypeid, files.luminosity, files.instLuminosity, filetypes.name \
FROM inputfiles,files, filetypes WHERE files.filetypeid=filetypes.filetypeid \
AND inputfiles.fileid=files.fileid AND inputfiles.jobid=%d"
                    % (jobID)
                )
                res = self.dbR_.query(command)
                if not res["OK"]:
                    self.log.error("Error getting job input files", result["Message"])
                else:
                    dbResult = res["Value"]
                    for record in dbResult:
                        if not checkreplica or (record[2] != "No"):
                            files.append(
                                {
                                    "FileName": record[0],
                                    "GotReplica": record[2],
                                    "EventStat": record[3],
                                    "EventType": record[4],
                                    "Luminosity": record[5],
                                    "InstLuminosity": record[6],
                                    "FileType": record[7],
                                }
                            )
                        if depth:
                            failed += self.getFileAncestorHelper(record[0], files, depth, checkreplica)
            else:
                failed.append(fileName)
        return failed

    #############################################################################
    def getFileAncestors(self, lfn, depth=0, replica=True):
        """ " iterates on the list of lfns and prepare the ancestor list using a
        recursive helper function.

        :param list lfn:
        :param int depth: the depth of the processing pass chain(how far to go)
        :param bool replica: take into account the replica flag
        """
        depth = min(10, max(1, depth))

        logicalFileNames = {"Failed": []}
        ancestorList = {}
        filesWithMetadata = {}
        self.log.debug("original", f"{lfn}")
        failed = []
        for fileName in lfn:
            files = []
            failed += self.getFileAncestorHelper(fileName, files, depth, replica)
            logicalFileNames["Failed"] = failed
            if files:
                ancestorList[fileName] = files
                tmpfiles = {}
                for i in files:
                    tmpattr = dict(i)
                    tmpfiles[tmpattr.pop("FileName")] = tmpattr
                filesWithMetadata[fileName] = tmpfiles
        logicalFileNames["Successful"] = ancestorList
        logicalFileNames["WithMetadata"] = filesWithMetadata
        return S_OK(logicalFileNames)

    #############################################################################
    def getFileDescendentsHelper(self, fileName, files, depth, production, checkreplica, productionFound=False):
        """Helper function for retrieving the file descendents.

        :param str fileName: actual file name
        :param list files: the descendents file list (producced files)
        :param int depth: the depth of the processing pass chain(how far to go)
        :param productin: production number
        :param bool checkreplica: take into account the replica flag
        :param bool productionFound: It breaks the check if the production found
          but we but we no longer are in it
        :return: the descendents of a file
        """
        failed = set()
        notprocessed = set()

        if depth:
            depth -= 1

            res = self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getFileDesJobId", [fileName])
            if not res["OK"]:
                self.log.error("Error getting fileId", res["Message"])
                failed.add(fileName)
            elif not res["Value"]:
                notprocessed.add(fileName)
            else:
                for jobID in [item[0] for item in res["Value"]]:
                    getProd = bool(production)

                    res = self.dbW_.executeStoredProcedure(
                        "BOOKKEEPINGORACLEDB.getFileAndJobMetadata", [jobID, getProd]
                    )
                    if not res["OK"]:
                        self.log.error("Error getting job output files", res["Message"])
                        failed.add(fileName)
                    elif not res["Value"]:
                        notprocessed.add(fileName)
                    else:
                        for record in res["Value"]:
                            inRequestedProd = getProd and int(record[3]) == int(production)
                            if not productionFound or inRequestedProd:
                                # If we have already found the production but we no longer are in it, break the recursive loop
                                if (not checkreplica or (record[2] != "No")) and (not getProd or inRequestedProd):
                                    files[record[0]] = {
                                        "GotReplica": record[2],
                                        "EventStat": record[4],
                                        "EventType": record[5],
                                        "Luminosity": record[6],
                                        "InstLuminosity": record[7],
                                        "FileType": record[8],
                                    }

                                if depth:
                                    # Only call if we are not at the correct depth
                                    newFailed, _newNotprocessed = self.getFileDescendentsHelper(
                                        record[0],
                                        files,
                                        depth,
                                        production,
                                        checkreplica,
                                        productionFound=inRequestedProd,
                                    )
                                    failed.update(newFailed)

        return sorted(failed), sorted(notprocessed)

    #############################################################################
    def getFileDescendents(self, lfn, depth=0, production=0, checkreplica=True):
        """iterates over a list of lfns and collects their descendents.

        :param str fileName: actual file name
        :param list files: the descendents file list (producced files)
        :param int depth: the depth of the processing pass chain(how far to go)
        :param productin: production number
        :param bool checkreplica: take into account the replica flag
        :returns: all descendents
        """
        logicalFileNames = {"Failed": [], "NotProcessed": []}
        ancestorList = {}
        filesWithMetadata = {}

        depth = min(10, max(1, depth))

        for fileName in lfn:
            files = {}
            failed, notprocessed = self.getFileDescendentsHelper(fileName, files, depth, production, checkreplica)
            logicalFileNames["Failed"] += failed
            logicalFileNames["NotProcessed"] += notprocessed
            if files:
                ancestorList[fileName] = list(files)
                filesWithMetadata[fileName] = files
        logicalFileNames["Successful"] = ancestorList
        logicalFileNames["WithMetadata"] = filesWithMetadata
        return S_OK(logicalFileNames)

    #############################################################################
    def checkfile(self, fileName):  # file
        """checks the status of a file.

        :param str fileName: logical file name
        :return: fileId, jobId, filetypeid
        """
        result = self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.checkfile", [fileName])
        if not result["OK"]:
            return result

        res = result["Value"]
        if res:
            return S_OK(res)
        self.log.warn("File not found! ", f"{fileName}")
        return S_ERROR(f"File not found: {fileName}")

    #############################################################################
    def checkFileTypeAndVersion(self, filetype, version):  # fileTypeAndFileTypeVersion(self, type, version):
        """checks the the format and the version.

        :param str filetype: file type
        :param str version: file type version
        :return: file type id
        """
        return self.dbR_.executeStoredFunctions("BOOKKEEPINGORACLEDB.checkFileTypeAndVersion", int, [filetype, version])

    #############################################################################
    def checkEventType(self, eventTypeId):  # eventType(self, eventTypeId):
        """checks the event type.

        :param int eventTypeId: event type
        :return: event type
        """
        result = S_ERROR()

        retVal = self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.checkEventType", [eventTypeId])
        if retVal["OK"]:
            value = retVal["Value"]
            if value:
                result = S_OK(value)
            else:
                self.log.info("Event type not found:", f"{eventTypeId}")
                result = S_ERROR(f"Event type not found: {eventTypeId}")
        else:
            result = retVal
        return result

    #############################################################################
    def insertJob(self, job):
        """inserts a job to the database.

        :param dict job: job attributes
        :returns: jobId
        """
        self.log.debug("Insert job into database!")
        attrList = {
            "ConfigName": None,
            "ConfigVersion": None,
            "DiracJobId": None,
            "DiracVersion": None,
            "EventInputStat": None,
            "ExecTime": None,
            "FirstEventNumber": None,
            "JobEnd": None,
            "JobStart": None,
            "Location": None,
            "Name": None,
            "NumberOfEvents": None,
            "Production": None,
            "ProgramName": None,
            "ProgramVersion": None,
            "StatisticsRequested": None,
            "WNCPUPOWER": None,
            "CPUTIME": None,
            "WNCACHE": None,
            "WNMEMORY": None,
            "WNMODEL": None,
            "WorkerNode": None,
            "RunNumber": None,
            "FillNumber": None,
            "WNCPUHS06": 0,
            "TotalLuminosity": 0,
            "Tck": "None",
            "StepID": None,
            "WNMJFHS06": 0,
            "HLT2Tck": "None",
            "NumberOfProcessors": 1,
        }

        for param in job:
            if not attrList.__contains__(param):
                self.log.error("insert job error: ", f" the job table not contain attribute {param}")
                return S_ERROR(f" The job table not contain attribute {param}")

            if param == "JobStart" or param == "JobEnd":  # We have to convert data format
                dateAndTime = job[param].split(" ")
                date = dateAndTime[0].split("-")
                time = dateAndTime[1].split(":")
                if len(time) > 2:
                    timestamp = datetime.datetime(
                        int(date[0]), int(date[1]), int(date[2]), int(time[0]), int(time[1]), int(time[2]), 0
                    )
                else:
                    timestamp = datetime.datetime(
                        int(date[0]), int(date[1]), int(date[2]), int(time[0]), int(time[1]), 0, 0
                    )
                attrList[param] = timestamp
            else:
                attrList[param] = job[param]

        try:
            conv = int(attrList["Tck"])
            attrList["Tck"] = str(hex(conv))
        except ValueError:
            pass  # it is already defined

        result = self.dbW_.executeStoredFunctions(
            "BOOKKEEPINGORACLEDB.insertJobsRow",
            int,
            [
                attrList["ConfigName"],
                attrList["ConfigVersion"],
                attrList["DiracJobId"],
                attrList["DiracVersion"],
                attrList["EventInputStat"],
                attrList["ExecTime"],
                attrList["FirstEventNumber"],
                attrList["JobEnd"],
                attrList["JobStart"],
                attrList["Location"],
                attrList["Name"],
                attrList["NumberOfEvents"],
                attrList["Production"],
                attrList["ProgramName"],
                attrList["ProgramVersion"],
                attrList["StatisticsRequested"],
                attrList["WNCPUPOWER"],
                attrList["CPUTIME"],
                attrList["WNCACHE"],
                attrList["WNMEMORY"],
                attrList["WNMODEL"],
                attrList["WorkerNode"],
                attrList["RunNumber"],
                attrList["FillNumber"],
                attrList["WNCPUHS06"],
                attrList["TotalLuminosity"],
                attrList["Tck"],
                attrList["StepID"],
                attrList["WNMJFHS06"],
                attrList["HLT2Tck"],
                attrList["NumberOfProcessors"],
            ],
        )
        return result

    #############################################################################
    def insertInputFile(self, jobID, fileId):
        """inserts the input file of a job.

        :param int jobID: internal bookkeeping job id
        :param int fileId: internal file id
        """
        result = self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.insertInputFilesRow", [fileId, jobID], False)
        return result

    #############################################################################
    def insertOutputFile(self, fileobject):
        """inserts an output file.

        :param dict fileobject: file attributes
        :returns: fileid
        """
        attrList = {
            "Adler32": None,
            "CreationDate": None,
            "EventStat": None,
            "EventTypeId": None,
            "FileName": None,
            "FileTypeId": None,
            "GotReplica": None,
            "Guid": None,
            "JobId": None,
            "MD5Sum": None,
            "FileSize": 0,
            "FullStat": None,
            "QualityId": "UNCHECKED",
            "Luminosity": 0,
            "InstLuminosity": 0,
            "VisibilityFlag": "Y",
        }

        for param in fileobject:
            if param not in attrList:
                self.log.error("insert file error: ", f" the files table not contain attribute {param} ")
                return S_ERROR(f" The files table not contain attribute {param}")

            if param == "CreationDate":  # We have to convert data format
                dateAndTime = fileobject[param].split(" ")
                date = dateAndTime[0].split("-")
                time = dateAndTime[1].split(":")
                timestamp = datetime.datetime(
                    int(date[0]), int(date[1]), int(date[2]), int(time[0]), int(time[1]), 0, 0
                )
                attrList[param] = timestamp
            else:
                attrList[param] = fileobject[param]
        utctime = datetime.datetime.utcnow()

        return self.dbW_.executeStoredFunctions(
            "BOOKKEEPINGORACLEDB.insertFilesRow",
            int,
            [
                attrList["Adler32"],
                attrList["CreationDate"],
                attrList["EventStat"],
                attrList["EventTypeId"],
                attrList["FileName"],
                attrList["FileTypeId"],
                attrList["GotReplica"],
                attrList["Guid"],
                attrList["JobId"],
                attrList["MD5Sum"],
                attrList["FileSize"],
                attrList["FullStat"],
                utctime,
                attrList["QualityId"],
                attrList["Luminosity"],
                attrList["InstLuminosity"],
                attrList["VisibilityFlag"],
            ],
        )

    #############################################################################
    def updateReplicaRow(self, fileID, replica):  # , name, location):
        """adds the replica flag.

        :param int fileID: internal bookkeeping file id
        :param str replica: replica flag
        """
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.updateReplicaRow", [fileID, replica], False)

    #############################################################################
    def deleteJob(self, jobID):
        """deletes a job.

        :param int jobID: internal bookkeeping job id
        """
        self.log.warn("Deleting job", jobID)
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.deleteJob", [jobID], False)

    #############################################################################
    def deleteInputFiles(self, jobID):
        """deletes the input files of a job.

        :param int jobid: internal bookkeeping job id
        """
        self.log.warn("Deleting input files of", jobID)
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.deleteInputFiles", [jobID], False)

    #############################################################################
    def deleteFile(self, fileID):
        """deletes a file.

        :param int fileid: internal bookkeeping file id
        """
        self.log.warn("Deleting file", fileID)
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.deletefile", [fileID], False)

    #############################################################################
    @staticmethod
    def deleteFiles(lfns):
        """For having the same interface as other catalogs. We do not delete file
        from the db.

        :param list lfns: list of lfns
        """
        return S_ERROR("Not Implemented !!" + lfns)

    #############################################################################
    def insertSimConditions(self, in_dict):
        """inserts a simulation conditions.

        :param dict in_dict: simulation condition attributes
        :return: simid
        """

        simdesc = in_dict.get("SimDescription", None)
        beamCond = in_dict.get("BeamCond", None)
        beamEnergy = in_dict.get("BeamEnergy", None)
        generator = in_dict.get("Generator", None)
        magneticField = in_dict.get("MagneticField", None)
        detectorCond = in_dict.get("DetectorCond", None)
        luminosity = in_dict.get("Luminosity", None)
        g4settings = in_dict.get("G4settings", None)
        visible = in_dict.get("Visible", "Y")
        return self.dbW_.executeStoredFunctions(
            "BOOKKEEPINGORACLEDB.insertSimConditions",
            int,
            [simdesc, beamCond, beamEnergy, generator, magneticField, detectorCond, luminosity, g4settings, visible],
        )

    #############################################################################
    def getSimConditions(self):
        """For retrieving the simulation conditions.

        :rerturn: the available simulation conditions
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getSimConditions", [])

    #############################################################################
    def insertDataTakingCond(self, conditions):
        """inserts a data taking condition:

        :param dict conditions: data taking conditions attributes.
        :returns: data quality id
        """
        datataking = {
            "Description": None,
            "BeamCond": None,
            "BeamEnergy": None,
            "MagneticField": None,
            "VELO": None,
            "IT": None,
            "TT": None,
            "OT": None,
            "RICH1": None,
            "RICH2": None,
            "SPD_PRS": None,
            "ECAL": None,
            "HCAL": None,
            "MUON": None,
            "L0": None,
            "HLT": None,
            "VeloPosition": None,
        }

        for param in conditions:
            if not datataking.__contains__(param):
                self.log.error("Can not insert data taking condition the files table not contains:", f"{param}")
                return S_ERROR(f"Can not insert data taking condition the files table not contains: {param} ")
            datataking[param] = conditions[param]

        res = self.dbW_.executeStoredFunctions(
            "BOOKKEEPINGORACLEDB.insertDataTakingCond",
            int,
            [
                datataking["Description"],
                datataking["BeamCond"],
                datataking["BeamEnergy"],
                datataking["MagneticField"],
                datataking["VELO"],
                datataking["IT"],
                datataking["TT"],
                datataking["OT"],
                datataking["RICH1"],
                datataking["RICH2"],
                datataking["SPD_PRS"],
                datataking["ECAL"],
                datataking["HCAL"],
                datataking["MUON"],
                datataking["L0"],
                datataking["HLT"],
                datataking["VeloPosition"],
            ],
        )
        return res

    def insertDataTakingCondDesc(self, dtDescription: str):
        """inserts a data taking condition just from the description string

        It reuses the existing oracle function until we decide what to do
        with this table.

        :param dtDescription: data taking conditions description
        :returns: data quality id
        """

        res = self.dbW_.executeStoredFunctions(
            "BOOKKEEPINGORACLEDB.insertDataTakingCond",
            int,
            [
                dtDescription,
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
                "None",
            ],
        )
        return res

    def deleteDataTakingCondition(self, dtcId):
        """it deletes a given data taking condition.

        :param int dtcId: Data taking condition ID
        """
        return self.dbW_.query("delete data_taking_conditions where DaqPeriodId=:dtcId", kwparams={"dtcId": dtcId})

    #############################################################################
    def removeReplica(self, fileNames):
        """removes the replica flag of a file.

        :param list fileNames: list LFNs
        :return: successfully deleted and failed to delete LFNs
        """
        result = S_ERROR()
        retVal = self.dbR_.executeStoredProcedure(
            packageName="BOOKKEEPINGORACLEDB.bulkcheckfiles", parameters=[], output=True, array=fileNames
        )
        failed = {}

        if not retVal["OK"]:
            result = retVal
        else:
            for i in retVal["Value"]:
                failed[i[0]] = f"The file {i[0]} does not exist in the BKK database!!!"
                fileNames.remove(i[0])
            if fileNames:
                retVal = self.dbW_.executeStoredProcedure(
                    packageName="BOOKKEEPINGORACLEDB.bulkupdateReplicaRow",
                    parameters=["No"],
                    output=False,
                    array=fileNames,
                )
                if not retVal["OK"]:
                    result = retVal
                else:
                    failed["Failed"] = list(failed)
                    failed["Successful"] = fileNames
                    result = S_OK(failed)
            else:  # when no files are exists
                files = {"Failed": [i[0] for i in retVal["Value"]], "Successful": []}
                result = S_OK(files)
        return result

    #############################################################################
    def getFileMetadata(self, lfns):
        """returns the metadata of a list of files.

        :param list lfns: list of LFNs
        :return: successful lfns with associated meta data and failed lfns.
        """
        result = {}

        for lfnList in breakListIntoChunks(lfns, 5000):
            retVal = self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getFileMetaData3", [], True, lfnList)
            if not retVal["OK"]:
                result = retVal
            else:
                for record in retVal["Value"]:
                    row = {
                        "ADLER32": record[1],
                        "CreationDate": record[2],
                        "EventStat": record[3],
                        "FullStat": record[10],
                        "EventType": record[4],
                        "FileType": record[5],
                        "GotReplica": record[6],
                        "GUID": record[7],
                        "MD5SUM": record[8],
                        "FileSize": record[9],
                        "DataqualityFlag": record[11],
                        "JobId": record[12],
                        "RunNumber": record[13],
                        "InsertTimeStamp": record[14],
                        "Luminosity": record[15],
                        "InstLuminosity": record[16],
                        "VisibilityFlag": record[17],
                    }
                    result[record[0]] = row

        retVal = {"Successful": result, "Failed": list(set(lfns) - set(result))}
        return S_OK(retVal)

    #############################################################################
    def __getProductionStatisticsForUsers(self, prod):
        """For retrieving the statistics of a production.

        :param int prod: production number
        :return: number of files, evenet stat, filesize end luminosity
        """
        command = (
            "SELECT COUNT(*), SUM(files.EventStat), SUM(files.FILESIZE), SUM(files.Luminosity), \
    SUM(files.instLuminosity) FROM files ,jobs WHERE jobs.jobid=files.jobid AND jobs.production=%d"
            % (prod)
        )
        return self.dbR_.query(command)

    #############################################################################
    def getProductionFilesForWeb(self, prod, ftypeDict, sortDict, startItem, maxitems):
        """For retrieving the production file used by WebApp.

        :param int prod: production number
        :param dict ftypeDict: dictionary which contains the file type.
        :param dict sortDict: the columns which will be sorted.
        :param int startItem: used for paging. The row number
        :param int maxitems: number of rows
        :return: production files and its meta data
        """
        command = ""
        parametersNames = [
            "Name",
            "FileSize",
            "FileType",
            "CreationDate",
            "EventType",
            "EventStat",
            "GotReplica",
            "InsertTimeStamp",
            "Luminosity",
            "InstLuminosity",
        ]
        records = []
        result = S_ERROR()

        totalrecords = 0
        nbOfEvents = 0
        filesSize = 0
        ftype = ftypeDict["type"]
        if sortDict:
            res = self.__getProductionStatisticsForUsers(prod)
            if not res["OK"]:
                self.log.error(res["Message"])
            else:
                totalrecords = res["Value"][0][0]
                nbOfEvents = res["Value"][0][1]
                filesSize = res["Value"][0][2]

        if ftype != "ALL":
            command = (
                "SELECT rnum, filename, filesize, name , creationdate, eventtypeId, \
eventstat,gotreplica, inserttimestamp , luminosity ,instLuminosity FROM \
(SELECT rownum rnum, filename, filesize, name , creationdate, \
eventtypeId, eventstat, gotreplica, inserttimestamp, luminosity,instLuminosity \
FROM (SELECT files.filename, files.filesize, filetypes.name , files.creationdate, \
files.eventtypeId, files.eventstat,files.gotreplica, \
files.inserttimestamp, files.luminosity, files.instLuminosity \
FROM jobs,files, filetypes WHERE jobs.jobid=files.jobid AND \
jobs.production=%s AND filetypes.filetypeid=files.filetypeid AND filetypes.name='%s' \
ORDER BY files.filename) WHERE rownum <= %d ) WHERE rnum > %d"
                % (prod, ftype, maxitems, startItem)
            )
        else:
            command = (
                "SELECT rnum, fname, fsize, name, fcreation, feventtypeid, \
feventstat, fgotreplica, finst, flumi, finstlumy FROM \
(SELECT rownum rnum, fname, fsize, ftypeid, fcreation, feventtypeid, \
feventstat, fgotreplica, finst, flumi, finstlumy \
FROM (SELECT files.filename fname, files.filesize fsize, filetypeid \
ftypeid, files.creationdate fcreation, files.eventtypeId feventtypeid, \
files.eventstat feventstat, files.gotreplica fgotreplica, \
files.inserttimestamp finst, files.luminosity flumi, files.instLuminosity finstlumy \
FROM jobs,files WHERE jobs.jobid=files.jobid AND \
jobs.production=%d ORDER BY files.filename) WHERE rownum <=%d)f , filetypes ft WHERE rnum > %d \
AND ft.filetypeid=f.ftypeid"
                % (prod, maxitems, startItem)
            )

        res = self.dbR_.query(command)
        if res["OK"]:
            dbResult = res["Value"]
            for record in dbResult:
                row = [record[1], record[2], record[3], record[4], record[5], record[6], record[7], record[8]]
                records += [row]
            result = S_OK(
                {
                    "TotalRecords": totalrecords,
                    "ParameterNames": parametersNames,
                    "Records": records,
                    "Extras": {"GlobalStatistics": {"Number of Events": nbOfEvents, "Files Size": filesSize}},
                }
            )
        else:
            result = res
        return result

    #############################################################################
    def exists(self, lfns):
        """checks the files in the databse.

        :param list lfns: list of LFNs
        :return: True or False depending of the file existence
        """
        result = {}
        for lfn in lfns:
            res = self.dbR_.executeStoredFunctions("BOOKKEEPINGORACLEDB.fileExists", int, [lfn])
            if not res["OK"]:
                return res
            if res["Value"] == 0:
                result[lfn] = False
            else:
                result[lfn] = True
        return S_OK(result)

    #############################################################################
    def addReplica(self, fileNames):
        """adds the replica flag to a file.

        :param list fileNames: list of LFNs
        :return: dictionary which contains the failed and successful lfns
        """
        retVal = self.dbR_.executeStoredProcedure(
            packageName="BOOKKEEPINGORACLEDB.bulkcheckfiles", parameters=[], output=True, array=fileNames
        )
        if not retVal["OK"]:
            return retVal

        failed = {}
        for i in retVal["Value"]:
            failed[i[0]] = f"The file {i[0]} does not exist in the BKK database!!!"
            fileNames.remove(i[0])

        if fileNames:
            retVal = self.dbW_.executeStoredProcedure(
                packageName="BOOKKEEPINGORACLEDB.bulkupdateReplicaRow",
                parameters=["Yes"],
                output=False,
                array=fileNames,
            )
            if not retVal["OK"]:
                return retVal
            else:
                failed["Failed"] = list(failed)
                failed["Successful"] = fileNames
                return S_OK(failed)
        else:  # when no files exist
            files = {"Failed": [i[0] for i in retVal["Value"]], "Successful": []}
            return S_OK(files)

    #############################################################################
    def getRunInformations(self, runnb):
        """For retrieving the run statistics.

        :param int runnb: run number
        :return: the run statistics
        """
        result = S_ERROR()
        command = (
            "SELECT DISTINCT j.fillnumber, conf.configname, conf.configversion, "
            "daq.description, j.jobstart, j.jobend, j.tck, j.TOTALLUMINOSITY "
            "FROM jobs j, configurations conf, data_taking_conditions daq, productionscontainer prod "
            "WHERE j.configurationid=conf.configurationid AND "
            "j.production<0 AND prod.daqperiodid=daq.daqperiodid AND j.production=prod.production AND j.runnumber=%d"
            % (runnb)
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        value = retVal["Value"]
        if not value:
            return S_ERROR("This run is missing in the BKK DB!")

        values = {"Configuration Name": value[0][1], "Configuration Version": value[0][2], "FillNumber": value[0][0]}
        values["DataTakingDescription"] = value[0][3]
        values["RunStart"] = value[0][4]
        values["RunEnd"] = value[0][5]
        values["Tck"] = value[0][6]
        values["TotalLuminosity"] = value[0][7]

        retVal = self.getRunProcessingPass(runnb)
        if not retVal["OK"]:
            result = retVal
        else:
            values["ProcessingPass"] = retVal["Value"]
            command = (
                "SELECT COUNT(*), SUM(files.EventStat), SUM(files.FILESIZE), SUM(files.fullstat), "
                "files.eventtypeid, SUM(files.luminosity), SUM(files.instLuminosity) FROM files,jobs "
                "WHERE files.JobId=jobs.JobId AND files.gotReplica='Yes' AND jobs.production<0 AND jobs.runnumber="
                + str(runnb)
                + " GROUP BY files.eventtypeid"
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                result = retVal
            else:
                value = retVal["Value"]
                if not value:
                    result = S_ERROR("Replica flag is not set!")
                else:
                    nbfile = []
                    nbevent = []
                    fsize = []
                    fstat = []
                    stream = []
                    luminosity = []
                    ilumi = []
                    for i in value:
                        nbfile += [i[0]]
                        nbevent += [i[1]]
                        fsize += [i[2]]
                        fstat += [i[3]]
                        stream += [i[4]]
                        luminosity += [i[5]]
                        ilumi += [i[6]]

                    values["Number of file"] = nbfile
                    values["Number of events"] = nbevent
                    values["File size"] = fsize
                    values["FullStat"] = fstat
                    values["Stream"] = stream
                    values["luminosity"] = luminosity
                    values["InstLuminosity"] = ilumi
                    result = S_OK(values)

        return result

    #############################################################################
    def getRunInformation(self, inputParams):
        """For retrieving only the requested information for a given run.

        :param dict inputParams: RunNumber, Fields (CONFIGNAME, CONFIGVERSION, JOBSTART, JOBEND,
            TCK, FILLNUMBER, PROCESSINGPASS, CONDITIONDESCRIPTION,CONDDB, DDDB), Statistics (NBOFFILES, EVENTSTAT,
            FILESIZE, FULLSTAT, LUMINOSITY, INSTLUMINOSITY, EVENTTYPEID)

        :returns: run statistics
        """
        runnb = inputParams.get("RunNumber", default)
        if runnb == default:
            return S_ERROR("A RunNumber must be given!")

        if isinstance(runnb, (str, int)):
            runnb = [runnb]
        runs = ",".join([str(run) for run in runnb])
        fields = inputParams.get(
            "Fields",
            [
                "CONFIGNAME",
                "CONFIGVERSION",
                "JOBSTART",
                "JOBEND",
                "TCK",
                "FILLNUMBER",
                "PROCESSINGPASS",
                "CONDITIONDESCRIPTION",
                "CONDDB",
                "DDDB",
            ],
        )
        statistics = inputParams.get("Statistics", [])
        configurationsFields = ["CONFIGNAME", "CONFIGVERSION"]
        jobsFields = ["JOBSTART", "JOBEND", "TCK", "FILLNUMBER", "PROCESSINGPASS"]
        conditionsFields = ["CONDITIONDESCRIPTION"]
        stepsFields = ["CONDDB", "DDDB"]
        selection = []
        tables = ["jobs"]
        conditions = f"jobs.runnumber in ({runs}) AND jobs.production <0 "

        for i in fields:
            if i.upper() in configurationsFields:
                if "configurations" not in tables:
                    tables.append("configurations")
                    conditions += " AND jobs.configurationid=configurations.configurationid "
                selection.append(f"configurations.{i}")
            elif i.upper() in jobsFields:
                if i.upper() == "PROCESSINGPASS":
                    selection.append("BOOKKEEPINGORACLEDB.getProductionProcessingPass(-1 * jobs.runnumber)")
                else:
                    selection.append(f"jobs.{i}")
            elif i.upper() in conditionsFields:
                if "productionscontainer" not in tables:
                    tables.extend(["productionscontainer", "data_taking_conditions"])
                    conditions += " AND jobs.production=productionscontainer.production"
                    conditions += " AND productionscontainer.daqperiodid=data_taking_conditions.daqperiodid "
                selection.append("data_taking_conditions.description")
            elif i.upper() in stepsFields:
                if "stepscontainer" not in tables:
                    tables.extend(["stepscontainer", "steps"])
                    conditions += (
                        " AND jobs.production=stepscontainer.production AND stepscontainer.stepid=steps.stepid "
                    )
                selection.append(f"steps.{i}")

        command = "SELECT jobs.runnumber, {} FROM {} WHERE {} ".format(
            ", ".join(selection), ", ".join(tables), conditions
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        values = {}
        for i in retVal["Value"]:
            rnb = i[0]
            i = i[1:]
            record = dict(zip(fields, i))
            values[rnb] = record

        if statistics:
            filesFields = [
                "NBOFFILES",
                "EVENTSTAT",
                "FILESIZE",
                "FULLSTAT",
                "LUMINOSITY",
                "INSTLUMINOSITY",
                "EVENTTYPEID",
            ]
            tables = "jobs, files f "
            conditions = (
                "jobs.jobid=f.jobid AND jobs.runnumber in (%s) AND jobs.production <0 AND f.gotreplica='Yes' GROUP BY "
                "jobs.runnumber,f.eventtypeid " % (runs)
            )
            selection = "jobs.runnumber, "
            for i in statistics:
                if i.upper() == "NBOFFILES":
                    selection += "COUNT(*), "
                elif i.upper() == "EVENTTYPEID":
                    selection += f"f.{i},"
                elif i.upper() in filesFields:
                    selection += f"sum(f.{i}), "
            selection = selection[:-1]
            command = f"SELECT {selection}  from {tables} where {conditions}"
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                return retVal
            for i in retVal["Value"]:
                rnb = i[0]
                if "Statistics" not in values[rnb]:
                    values[rnb]["Statistics"] = []
                i = i[1:]
                record = dict(zip(statistics, i))
                values[rnb]["Statistics"] += [record]
        return S_OK(values)

    #############################################################################
    def getProductionFilesStatus(self, productionid=None, lfns=None):
        """the status of the files produced by a production.

        :param int productionid: production number
        :param list lfns: list of LFNs
        :return: replica, noreplica, missing
        """

        if lfns is None:
            lfns = []
        result = {}
        missing = []
        replicas = []
        noreplicas = []
        if productionid is not None:
            command = (
                "SELECT files.filename, files.gotreplica FROM files,jobs WHERE files.jobid=jobs.jobid AND jobs.production=%d "
                % (productionid)
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                return retVal
            files = retVal["Value"]
            for lfn in files:
                if lfn[1] == "Yes":
                    replicas += [lfn[0]]
                else:
                    noreplicas += [lfn[0]]
            result["replica"] = replicas
            result["noreplica"] = noreplicas
        elif lfns:
            for lfn in lfns:
                command = f" SELECT files.filename, files.gotreplica FROM files WHERE filename='{lfn}' "
                retVal = self.dbR_.query(command)
                if not retVal["OK"]:
                    return retVal
                value = retVal["Value"]
                if not value:
                    missing += [lfn]
                else:
                    for i in value:
                        if i[1] == "Yes":
                            replicas += [i[0]]
                        else:
                            noreplicas += [i[0]]
            result["replica"] = replicas
            result["noreplica"] = noreplicas
            result["missing"] = missing

        return S_OK(result)

    #############################################################################
    def getFileCreationLog(self, lfn):
        """For retrieving the Log file.

        :param str lfn: logical file name
        :return: the logs of a file
        """

        result = S_ERROR("getFileCreationLog error!")
        command = f"SELECT files.jobid FROM files WHERE files.filename='{lfn}' "
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            result = retVal
        elif not retVal["Value"]:
            result = S_ERROR("Job not in the DB")
        else:
            jobid = retVal["Value"][0][0]
            command = (
                "SELECT filename FROM files WHERE (files.filetypeid=17 OR files.filetypeid=9) AND files.jobid=%d "
                % (jobid)
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                result = retVal
            elif not retVal["Value"]:
                result = S_ERROR("Log file is not exist!")
            else:
                result = S_OK(retVal["Value"][0][0])
        return result

    #############################################################################
    def insertEventTypes(self, evid, desc, primary):
        """inserts an event type.

        :param int evid: event type id
        :param str desc: event type description
        :param str primary: event type short description
        """
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.insertEventTypes", [desc, evid, primary], False)

    #############################################################################
    def updateEventType(self, evid, desc, primary):
        """updates and existing event type."""
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.updateEventTypes", [desc, evid, primary], False)

    #############################################################################
    def getProductionSummary(
        self, cName, cVersion, conddesc=default, processing=default, production=default, ftype=default, evttype=default
    ):
        """For retrieving the statistics for a given data set.

        :param str cName: configuration name
        :param str: cVersion: configuration version
        :param str: conddesc: simulation or data taking description
        :param str processing: processing pass
        :paran int production: production number
        :param str ftype: file type
        :param int evttype: event type id
        :return: production statistics
        """

        tables = " productionoutputfiles prod, productionscontainer cont, simulationconditions sim, \
data_taking_conditions daq, configurations c "
        condition = (
            "cont.production=prod.production AND c.configurationid=cont.configurationid %s "
            % self.__buildVisible(visible="Y", replicaFlag="Yes")
        )

        condition, tables = self.__buildConfiguration(cName, cVersion, condition, tables)

        condition, tables = self.__buildProduction(production, condition, tables, useMainTables=False)

        condition, tables = self.__buildEventType(evttype, condition, tables, useMainTables=False)

        retVal = self.__buildProcessingPass(processing, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildFileTypes(ftype, condition, tables, useMainTables=False)

        retVal = self._buildConditions(default, conddesc, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        command = (
            "SELECT c.configname, c.configversion, sim.simdescription, daq.description, \
 cont.processingid, prod.eventtypeid,e.description, prod.production, ftypes.name, SUM(f.eventstat) \
FROM jobs j, files f, filetypes ftypes, eventtypes e, %s WHERE j.jobid= f.jobid AND \
f.gotreplica='Yes' AND prod.stepid= j.stepid AND e.eventtypeid=f.eventtypeid AND prod.eventtypeid=f.eventtypeid AND \
prod.stepid= j.stepid AND sim.simid(+)=cont.simid AND prod.filetypeid=f.filetypeid AND \
prod.filetypeid=ftypes.filetypeid AND daq.daqperiodid(+)=cont.daqperiodid  AND \
prod.production = cont.production AND %s GROUP BY c.configname, c.configversion, sim.simdescription, \
daq.description, cont.processingid, prod.eventtypeid, e.description, prod.production, ftypes.name"
            % (tables, condition)
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        parameters = [
            "ConfigurationName",
            "ConfigurationVersion",
            "ConditionDescription",
            "Processing pass ",
            "EventType",
            "EventType description",
            "Production",
            "FileType",
            "Number of events",
        ]
        dbResult = retVal["Value"]
        records = []
        nbRecords = 0
        for record in dbResult:
            if record[2] is not None:
                conddesc = record[2]
            else:
                conddesc = record[3]
            row = [record[0], record[1], conddesc, record[4], record[5], record[6], record[7], record[8], record[9]]
            records += [row]
            nbRecords += 1
        result = S_OK({"TotalRecords": nbRecords, "ParameterNames": parameters, "Records": records, "Extras": {}})

        return result

    #############################################################################
    def getProductionSimulationCond(self, prod):
        """For retrieving the simulation or data taking description of a
        production.

        :param int prod: production number
        :return: simulation condition
        """
        simdesc = None
        daqdesc = None

        command = (
            "SELECT DISTINCT sim.simdescription, daq.description FROM \
simulationconditions sim, data_taking_conditions daq, productionscontainer prod \
WHERE sim.simid(+)=prod.simid AND daq.daqperiodid(+)=prod.daqperiodid AND prod.production="
            + str(prod)
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        else:
            value = retVal["Value"]
            if value:
                simdesc = value[0][0]
                daqdesc = value[0][1]
            else:
                return S_ERROR("Simulation condition or data taking condition not exist!")

        if simdesc is not None:
            return S_OK(simdesc)
        else:
            return S_OK(daqdesc)

    #############################################################################
    def getFileHistory(self, lfn):
        """For retrieving the ancestor for a given file.

        :param str lfn: logical file name
        :retun: files and associated meta data
        """

        command = (
            "SELECT  files.fileid, files.filename,files.adler32, \
files.creationdate, files.eventstat, files.eventtypeid, files.gotreplica, \
files.guid, files.jobid, files.md5sum, files.filesize, files.fullstat, dataquality.dataqualityflag, \
files.inserttimestamp, files.luminosity, files.instLuminosity FROM files, dataquality \
WHERE files.fileid IN (SELECT inputfiles.fileid FROM files,inputfiles WHERE \
files.jobid= inputfiles.jobid AND files.filename='%s')\
AND files.qualityid= dataquality.qualityid"
            % lfn
        )

        return self.dbR_.query(command)

    #############################################################################
    #
    #          MONITORING
    #############################################################################
    def getProductionNbOfJobs(self, prodid):
        """Number of jobs for given production.

        :param int prodid: production number
        :return: the number of jobs
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getJobsNb", [prodid])

    #############################################################################
    def getProductionNbOfEvents(self, prodid):
        """Number of event for a given production.

        :param int prodid: production number
        :return: the number of events
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getNumberOfEvents", [prodid])

    #############################################################################
    def getProductionSizeOfFiles(self, prodid):
        """Size of the files for a given production.

        :param int prodid: production number
        :return: the size of files
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getSizeOfFiles", [prodid])

    #############################################################################
    def getProductionNbOfFiles(self, prodid):
        """For retrieving number of files for a given production.

        :param int prodid: production number
        :return: the number of files
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getNbOfFiles", [prodid])

    #############################################################################
    @deprecated("Unused?")
    def getProductionInformation(self, prodid):
        """For retrieving production statistics.

        :param int prodid: production number
        :return: the statistics of a production
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getProductionInformation", [prodid])

    #############################################################################
    def getSteps(self, prodid):
        """For retrieving the production step.

        :param int prodid: production number

        :return: the steps used by a production, with resolved DB tags
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getSteps", [prodid])

    #############################################################################
    def getNbOfJobsBySites(self, prodid):
        """the number of successfully finished jobs at different Grid sites for a
        given production.

        :param int prodid: production number
        :return: number of jobs
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getJobsbySites", [prodid])

    #############################################################################
    def getConfigsAndEvtType(self, prodid):
        """For retrieving the configuration name, version and event type.

        :param int prodid: production number
        :return: the configurations and event type of a production
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getConfigsAndEvtType", [prodid])

    #############################################################################
    def getAvailableTags(self):
        """For retrieving the database tags.

        :return: the tags
        """
        command = "SELECT name, tag FROM tags ORDER BY inserttimestamp DESC"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        parameters = ["TagName", "TagValue"]
        dbResult = retVal["Value"]
        records = []
        nbRecords = 0
        for record in dbResult:
            row = [record[0], record[1]]
            records += [row]
            nbRecords += 1
        return S_OK({"TotalRecords": nbRecords, "ParameterNames": parameters, "Records": records, "Extras": {}})

    #############################################################################
    def getProductionProcessedEvents(self, prodid):
        """For retreiving all events in specific production.

        :param int prodid: production number
        :return: the processed event by a production
        """
        return self.dbR_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getProcessedEvents", int, [prodid])

    #############################################################################
    def getRunsForAGivenPeriod(self, in_dict):
        """For retrieving list of runs.

        :param dict in_dict: bkQuery bookkeeping query
        :return: the runs for a given conditions
        """
        condition = ""
        startDate = in_dict.get("StartDate", default)
        endDate = in_dict.get("EndDate", default)
        allowOutside = in_dict.get("AllowOutsideRuns", default)

        if allowOutside != default:
            if startDate == default and endDate == default:
                return S_ERROR("The Start and End date must be given!")
            else:
                condition += f" AND jobs.jobstart >= TO_TIMESTAMP ('{startDate}','YYYY-MM-DD HH24:MI:SS')"
                condition += f" AND jobs.jobstart <= TO_TIMESTAMP ('{endDate}','YYYY-MM-DD HH24:MI:SS')"
        else:
            if startDate != default:
                condition += f" AND jobs.jobstart >= TO_TIMESTAMP ('{startDate}','YYYY-MM-DD HH24:MI:SS')"
            if endDate != default:
                condition += f" AND jobs.jobend <= TO_TIMESTAMP ('{endDate}','YYYY-MM-DD HH24:MI:SS')"
            elif startDate != default and endDate == default:
                currentTimeStamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
                condition += (
                    " AND jobs.jobend <= TO_TIMESTAMP ('" + str(currentTimeStamp) + "','YYYY-MM-DD HH24:MI:SS')"
                )
                condition += f" AND jobs.jobend <= TO_TIMESTAMP ('{str(currentTimeStamp)}','YYYY-MM-DD HH24:MI:SS')"

        command = " SELECT jobs.runnumber FROM jobs WHERE jobs.production < 0" + condition
        retVal = self.dbR_.query(command)
        runIds = []
        if retVal["OK"]:
            records = retVal["Value"]
            for record in records:
                if record[0] is not None:
                    runIds += [record[0]]
        else:
            return retVal

        check = in_dict.get("CheckRunStatus", False)
        if check:
            processedRuns = []
            notProcessedRuns = []
            for i in runIds:
                command = (
                    "SELECT files.filename FROM files,jobs WHERE jobs.jobid=files.jobid \
AND files.gotreplica='Yes' AND jobs.production<0 AND jobs.runnumber=%d"
                    % (i)
                )
                retVal = self.dbR_.query(command)
                if retVal["OK"]:
                    files = retVal["Value"]
                    ok = True
                    for lfn in files:
                        name = lfn[0]
                        retVal = self.getFileDescendents([name], 1, 0, True)
                        successful = retVal["Value"]["Successful"]
                        if not successful:
                            ok = False
                    if ok:
                        processedRuns += [i]
                    else:
                        notProcessedRuns += [i]
            return S_OK({"Runs": runIds, "ProcessedRuns": processedRuns, "NotProcessedRuns": notProcessedRuns})
        else:
            return S_OK({"Runs": runIds})
        return S_ERROR()

    #############################################################################

    # FIXME: is this useful at all? Does prodrunview still exist?
    def getProductionsFromView(self, in_dict):
        """For retrieving productions.

        :param dict in_dict: bkQuery bookkeeping query
        :return: the productions using the bookkeeping view
        """
        run = in_dict.get("RunNumber", in_dict.get("Runnumber", default))
        proc = in_dict.get("ProcessingPass", in_dict.get("ProcPass", default))
        result = S_ERROR()
        if "Runnumber" in in_dict:
            self.log.verbose("The Runnumber has changed to RunNumber!")

        if run != default:
            if proc != default:
                retVal = self.__getProcessingPassId(proc.split("/")[1:][0], proc)
                if retVal["OK"]:
                    processingid = retVal["Value"]
                    command = (
                        "SELECT DISTINCT prod.production FROM productionoutputfiles prod, \
prodrunview prview, productionscontainer cont WHERE \
prod.production=prview.production AND prview.runnumber=%d AND \
prod.production>0 AND prod.production=cont.production AND cont.processingid=%d"
                        % (run, processingid)
                    )
                    result = self.dbR_.query(command)
            else:
                result = S_ERROR("The processing pass is missing!")
        else:
            result = S_ERROR("The run number is missing!")

        return result

    #############################################################################
    def getRunFilesDataQuality(self, runs):
        """For retrieving list of files.

        :param list runs: list of run numbers
        :retun: the files with data quality
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getRunQuality", [], True, runs)

    #############################################################################
    def getRunAndProcessingPassDataQuality(self, runnb, processing):
        """For retrieving the data quality flag for run and processing pass.

        :param int runnb: run number
        :param str processing: processing pass
        :return: data quality
        """
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getQFlagByRunAndProcId", str, [runnb, processing])

    #############################################################################
    def getRunWithProcessingPassAndDataQuality(self, procpass, flag=default):
        """For retrieving a list of runs for a given processing pass and data
        quality flag.

        :param str procpass: processing pass
        :param str flag: file data quality flag
        :return: runs
        """
        retVal = self.__getProcessingPassId(procpass.split("/")[1:][0], procpass)
        if retVal["OK"]:
            processingid = retVal["Value"]
            qualityid = None
            if flag != default:
                retVal = self.__getDataQualityId(flag)
                if retVal["OK"]:
                    qualityid = retVal["Value"]
                else:
                    return retVal
        retVal = self.dbW_.executeStoredProcedure(
            "BOOKKEEPINGORACLEDB.getRunByQflagAndProcId", [processingid, qualityid]
        )
        if not retVal["OK"]:
            return retVal
        else:
            result = [i[0] for i in retVal["Value"]]
        return S_OK(result)

    #############################################################################
    def setFilesInvisible(self, lfns):
        """sets a given list of lfn invisible.

        :param list lfns: list of LFNs
        """

        for i in lfns:
            retVal = self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.setFileInvisible", [i], False)
            if not retVal["OK"]:
                return retVal
        return S_OK("The files are invisible!")

    #############################################################################
    def setFilesVisible(self, lfns):
        """sets a given list of lfn visible.

        :param list lfns: list of LFNs
        """
        for i in lfns:
            res = self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.setFileVisible", [i], False)
            if not res["OK"]:
                return res
        return S_OK("The files are visible!")

    #############################################################################
    def getFiles(
        self,
        simdesc,
        datataking,
        procPass,
        ftype,
        evt,
        configName=default,
        configVersion=default,
        production=default,
        flag=default,
        startDate=None,
        endDate=None,
        nbofEvents=False,
        startRunID=None,
        endRunID=None,
        runnumbers=None,
        replicaFlag=default,
        visible=default,
        filesize=False,
        tcks=None,
        jobStart=None,
        jobEnd=None,
        smog2States=None,
    ):
        """returns a list of lfns.

        :param str simdesc: simulation condition description
        :param str datataking: data taking condition description
        :pram str procPass: processing pass
        :param str ftype: file type
        :param int evt: event type
        :param str configName: configuration name
        :param str configVersion: configuration version
        :param int production: production number
        :param str flag: data quality flag
        :param datetime startDate: job/run insert start time stamp
        :param datetime endDate: job/run insert end time stamp
        :param bool nbofEvents: count number of events
        :param int startRunID: start run number
        :param int endRunID: end run number
        :param list runnumbers: list of run numbers
        :param str replicaFlag: file replica flag
        :param str visible: file visibility flag
        :param bool filesize: only sum the files size
        :param list tcks: list of run TCKs
        :param datetime jobStart: job starte date
        :param datetime jobEnd: job end date
        :returns: list of files
        """

        if runnumbers is None:
            runnumbers = []

        if tcks is None:
            tcks = []

        condition = ""
        tables = " files f, jobs j"

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        condition, tables = self.__buildProduction(production, condition, tables)

        retVal = self.__buildTCKS(tcks, condition)
        if not retVal["OK"]:
            return retVal
        condition = retVal["Value"]

        retVal = self.__buildProcessingPass(procPass, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildFileTypes(ftype, condition, tables)

        retVal = self._buildRunnumbers(runnumbers, startRunID, endRunID, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildEventType(evt, condition, tables)

        condition = self.__buildStartenddate(startDate, endDate, condition)

        condition = self.__buildJobsStartJobEndDate(jobStart, jobEnd, condition)

        retVal = self.__buildDataquality(flag, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition = self.__buildReplicaflag(replicaFlag, condition)

        condition, tables = self.__buildVisibilityflag(visible, condition, tables)

        retVal = self._buildConditions(simdesc, datataking, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        retVal = self._buildSMOG2Conditions(smog2States, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        # hint = ''
        # if (not startDate and not endDate) and tables.strip() == 'files f,jobs j  ,filetypes ft':
        #  hint = '/*+INDEX(j JOBS_PRODUCTIONID) INDEX(f FILES_JOB_EVENT_FILETYPE) INDEX(ft FILETYPES_ID_NAME)*/'

        if nbofEvents:
            command = f"SELECT SUM(f.eventstat) FROM {tables} WHERE f.jobid= j.jobid {condition} "
        elif filesize:
            command = f"SELECT SUM(f.filesize) FROM {tables} WHERE f.jobid= j.jobid {condition} "
        else:
            command = f"SELECT DISTINCT f.filename FROM {tables} WHERE f.jobid= j.jobid {condition} "
        return self.dbR_.query(command)

    #############################################################################
    @staticmethod
    def __buildConfiguration(configName, configVersion, condition, tables):
        """it constructs the condition string for a given configName and configVersion.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str condition: condition string
        :param str tables: tables used by join
        :return: condition and tables
        """

        if configName not in [default, None, ""] and configVersion not in [default, None, ""]:
            if "configurations" not in tables.lower():
                tables += " , configurations c"
            if "productionscontainer" in tables.lower():
                condition += " AND c.configurationid=cont.configurationid"
            elif "jobs" in tables.lower():
                condition += " AND c.configurationid=j.configurationid"
            else:
                raise RuntimeError("No table to join configuration, exiting...")
            condition += f" AND c.configname='{configName}'"
            condition += f" AND c.configversion='{configVersion}'"

        return condition, tables

    @staticmethod
    def __buildVisible(condition=None, visible=default, replicaFlag=default):
        """It makes the condition for a given visibility flag and replica flag."""
        if condition is None:
            condition = ""
        if not visible.upper().startswith("A"):
            if visible.upper().startswith("Y"):
                condition += " AND prod.visible='Y'"
            elif visible.upper().startswith("N"):
                condition += " AND prod.visible='N'"
        if replicaFlag.upper() != default:
            condition += f" AND prod.gotreplica='{replicaFlag}'"

        return condition

    #############################################################################
    @staticmethod
    def __buildProduction(production, condition, tables, useMainTables=True):
        """it adds the production which can be a list or string to the jobs table.

        :param list, int production: the production number(s)
        :param str condition: contains the conditions
        :param str tables: contains the tables.
        :param bool useMainTables: It is better not to use the view in some cases. This variable is used to
        disable the view usage.
        """

        table = "prod"
        if production not in [default, None]:
            if useMainTables:
                table = "j"
            else:
                if "productionoutputfiles" not in tables.lower():
                    tables += " , productionoutputfiles prod"

            if isinstance(production, list) and production:
                condition += " AND "
                cond = " ("
                for i in production:
                    cond += f" {table}.production={str(i)} or "
                cond = cond[:-3] + ")"
                condition += cond
            elif isinstance(production, (str, int)):
                condition += f" AND {table}.production={str(production)}"

        return condition, tables

    #############################################################################
    @staticmethod
    def __buildTCKS(tcks, condition):
        """it adds the tck to the jobs table.

        :param list tcks: list of run TCKs
        :param str condition: condition string
        :return: condition
        """

        if tcks not in [None, default]:
            if isinstance(tcks, list):
                if default in tcks:
                    tcks.remove(default)
                if tcks:
                    condition += " AND (" + " or ".join([f" j.tck='{i}'" for i in tcks]) + ")"
            elif isinstance(tcks, str):
                condition += f" AND j.tck='{tcks}'"
            else:
                return S_ERROR("The TCK should be a list or a string")

        return S_OK(condition)

    #############################################################################
    def __buildProcessingPass(self, procPass, condition, tables, useMainTables=True):
        """It adds the processing pass condition to the query.

        :param str procPass: processing pass for example: /Real Data/Reco20
        :param str condition: contains the conditions
        :param str tables: contains the tables.
        :param bool useMainTables: It is better not to use the view in some cases. This variable is used to
        disable the view usage.
        """
        if procPass not in [default, None]:
            if not re.search("^/", procPass):
                procPass = procPass.replace(procPass, f"/{procPass}")
            command = (
                "SELECT v.id FROM (SELECT DISTINCT SYS_CONNECT_BY_PATH(name, '/') Path, id ID \
                                           FROM processing v   \
                                           START WITH id in (SELECT DISTINCT id FROM processing WHERE name='%s') \
                                              CONNECT BY NOCYCLE PRIOR id=parentid) v \
                     WHERE v.path='%s'"
                % (procPass.split("/")[1], procPass)
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                return retVal

            if len(retVal["Value"]) < 1:
                return S_ERROR("No file found! Processing pass is missing!")

            pro = "("
            for i in retVal["Value"]:
                pro += f"{str(i[0])},"
            pro = pro[:-1]
            pro += ")"
            condition += f" and cont.processingid in {pro}"

            if useMainTables:
                condition += " and cont.production=j.production "

            if "productionscontainer" not in tables.lower():
                tables += " , productionscontainer cont"
        return S_OK((condition, tables))

    #############################################################################
    @staticmethod
    def __buildFileTypes(ftype, condition, tables, useMainTables=True):
        """it adds the file type to the files list.

        :param list, str ftype it is used to construct the file type query filter
        using a given file type or a list of filetypes.
        :param str condition It contains the where conditions
        :param str tables it containes the tables.
        :param str visible the default value is 'ALL'. [Y,N]
        :param bool useView It is better not to use the view in some cases. This variable is used to
        disable the view usage.
        """

        if ftype not in [default, None]:
            if tables.lower().find("filetypes") < 0:
                tables += " ,filetypes ft"
            if isinstance(ftype, list) and ftype:
                condition += " AND "
                cond = " ("
                for i in ftype:
                    cond += f" ft.name='{i}' or "
                cond = cond[:-3] + ")"
                condition += cond
            elif isinstance(ftype, str):
                condition += f" AND ft.name='{ftype}'"
            else:
                return S_ERROR("File type problem!")

            if useMainTables:
                condition += " AND f.filetypeid=ft.filetypeid"
            else:
                condition += " AND ft.filetypeid=prod.filetypeid"

        if isinstance(ftype, str) and ftype.upper() == "RAW" and "jobs" in tables:
            # we know the production of a run is less than 0.
            # this is needed to speed up the queries when the file type is raw
            # (we reject all recostructed + stripped jobs/files. ).
            condition += " AND j.production<0"
        return condition, tables

    #############################################################################
    @staticmethod
    def _buildRunnumbers(runnumbers, startRunID, endRunID, condition, tables, useMainTables=True):
        """it adds the run numbers or start end run to the jobs table.

        :param list runnumbers: list of runs
        :param int startRunID: start run number
        :param int endRunID: end run number
        :param str condition: condition string
        :param str tables: tables used by join
        :return: condition and tables
        """
        if useMainTables:
            table = "j"
        else:
            table = "prview"
            if runnumbers and runnumbers != default:
                condition += " AND prview.production=cont.production "
                if "prodrunview" not in tables.lower():
                    tables += " , prodrunview prview"
        if isinstance(runnumbers, int):
            condition += f" AND {table}.runnumber={str(runnumbers)} "
        elif isinstance(runnumbers, str) and runnumbers.upper() != default:
            condition += f" AND {table}.runnumber={str(runnumbers)} "
        elif isinstance(runnumbers, list) and runnumbers:
            cond = " ("
            for i in runnumbers:
                cond += f" {table}.runnumber={str(i)} OR "
            cond = cond[:-3] + ")"
            if startRunID is not None and endRunID is not None:
                condition += " AND ({}.runnumber>={} AND {}.runnumber<={} OR {}) ".format(
                    table,
                    str(startRunID),
                    table,
                    str(endRunID),
                    cond,
                )
            elif startRunID is not None or endRunID is not None:
                condition += f" AND {cond} "
            elif startRunID is None or endRunID is None:
                condition += f" AND {cond} "
        else:
            if (isinstance(startRunID, str) and startRunID.upper() != default) or (
                isinstance(startRunID, int) and startRunID is not None
            ):
                condition += f" AND {table}.runnumber>={str(startRunID)} "
            if (isinstance(endRunID, str) and endRunID.upper() is not default) or (
                isinstance(endRunID, int) and endRunID is not None
            ):
                condition += f" AND {table}.runnumber<={str(endRunID)} "
        return S_OK((condition, tables))

    #############################################################################
    @staticmethod
    def __buildEventType(evt, condition, tables, useMainTables=True):
        """adds the event type to the files table.

    :param list, str evt it is used to construct the event type query filter using a \
    given event type or a list of event types.
    :param str condition It contains the where conditions
    :param str tables it containes the tables.
    :param str visible the default value is 'ALL'. [Y,N]
    :param bool useView It is better not to use the view in some cases. This variable is used to
    disable the view usage.
    """

        table = "prod"
        if evt not in [0, None, default]:
            if useMainTables:
                table = "f"
            else:
                if "productionoutputfiles" not in tables.lower():
                    tables += " , productionoutputfiles prod"

            if isinstance(evt, (list, tuple)) and evt:
                condition += " AND "
                cond = " ("
                for i in evt:
                    cond += f" {table}.eventtypeid={(str(i))} or "
                cond = cond[:-3] + ")"
                condition += cond
            elif isinstance(evt, (str, int)):
                condition += f" AND {table}.eventtypeid={str(evt)}"
            if useMainTables:
                if isinstance(evt, (list, tuple)) and evt:
                    condition += " AND "
                    cond = " ("
                    for i in evt:
                        cond += f" {table}.eventtypeid={(str(i))} or "
                    cond = cond[:-3] + ")"
                    condition += cond
                elif isinstance(evt, (str, int)):
                    condition += f" AND {table}.eventtypeid={str(evt)}"
        return condition, tables

    #############################################################################
    @staticmethod
    def __buildStartenddate(startDate, endDate, condition):
        """it adds the start and end date to the files table.

        :param datetime startDate:  file insert start date
        :param datetime endDate: file insert end date
        :param str condition: condition string
        :return: condition
        """
        if startDate not in [None, default, []]:
            condition += f" AND f.inserttimestamp >= TO_TIMESTAMP ('{str(startDate)}','YYYY-MM-DD HH24:MI:SS')"

        if endDate not in [None, default, []]:
            condition += f" AND f.inserttimestamp <= TO_TIMESTAMP ('{str(endDate)}','YYYY-MM-DD HH24:MI:SS')"
        elif startDate not in [None, default, []] and endDate in [None, default, []]:
            currentTimeStamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
            condition += " AND f.inserttimestamp <= TO_TIMESTAMP ('%s','YYYY-MM-DD HH24:MI:SS')" % (
                str(currentTimeStamp)
            )
        return condition

    #############################################################################
    @staticmethod
    def __buildJobsStartJobEndDate(jobStartDate, jobEndDate, condition):
        """it adds the start and end date to the files table.

        :param datetime startDate:  file insert start date
        :param datetime jobStartDate:  file insert start date
        :param datetime jobEndDate: file insert end date
        :param str condition: condition string
        :return: condition
        """
        if jobStartDate not in [None, default, []]:
            condition += f" AND j.jobstart >= TO_TIMESTAMP ('{str(jobStartDate)}','YYYY-MM-DD HH24:MI:SS')"

        if jobEndDate not in [None, default, []]:
            condition += f" AND j.jobend <= TO_TIMESTAMP ('{str(jobEndDate)}','YYYY-MM-DD HH24:MI:SS')"
        elif jobStartDate not in [None, default, []] and jobEndDate in [None, default, []]:
            currentTimeStamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
            condition += f" AND j.jobend <= TO_TIMESTAMP ('{str(currentTimeStamp)}','YYYY-MM-DD HH24:MI:SS')"
        return condition

    #############################################################################
    def __buildDataquality(self, flag, condition, tables):
        """it adds the data quality to the files table.

        :param str flag: data quality flag
        :param str condition: condition string
        :param str tables: tables used by join
        :return: condition and tables
        """
        if flag not in [default, None]:
            if isinstance(flag, (list, tuple)):
                conds = " ("
                for i in flag:
                    quality = None
                    command = f"SELECT QualityId FROM dataquality WHERE dataqualityflag='{str(i)}'"
                    res = self.dbR_.query(command)
                    if not res["OK"]:
                        self.log.error("Data quality problem:", res["Message"])
                    elif not res["Value"]:
                        return S_ERROR("No file found! Dataquality is missing!")
                    else:
                        quality = res["Value"][0][0]
                    conds += " f.qualityid=" + str(quality) + " or"
                condition += " and" + conds[:-3] + ")"
            else:
                quality = None
                command = "SELECT QualityId FROM dataquality WHERE dataqualityflag='" + str(flag) + "'"
                res = self.dbR_.query(command)
                if not res["OK"]:
                    self.log.error("Data quality problem:", res["Message"])
                elif not res["Value"]:
                    return S_ERROR("No file found! Dataquality is missing!")
                else:
                    quality = res["Value"][0][0]

                condition += " AND f.qualityid=" + str(quality)
        return S_OK((condition, tables))

    #############################################################################
    @staticmethod
    def __buildReplicaflag(replicaFlag, condition):
        """it adds the replica flag to the files table.

        :param str replicaFlag: file replica flag
        :param str condition: condition string
        :return: condition and tables
        """
        if replicaFlag in ["Yes", "No"]:
            condition += f" AND f.gotreplica='{replicaFlag}' "
        return condition

    #############################################################################
    @staticmethod
    def __buildVisibilityflag(visible, condition, tables):
        """it adds the visibility flag to the files table.

        :param str visible: visibility flag
        :param str condition: condition string
        :param str tables: tables used by the join
        :return: condition and tables
        """
        if not visible.upper().startswith("A"):
            if visible.upper().startswith("Y"):
                condition += " AND f.visibilityflag='Y'"
            elif visible.upper().startswith("N"):
                condition += " AND f.visibilityflag='N'"
        if tables.upper().find("FILES") < 0:
            tables += " , file f "
        if tables.upper().find("JOBS") < 0:
            tables += " , jobs j "
        return condition, tables

    #############################################################################
    def _buildConditions(self, simdesc, datataking, condition, tables):
        """adds the data taking or simulation conditions to the query.

        :param str simdesc it is used to construct the simulation condition query filter
        :param str datataking it is used to construct the data taking condition query filter
        :param str condition It contains the where conditions
        :param str tables it containes the tables.
        """
        if tables is None:
            tables = ""
        if condition is None:
            condition = ""
        if simdesc != default or datataking != default:
            conddesc = simdesc if simdesc != default else datataking
            retVal = self.__getConditionString(conddesc, "cont")
            if not retVal["OK"]:
                return retVal
            condition += retVal["Value"]
            if tables.upper().find("PRODUCTIONSCONTAINER") < 0:
                tables += " , productionscontainer cont "

        return S_OK((condition, tables))

    #############################################################################
    def _SMOG2States2SQL(self, smog2States):
        """convert the list of possible states (or "Undefined") into SQL conditions.

        :param list[str]: smog2state: the list of states, assumes OR.
        """
        if not isinstance(smog2States, list):
            return S_ERROR("incorrect SMOG2 type, expected a list of strings")
        cond = ",".join([f"'{state}'" for state in smog2States if state != "Undefined"])
        if cond != "":
            cond = f"(s2s.state in ({cond}))"
        if "Undefined" in smog2States:
            if cond != "":
                cond = f" AND (s2s.id IS NULL OR {cond})"
            else:
                cond = " AND s2s.id IS NULL"
        elif cond != "":
            cond = f" AND {cond}"
        return S_OK(f" AND runs.smog2_id = s2s.id (+) {cond}")

    #############################################################################
    def _buildSMOG2Conditions(self, smog2States, condition, tables):
        """adds SMOG2 state condition to the query.
        WARNING: depends from the previously defined conditions!
        At the moment can be called ONLY from:
            getFiles()
            getFilesWithMetadata()
            getFilesSummary()

        :param list[str] smog2states: a list of SMOG2 states
        """
        if not smog2States:
            return S_OK((condition, tables))  # don't wake the sleeping bear...

        # we need the reference to runnumber, which explicitly exists in:
        #   runstatus - O(runs*1), not used in targeted functions
        #   newrunquality - O(runs*n), not used in targeted functions
        #   prodrunview - O(prods = runs*m)
        #   jobs - O(inf), has runnumber index, but partitioned by production
        # note that some tables specify production only
        #   productionscontainer - O(prods)
        #   stepscontainer - O(prods*k)
        #   productionoutputfiles - O(prods*l)
        # and production in general works with many runs.
        #
        # So runs.smog2_id link shuld be jobs.runnumber = runs.runnumber (+) (AND state conditions),
        # LEFT (OUTER) JOINT allows deal with "Undefined" case.
        #
        # If SMOG2= has to work in not jobs based (f.e. production) queries, that should
        # be implemented separately, with proper grouping (f.e. prodrunview can easily
        # multiply results and it is updated asyncroneously to file registration)

        if not tables or "jobs j" not in tables.lower():
            raise RuntimeError("No jobs to join SMOG2, exiting...")
        if condition is None:
            condition = ""  # all jobs for SMOG2 conditions... the list can be long...
        condRet = self._SMOG2States2SQL(smog2States)
        if not condRet["OK"]:
            return condRet
        if "smog2" not in tables.lower():
            tables += " , smog2 s2s"
        if "runs" not in tables.lower():
            tables += " , runs"
        condition += f" AND j.runnumber = runs.runnumber (+) {condRet['Value']} "
        return S_OK((condition, tables))

    #############################################################################
    def getVisibleFilesWithMetadata(
        self,
        simdesc,
        datataking,
        procPass,
        ftype,
        evt,
        configName=default,
        configVersion=default,
        production=default,
        flag=default,
        startDate=None,
        endDate=None,
        nbofEvents=False,
        startRunID=None,
        endRunID=None,
        runnumbers=None,
        replicaFlag="Yes",
        tcks=None,
        jobStart=None,
        jobEnd=None,
    ):
        """For  retrieving only visible files.

        :param str simdesc: simulation desctription
        :param str datataking: data taking description
        :param str procPass: processing pass
        :param str ftype: file type
        :param str evt: event type
        :param str configName: configuration name
        :param str configVersion: configuration version
        :param int production: production number
        :param str flag: data quality
        :param datetime startDate: job start insert time stamp
        :param datetime endDate: job end insert time stamp
        :param bool nbofEvemts: count number of events
        :param int startRunID: start run number
        :param int endRunID: end run number
        :param str replicaFlag: file replica flag
        :param list tcks: run TCKs
        :param datetime jobStart: job starte date
        :param datetime jobEnd: job end date
        :return: the visible files
        """
        conddescription = datataking
        if simdesc != default:
            conddescription = simdesc

        selection = " DISTINCT f.filename, f.eventstat, j.eventinputstat, \
j.runnumber, j.fillnumber, f.filesize, j.totalluminosity, f.luminosity, f.instLuminosity, j.tck "

        return self.getFilesWithMetadata(
            configName,
            configVersion,
            conddescription,
            procPass,
            evt,
            production,
            ftype,
            flag,
            "Y",
            "Yes",
            startDate,
            endDate,
            runnumbers,
            startRunID,
            endRunID,
            tcks,
            jobStart,
            jobEnd,
            selection,
        )

    #############################################################################
    def getFilesSummary(
        self,
        configName,
        configVersion,
        conditionDescription=default,
        processingPass=default,
        eventType=default,
        production=default,
        fileType=default,
        dataQuality=default,
        startRun=default,
        endRun=default,
        visible=default,
        startDate=None,
        endDate=None,
        runNumbers=None,
        replicaFlag=default,
        tcks=default,
        jobStart=None,
        jobEnd=None,
        smog2States=None,
    ):
        """File summary for a given data set.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: simulation or data taking condition
        :param str processingPass: processing pass
        :param int eventType: event type
        :param int production: production number
        :param str filetype: file type
        :param str dataQuality: data quality
        :param int startRun: satart run number
        :param int endRun: end run number
        :param str visible: visibility flag
        :param datetime startDate: job start insert time stamp
        :param datetime endDate: job end insert time stamp
        :param list runNumbers: list of run numbers
        :param str replicaFlag: file replica flag
        :param list tcks: list of run TCKs
        :param datetime jobStart: job starte date
        :param datetime jobEnd: job end date
        :retun: the number of event, files, etc for a given data set
        """

        if runNumbers is None:
            runNumbers = []

        tables = " files f, jobs j, productionscontainer cont, productionoutputfiles prod "
        condition = (
            " AND cont.production=prod.production AND j.production=prod.production AND j.stepid=prod.stepid AND \
prod.eventtypeid=f.eventtypeid %s "
            % self.__buildVisible(visible=visible, replicaFlag=replicaFlag)
        )

        condition = self.__buildStartenddate(startDate, endDate, condition)

        condition = self.__buildJobsStartJobEndDate(jobStart, jobEnd, condition)

        retVal = self._buildRunnumbers(runNumbers, startRun, endRun, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        retVal = self.__buildTCKS(tcks, condition)
        if not retVal["OK"]:
            return retVal
        condition = retVal["Value"]

        retVal = self._buildConditions(default, conditionDescription, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildVisibilityflag(visible, condition, tables)

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        condition, tables = self.__buildProduction(production, condition, tables, useMainTables=False)

        condition, tables = self.__buildEventType(eventType, condition, tables, useMainTables=False)

        if production != default:
            condition += " and j.production=" + str(production)

        condition, tables = self.__buildFileTypes(fileType, condition, tables, useMainTables=False)

        condition = self.__buildReplicaflag(replicaFlag, condition)

        retVal = self.__buildProcessingPass(processingPass, condition, tables, useMainTables=False)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        retVal = self.__buildDataquality(dataQuality, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        retVal = self._buildSMOG2Conditions(smog2States, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        command = (
            "SELECT COUNT(DISTINCT fileid), \
SUM(f.EventStat), SUM(f.FILESIZE), \
SUM(f.luminosity),SUM(f.instLuminosity) FROM  %s WHERE \
j.jobid=f.jobid AND \
prod.production=cont.production AND prod.filetypeid=f.filetypeid %s"
            % (tables, condition)
        )
        return self.dbR_.query(command)

    #############################################################################
    def getLimitedFiles(
        self,
        configName,
        configVersion,
        conddescription=default,
        processing=default,
        evt=default,
        production=default,
        filetype=default,
        quality=default,
        runnb=default,
        startitem=0,
        maxitems=10,
    ):
        """For retrieving a subset of files.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: simulation or data taking condition
        :param str processing: processing pass
        :param int evt: event type
        :param int production: production number
        :param str filetype: file type
        :param str quality: data quality
        :param int runnb: run number
        :param int startitem: staring row number
        :pram int maxitems: maximum returned rows
        :return: a list of limited number of files
        """

        tables = " files f, jobs j, productionoutputfiles prod, productionscontainer cont, filetypes ft, dataquality d "
        condition = (
            " AND cont.production=prod.production AND d.qualityid=f.qualityid AND \
j.production=prod.production AND j.stepid=prod.stepid AND prod.eventtypeid=f.eventtypeid %s "
            % self.__buildVisible(visible="Y", replicaFlag="Yes")
        )

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        retVal = self._buildConditions(default, conddescription, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildProduction(production, condition, tables, useMainTables=False)

        condition = self.__buildReplicaflag("Yes", condition)

        retVal = self.__buildDataquality(quality, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        retVal = self.__buildProcessingPass(processing, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        condition, tables = self.__buildEventType(evt, condition, tables, useMainTables=False)

        condition, tables = self.__buildFileTypes(filetype, condition, tables, useMainTables=False)

        retVal = self._buildRunnumbers(runnb, None, None, condition, tables)
        if not retVal["OK"]:
            return retVal
        condition, tables = retVal["Value"]

        # TODO: Distinct is being used here to work around https://its.cern.ch/jira/browse/LHCBDIRAC-895
        command = (
            "SELECT DISTINCT fname, fstat, fsize, fcreation, jstat, jend, jnode, ftypen, evttypeid, \
jrun, jfill, ffull, dflag,   jevent, jtotal, flum, finst, jtck FROM \
(SELECT rownum r, fname, fstat, fsize, fcreation, jstat, jend, jnode, ftypen, \
evttypeid, jrun, jfill, ffull, dflag,   jevent, jtotal, flum, finst, jtck FROM \
(SELECT ROWNUM r, f.FileName fname, f.EventStat fstat, f.FileSize fsize, \
f.CreationDate fcreation, j.JobStart jstat, j.JobEnd jend, j.WorkerNode jnode, \
ft.Name ftypen, f.eventtypeid evttypeid, j.runnumber jrun, j.fillnumber jfill, \
f.fullstat ffull, d.dataqualityflag dflag,j.eventinputstat jevent, j.totalluminosity jtotal, \
f.luminosity flum, f.instLuminosity finst, j.tck jtck, j.WNMJFHS06,j.HLT2TCK, \
j.NumberOfProcessors FROM %s WHERE j.jobid=f.jobid AND \
ft.filetypeid=prod.filetypeid AND f.filetypeid=prod.filetypeid AND f.gotreplica='Yes' AND f.visibilityflag='Y' %s) WHERE \
rownum <=%d ) WHERE r >%d"
            % (tables, condition, int(maxitems), int(startitem))
        )
        return self.dbR_.query(command)

    #############################################################################
    def getDataTakingCondId(self, condition):
        """For retrieving the data quality id.

        :param dict condition: data taking attributes
        :return: the data taking conditions identifier
        """
        command = "SELECT DaqPeriodId FROM data_taking_conditions WHERE "
        for param in condition:
            if isinstance(condition[param], str) and not condition[param].strip():
                command += str(param) + " is NULL AND "
            elif condition[param] is not None:
                command += str(param) + "='" + condition[param] + "' AND "
            else:
                command += str(param) + " is NULL AND "

        command = command[:-4]
        res = self.dbR_.query(command)
        if res["OK"]:
            if not res["Value"]:
                command = "SELECT DaqPeriodId FROM data_taking_conditions WHERE "
                for param in condition:
                    if param != "Description":
                        if isinstance(condition[param], str) and not condition[param].strip():
                            command += str(param) + " is NULL AND "
                        elif condition[param] is not None:
                            command += str(param) + "='" + condition[param] + "' AND "
                        else:
                            command += str(param) + " is NULL AND "

                command = command[:-4]
                retVal = self.dbR_.query(command)
                if retVal["OK"]:
                    if retVal["Value"]:
                        return S_ERROR(
                            "Only the Description is different, the other attributes are the same and they are exists in the DB!"
                        )
        return res

    #############################################################################
    def getDataTakingCondDesc(self, condition):
        """For retrieving the data taking conditions which fullfill for given
        condition.

        :param dict condition: data taking attributes
        :return: the data taking description which adequate a given conditions.
        """
        command = "SELECT description FROM data_taking_conditions WHERE "
        for param in condition:
            if isinstance(condition[param], str) and not condition[param].strip():
                command += str(param) + " is NULL and "
            elif condition[param] is not None:
                command += str(param) + "='" + condition[param] + "' and "
            else:
                command += str(param) + " is NULL and "

        command = command[:-4]
        res = self.dbR_.query(command)
        if res["OK"]:
            if not res["Value"]:
                command = "SELECT DaqPeriodId FROM data_taking_conditions WHERE "
                for param in condition:
                    if param != "Description":
                        if isinstance(condition[param], str) and not condition[param].strip():
                            command += str(param) + " is NULL and "
                        elif condition[param] is not None:
                            command += str(param) + "='" + condition[param] + "' and "
                        else:
                            command += str(param) + " is NULL and "

                command = command[:-4]
                retVal = self.dbR_.query(command)
                if retVal["OK"]:
                    if retVal["Value"]:
                        return S_ERROR(
                            "Only the Description is different, the other attributes are the same and they are exists in the DB!"
                        )
        return res

    #############################################################################
    def getStepIdandNameForRUN(self, programName, programVersion, conddb, dddb):
        """For retrieving the steps which is used by given application, conddb, dddb.

        :param str programName: application name
        :param str programVersion: application version
        :param str conddb: CONDB database tag
        :param str dddb: DDDB database tag
        :return: the step used to process data
        """
        dataset = {
            "Step": {
                "StepName": "Real Data",
                "ApplicationName": programName,
                "ApplicationVersion": programVersion,
                "ProcessingPass": "Real Data",
                "Visible": "Y",
                "CONDDB": None,
                "DDDB": None,
            },
            "OutputFileTypes": [{"FileType": "RAW", "Visible": "Y"}],
        }
        condition = ""
        if conddb is None or conddb == "":
            condition += " and CondDB is NULL "
            dataset["Step"].pop("CONDDB")
        else:
            condition += f" and CondDB='{conddb}' "
            dataset["Step"]["CONDDB"] = conddb

        if dddb is None or dddb == "":
            condition += " and DDDB is NULL "
            dataset["Step"].pop("DDDB")
        else:
            condition += f" and DDDB='{dddb}'"
            dataset["Step"]["DDDB"] = dddb

        command = (
            "SELECT stepid, stepname from steps where applicationname='%s' \
    and applicationversion='%s' %s "
            % (programName, programVersion, condition)
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        if not retVal["Value"]:
            retVal = self.insertStep(dataset)
            if retVal["OK"]:
                return S_OK([retVal["Value"], "Real Data"])
            else:
                return retVal
        else:
            return S_OK([retVal["Value"][0][0], retVal["Value"][0][1]])

    #############################################################################
    def __getPassIds(self, name):
        """For retrieving processing pass ids.

        :param str name: processing pass name for example: Sim10
        :return: the processing pass ids for a given processing pass name
        """
        command = f"SELECT id from processing where name='{name}'"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        return S_OK([i[0] for i in retVal["Value"]])

    #############################################################################
    def __getprocessingid(self, processingpassid):
        """For retrieving processing pass for a given id.

        :param int processongpassid: processing pass id
        :return: processing pass
        """
        command = (
            'SELECT name "Name", CONNECT_BY_ISCYCLE "Cycle", \
   LEVEL, SYS_CONNECT_BY_PATH(name, \'/\') "Path", id "ID" \
   FROM processing \
   START WITH id='
            + str(processingpassid)
            + '\
   CONNECT BY NOCYCLE PRIOR  parentid=id AND LEVEL <= 5 \
   ORDER BY  Level desc, "Name", "Cycle", "Path"'
        )
        return self.dbR_.query(command)

    #############################################################################
    @staticmethod
    def __checkprocessingpass(opath, values):
        """checks the processing pass: compare the processing passes.

        :param list opath: processing pass names
        :param list values: processing pass names
        """
        if len(opath) != len(values):
            return False
        j = 0
        for i in values:
            if i[0] != opath[j]:
                return False
            j += 1
        return True

    #############################################################################
    def __insertprocessing(self, values, parentid=None, ids=None):
        """inserts a processing pass.

        :param list values: processing pass names: Reco09, Stripping19
        :param int parentid: the parent processing pass
        :param list ids: keeps all processing pass ids
        """
        if ids is None:
            ids = []
        for i in values:
            command = ""
            if parentid is not None:
                command = f"SELECT id from processing WHERE name='{i}' AND parentid={parentid}"
            else:
                command = f"SELECT id from processing WHERE name='{i}' AND parentid is null"
            retVal = self.dbR_.query(command)
            if retVal["OK"]:
                if not retVal["Value"]:
                    if parentid is not None:
                        command = "SELECT max(id)+1 from processing"
                        retVal = self.dbR_.query(command)
                        if retVal["OK"]:
                            processingpassid = retVal["Value"][0][0]
                            ids += [processingpassid]
                            command = "insert into processing(id,parentid,name)values(%d,%d,'%s')" % (
                                processingpassid,
                                parentid,
                                i,
                            )
                            retVal = self.dbW_.query(command)
                            if not retVal["OK"]:
                                self.log.error(retVal["Message"])
                            values.remove(i)
                            self.__insertprocessing(values, processingpassid, ids)
                    else:
                        command = "SELECT max(id)+1 FROM processing"
                        retVal = self.dbR_.query(command)
                        if retVal["OK"]:
                            processingpassid = retVal["Value"][0][0]
                            if processingpassid is None:
                                processingpassid = 1
                            ids += [processingpassid]
                            command = "insert into processing(id,parentid,name)values(%d,null,'%s')" % (
                                processingpassid,
                                i,
                            )
                            retVal = self.dbW_.query(command)
                            if not retVal["OK"]:
                                self.log.error(retVal["Message"])
                            values.remove(i)
                            self.__insertprocessing(values, processingpassid, ids)
                else:
                    values.remove(i)
                    parentid = retVal["Value"][0][0]
                    ids += [parentid]
                    self.__insertprocessing(values, parentid, ids)

    #############################################################################
    def addProcessing(self, path):
        """adds a new processing pass.

        :param str path: processing pass for example: /Real Data/Reco19/Striping29
        """
        lastindex = len(path) - 1
        retVal = self.__getPassIds(path[lastindex])
        stepids = []
        if not retVal["OK"]:
            return retVal

        ids = retVal["Value"]
        if len(ids) == 0:
            newpath = list(path)
            self.__insertprocessing(newpath, None, stepids)
            return S_OK(stepids[-1:])
        else:
            for i in ids:
                procs = self.__getprocessingid(i)
                if len(procs) > 0:
                    if self.__checkprocessingpass(path, procs):
                        return S_OK()
            newpath = list(path)
            self.__insertprocessing(newpath, None, stepids)
            return S_OK(stepids[-1:])

    #############################################################################
    def insertproductionscontainer(self, prod, processingid, simid, daqperiodid, configName, configVersion):
        """inserts a production to the productions container.

        :param int prod: production number
        :param int processingid: processing pass id
        :param int simid: simulation condition id
        :param int daqperiodid: data taking condition id
        :param str configName: configuration name
        :param str configVersion: configuration version
        """
        return self.dbW_.executeStoredProcedure(
            "BOOKKEEPINGORACLEDB.insertproductionscontainer",
            [prod, processingid, simid, daqperiodid, configName, configVersion],
            False,
        )

    #############################################################################
    def addProductionSteps(self, steps, prod):
        """adds a step to a production. The steps which used by the production.

        :param list steps: list of dict of steps [{'StepId':123}, {'StepId':321}]
        :param int prod: production number
        """
        level = 1
        for step in steps:
            retVal = self.dbW_.executeStoredProcedure(
                "BOOKKEEPINGORACLEDB.insertStepsContainer", [prod, step["StepId"], level], False
            )
            if not retVal["OK"]:
                return retVal
            level += 1
        return S_OK()

    #############################################################################
    def checkProcessingPassAndSimCond(self, production):
        """checks the processing pass and simulation condition.

        :param int production: production number
        """
        return self.dbR_.query("SELECT count(*) FROM productionscontainer WHERE production=" + str(production))

    #############################################################################
    def addProduction(
        self,
        production,
        simcond=None,
        daq=None,
        steps=default,
        inputproc="",
        configName=None,
        configVersion=None,
        eventType=None,
    ):
        """adds a production to the productions container table.

        :param int production: production number
        :param str simcond: simulation condition description
        :param str daq: data taking description
        :param list steps: list of dictionaries of steps (min fields {'Visible': 'Y/N', 'StepID': '123'})
        :param str inputproc: input processing pass
        :param str configName: configuration name
        :param str configVersion: configuration version
        :param int eventType: eventTyoe
        """
        self.log.verbose("Adding production", production)
        path = []
        if inputproc != "":
            if inputproc[0] != "/":
                inputproc = "/" + inputproc
            path = inputproc.split("/")[1:]

        for step in steps:
            if step["Visible"] == "Y":
                res = self.getAvailableSteps({"StepId": step["StepId"]})
                if not res["OK"]:
                    self.log.error(res["Message"])
                    return res
                if res["Value"]["TotalRecords"] > 0:
                    procpas = res["Value"]["Records"][0][9]
                    path += [procpas]
                else:
                    self.log.error("Missing step", f"(StepID: {step['StepId']})")
                    return S_ERROR("Missing step")

        if not path:
            self.log.error("You have to define the input processing pass or you have to have a visible step!")
            return S_ERROR("You have to define the input processing pass or you have to have a visible step!")
        processingid = None
        retVal = self.addProcessing(path)
        if not retVal["OK"]:
            self.log.error("Failed adding processing", path)
            return retVal
        if not retVal["Value"]:
            return S_ERROR("The processing pass already exists! Write to lhcb-bookkeeping@cern.ch")
        processingid = retVal["Value"][0]
        retVal = self.addProductionSteps(steps, production)
        if not retVal["OK"]:
            return retVal

        sim = None
        did = None
        if daq is not None:
            retVal = self._getDataTakingConditionId(daq)
            if not retVal["OK"]:
                return retVal
            if retVal["Value"] > -1:
                did = retVal["Value"]
            else:
                return S_ERROR("Data taking condition is missing")
        if simcond is not None:
            retVal = self.__getSimulationConditionId(simcond)
            if not retVal["OK"]:
                return retVal
            if retVal["Value"] == -1:
                return S_ERROR("Simulation condition is missing")
            sim = retVal["Value"]
        retVal = self.insertproductionscontainer(production, processingid, sim, did, configName, configVersion)
        if not retVal["OK"]:
            return retVal

        return self.insertProductionOutputFiletypes(production, steps, eventType)

    #############################################################################
    def insertProductionOutputFiletypes(self, production, steps, eventType):
        """This method is used to register the output filetypes for a given
        production.

        :param int production: it is the production number
        :param list steps: it contains all the steps and output file types
        :param number/list eventtype: given event type which will be produced by the jobs
        :returns: S_OK/S_ERROR
        """
        # if we have some specific file type version, it can be added to this dictionary
        fileTypeMap = {"RAW": "MDF"}
        eventtypes = []
        if eventType:
            if isinstance(eventType, (str, int)):
                eventtypes.append(int(eventType))
            elif isinstance(eventType, list):
                eventtypes = eventType
            else:
                return S_ERROR(f"{eventType} event type is not valid!")
        self.log.verbose("The following event types will be inserted:", f"{eventtypes}")

        for step in steps:
            # the runs have more than one event type
            for eventtype in eventtypes:
                for ftype in step.get("OutputFileTypes", {}):
                    fversion = fileTypeMap.get(ftype.get("FileType"), "ROOT")
                    result = self.checkFileTypeAndVersion(ftype.get("FileType"), fversion)
                    if not result["OK"]:
                        return S_ERROR(f"The type:{ftype.get('FileType')}, version:{fversion} is missing.")
                    else:
                        fileTypeid = int(result["Value"])
                    retVal = self.dbW_.executeStoredProcedure(
                        "BOOKKEEPINGORACLEDB.insertProdnOutputFtypes",
                        [production, step["StepId"], fileTypeid, ftype.get("Visible", "Y"), eventtype],
                        False,
                    )
                    if not retVal["OK"]:
                        return retVal

        return S_OK()

    #############################################################################
    def getEventTypes(self, configName=default, configVersion=default, prod=default):
        """For retrieving the event type for given conditions.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param int prod: production number
        :return: event types
        """

        tables = "productionoutputfiles prod, productionscontainer cont, eventtypes e, configurations c "
        condition = " cont.production=prod.production and prod.eventtypeid=e.eventtypeid %s " % self.__buildVisible(
            visible="Y", replicaFlag="Yes"
        )

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        condition, tables = self.__buildProduction(prod, condition, tables, useMainTables=False)

        command = "SELECT e.eventtypeid, e.description FROM  {} WHERE {} GROUP BY e.eventtypeid, e.description".format(
            tables,
            condition,
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        records = [list(record) for record in retVal["Value"]]
        return S_OK({"ParameterNames": ["EventType", "Description"], "Records": records, "TotalRecords": len(records)})

    #############################################################################
    def getProcessingPassSteps(self, procpass=default, cond=default, stepname=default):
        """For retrieving the step metadata for given condition.

        :param str procpass: processing pass
        :param str cond: data taking or simulation condition
        :param str stepname: name of the step
        :return: the steps with metadata
        """
        processing = {}
        condition = ""

        if procpass != default:
            condition += (
                " AND prod.processingid in"
                "(SELECT v.id FROM (SELECT DISTINCT SYS_CONNECT_BY_PATH(name, '/') Path, id ID "
                "FROM processing v START WITH id in "
                "(SELECT DISTINCT id FROM processing WHERE name='%s') "
                "CONNECT BY NOCYCLE PRIOR  id=parentid) v WHERE v.path='%s')" % (procpass.split("/")[1], procpass)
            )

        if cond != default:
            retVal = self.__getConditionString(cond, "prod")
            if retVal["OK"]:
                condition += retVal["Value"]
            else:
                return retVal

        if stepname != default:
            condition += f" AND s.processingpass='{stepname}' "

        command = (
            "SELECT DISTINCT s.stepid,s.stepname,s.applicationname,s.applicationversion, "
            "s.optionfiles,s.dddb, s.conddb,s.extrapackages,s.visible, cont.step FROM "
            "steps s, productionscontainer prod, stepscontainer cont WHERE "
            "cont.stepid=s.stepid AND prod.production=cont.production %s ORDER BY cont.step" % (condition)
        )

        retVal = self.dbR_.query(command)
        records = []
        # parametersNames = [ 'StepId', 'StepName','ApplicationName', 'ApplicationVersion',
        # 'OptionFiles','DDDB','CONDDB','ExtraPackages','Visible']
        parametersNames = ["id", "name"]
        if retVal["OK"]:
            nb = 0
            for i in retVal["Value"]:
                # records = [[i[0],i[1],i[2],i[3],i[4],i[5],i[6], i[7], i[8]]]
                records = [
                    ["StepId", i[0]],
                    ["StepName", i[1]],
                    ["ApplicationName", i[2]],
                    ["ApplicationVersion", i[3]],
                    ["OptionFiles", i[4]],
                    ["DDDB", i[5]],
                    ["CONDDB", i[6]],
                    ["ExtraPackages", i[7]],
                    ["Visible", i[8]],
                ]
                step = f"Step-{i[0]}"
                processing[step] = records
                nb += 1
        else:
            return retVal

        return S_OK({"Parameters": parametersNames, "Records": processing, "TotalRecords": nb})

    #############################################################################
    def getProductionProcessingPassSteps(self, prod):
        """For retrieving the processing pass of a given production.

        :param int prod: production number
        :return: the production processing pass
        """
        processing = {}
        retVal = self.getProductionProcessingPass(prod)
        if not retVal["OK"]:
            return retVal
        procpass = retVal["Value"]

        condition = ""

        if procpass != default:
            condition += (
                " AND prod.processingid in "
                "(SELECT v.id from (SELECT DISTINCT SYS_CONNECT_BY_PATH(name, '/') Path, id ID "
                "FROM processing v START WITH id in (SELECT DISTINCT id from processing where name='%s') "
                "CONNECT BY NOCYCLE PRIOR  id=parentid) v where v.path='%s')" % (procpass.split("/")[1], procpass)
            )

        command = (
            "SELECT DISTINCT s.stepid,s.stepname,s.applicationname,s.applicationversion, "
            "s.optionfiles,s.dddb, s.conddb,s.extrapackages,s.visible, cont.step "
            "FROM steps s, productionscontainer prod, stepscontainer cont "
            "WHERE cont.stepid=s.stepid AND prod.production=cont.production %s AND prod.production=%d ORDER BY cont.step"
            % (condition, prod)
        )

        retVal = self.dbR_.query(command)
        records = []
        # parametersNames = [ 'StepId', 'StepName','ApplicationName',
        # 'ApplicationVersion','OptionFiles','DDDB','CONDDB','ExtraPackages','Visible']
        parametersNames = ["id", "name"]
        if retVal["OK"]:
            nb = 0
            for i in retVal["Value"]:
                # records = [[i[0],i[1],i[2],i[3],i[4],i[5],i[6], i[7], i[8]]]
                records = [
                    ["StepId", i[0]],
                    ["ProcessingPass", procpass],
                    ["ApplicationName", i[2]],
                    ["ApplicationVersion", i[3]],
                    ["OptionFiles", i[4]],
                    ["DDDB", i[5]],
                    ["CONDDB", i[6]],
                    ["ExtraPackages", i[7]],
                    ["Visible", i[8]],
                ]
                step = i[1]
                processing[step] = records
                nb += 1
        else:
            return retVal

        return S_OK({"Parameters": parametersNames, "Records": processing, "TotalRecords": nb})

    #############################################################################
    def getRuns(self, cName, cVersion):
        """For retrieving list of runs.

        :param str cName: configuration name
        :param str cVersion: configuration version
        :return: runs
        """
        return self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.getRuns", [cName, cVersion])

    #############################################################################
    def getRunAndProcessingPass(self, runnb):
        """For retrieving the processing pass of a given run.

        :param int runnb: run number
        :return: the processing pass of a run
        """
        command = "SELECT DISTINCT runnumber, processingpass from table (BOOKKEEPINGORACLEDB.getRunProcPass(%d))" % (
            runnb
        )
        return self.dbR_.query(command)
        # return self.dbR_.executeStoredProcedure('BOOKKEEPINGORACLEDB.getRunProcPass', [runnb])

    #############################################################################
    def getNbOfRawFiles(self, runid, eventtype, replicaFlag="Yes", visible="Y", isFinished=default):
        """For retrieving the number of raw files for a given condition.

        :param int runid: run number
        :param int eventtype: event type
        :param str replicaFlag: file replica flag
        :param str visible: file visibility flag
        :param str isFinished: the run status
        :retun: the number of raw files
        """
        condition = ""
        tables = "jobs, files"
        if eventtype != default:
            condition = " AND files.eventtypeid=%d" % (eventtype)

        if visible != default:
            condition += f" AND files.visibilityFlag='{visible}'"

        if replicaFlag != default:
            condition += f" AND files.gotreplica='{replicaFlag}'"

        if isFinished != default:
            tables += " , runstatus"
            condition += f" AND jobs.runnumber=runstatus.runnumber and runstatus.finished='{isFinished}' "

        command = (
            "SELECT COUNT(*) FROM %s WHERE jobs.jobid=files.jobid AND jobs.production<0 AND jobs.runnumber=%d %s"
            % (tables, runid, condition)
        )
        return self.dbR_.query(command)

    #############################################################################
    def getFileTypeVersion(self, lfns):
        """For retrieving the file type version.

        :param list lfns: list of lfns
        :return: the format of an lfn
        """
        retVal = self.dbR_.executeStoredProcedure("BOOKKEEPINGORACLEDB.bulkgetTypeVesrsion", [], True, lfns)
        if not retVal["OK"]:
            return retVal
        values = {}
        for i in retVal["Value"]:
            values[i[0]] = i[1]
        return S_OK(values)

    #############################################################################
    def insertRuntimeProject(self, projectid, runtimeprojectid):
        """inserts a runtime project.

        :param int projectid: run time project stepid
        :param int runtimeprojectid: reference to other step
        """
        return self.dbW_.executeStoredProcedure(
            "BOOKKEEPINGORACLEDB.insertRuntimeProject", [projectid, runtimeprojectid], False
        )

    #############################################################################
    def updateRuntimeProject(self, projectid, runtimeprojectid):
        """changes the runtime project.

        :param int projectid: run time project stepid
        :param int runtimeprojectid: new run time project stepid (new reference to a stepid)
        """
        return self.dbW_.executeStoredProcedure(
            "BOOKKEEPINGORACLEDB.updateRuntimeProject", [projectid, runtimeprojectid], False
        )

    def removeRuntimeProject(self, stepid):
        """removes the runtime project.

        :param int stepid: step id
        """
        return self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.removeRuntimeProject", [stepid], False)

    #############################################################################
    def getTCKs(
        self,
        configName,
        configVersion,
        conddescription=default,
        processing=default,
        evt=default,
        production=default,
        filetype=default,
        quality=default,
        runnb=default,
    ):
        """TCKs for a given data set.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: data taking condition
        :param str processing: processing pass
        :param int evt: event type
        :param int production: production number
        :param str filetype: file type
        :param str quality: data quality
        :param int runnb: run number
        :return: the TCKs for a given dataset
        """

        return self.getFilesWithMetadata(
            configName=configName,
            configVersion=configVersion,
            conddescription=conddescription,
            processing=processing,
            evt=evt,
            production=production,
            filetype=filetype,
            quality=quality,
            visible="Y",
            replicaflag="Yes",
            runnumbers=runnb,
            selection=" DISTINCT j.tck ",
        )

    #############################################################################
    def __prepareStepMetadata(
        self,
        configName,
        configVersion,
        cond=default,
        procpass=default,
        evt=default,
        production=default,
        filetype=default,
        runnb=default,
        visible=default,
        replica=default,
        selection="",
    ):
        """it generates the sql command depending on the selection.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str procpass: processing pass
        :param int evt: event type
        :param int production: production number
        :param str filetype: file type
        :param int runnb: run number
        :param str selection: select state
        :return: sql command
        """
        condition = ""
        tables = (
            "steps s, productionscontainer cont, stepscontainer scont, productionoutputfiles prod, configurations c"
        )
        if configName != default:
            condition += f" AND c.configname='{configName}' "

        if configVersion != default:
            condition += f" AND c.configversion='{configVersion}' "

        if procpass != default:
            condition += (
                " AND cont.processingid IN (\
SELECT v.id from (SELECT DISTINCT SYS_CONNECT_BY_PATH(name, '/') Path, id ID \
FROM processing v START WITH id IN (SELECT DISTINCT id FROM processing WHERE name='%s') \
CONNECT BY NOCYCLE PRIOR  id=parentid) v WHERE v.path='%s' \
                       )"
                % (procpass.split("/")[1], procpass)
            )

        if cond != default:
            retVal = self.__getConditionString(cond, "cont")
            if not retVal["OK"]:
                return retVal
            condition += retVal["Value"]

        if evt != default:
            condition += f"  AND prod.eventtypeid={str(evt)} "

        if production != default:
            condition += " AND prod.production=" + str(production)

        if runnb != default:
            tables += " ,prodrunview rview"
            condition += " AND rview.production=prod.production AND rview.runnumber=%d AND prod.production<0" % (runnb)

        if filetype != default:
            tables += ", filetypes ftypes"
            condition += f" AND ftypes.name='{filetype}' AND prod.filetypeid=ftypes.filetypeid "

        if visible != default:
            condition += f" AND prod.visible='{visible}'"

        if replica != default:
            condition += f" AND prod.gotreplica='{replica}'"

        command = (
            "SELECT %s FROM  %s WHERE "
            "scont.stepid=s.stepid AND cont.production=prod.production AND "
            "c.configurationid=cont.configurationid AND prod.production=scont.production %s "
            "ORDER BY scont.step" % (selection, tables, condition)
        )
        return command

    #############################################################################
    def getStepsMetadata(
        self,
        configName,
        configVersion,
        cond=default,
        procpass=default,
        evt=default,
        production=default,
        filetype=default,
        runnb=default,
    ):
        """Step metadata, which describes how the data set is created.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str procpass: processing pass
        :param int evt: event type
        :param int production: production number
        :param str filetype: file type
        :param int runnb: run number
        :return: the steps with metadata
        """

        command = None
        processing = {}
        productions = None
        result = None
        if "MC" in configName.upper():
            command = self.__prepareStepMetadata(
                configName,
                configVersion,
                cond,
                procpass,
                evt,
                production,
                filetype,
                runnb,
                "Y",
                "Yes",
                selection="prod.production",
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                result = retVal
            else:
                productions = {i[0] for i in retVal["Value"]}
                self.log.debug("Productions:", f"{str(productions)}")
                parametersNames = ["id", "name"]
                for prod in productions:
                    retVal = self.getSteps(prod)
                    if not retVal:
                        result = retVal
                    else:
                        nb = 0
                        steps = retVal["Value"]
                        for (
                            stepName,
                            appName,
                            appVersion,
                            optionFiles,
                            dddb,
                            conddb,
                            extrapackages,
                            stepid,
                            visible,
                        ) in steps:
                            records = [
                                ["StepId", stepid],
                                ["StepName", stepName],
                                ["ApplicationName", appName],
                                ["ApplicationVersion", appVersion],
                                ["OptionFiles", optionFiles],
                                ["DDDB", dddb],
                                ["CONDDB", conddb],
                                ["ExtraPackages", extrapackages],
                                ["Visible", visible],
                            ]
                            step = "Step-%d" % stepid
                            processing[step] = records
                            nb += 1
                        result = S_OK({"Parameters": parametersNames, "Records": processing, "TotalRecords": nb})
        else:
            # #Now we are getting the metadata for a given run
            command = self.__prepareStepMetadata(
                configName,
                configVersion,
                cond,
                procpass,
                evt,
                production,
                filetype,
                runnb,
                "Y",
                "Yes",
                selection="DISTINCT s.stepid,s.stepname,s.applicationname, \
s.applicationversion,s.optionfiles,s.dddb, s.conddb,s.extrapackages,s.visible, scont.step",
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                result = retVal
            else:
                nb = 0
                parametersNames = ["id", "name"]
                steps = retVal["Value"]
                for (
                    stepid,
                    stepName,
                    appName,
                    appVersion,
                    optionFiles,
                    dddb,
                    conddb,
                    extrapackages,
                    visible,
                    _,
                ) in steps:
                    records = [
                        ["StepId", stepid],
                        ["StepName", stepName],
                        ["ApplicationName", appName],
                        ["ApplicationVersion", appVersion],
                        ["OptionFiles", optionFiles],
                        ["DDDB", dddb],
                        ["CONDDB", conddb],
                        ["ExtraPackages", extrapackages],
                        ["Visible", visible],
                    ]
                    step = "Step-%d" % stepid
                    nb += 1
                    processing[step] = records

                result = S_OK({"Parameters": parametersNames, "Records": processing, "TotalRecords": nb})

        return result

    #############################################################################
    def getDirectoryMetadata(self, lfn):
        """For retrieving the directory metadata.

        :param list lfn: list of lfns: for example: ['/lhcb/MC/2016/LOG/00057824/0010/']
        :return: a directory meradata
        """

        self.log.verbose("Getting directory metadata:", f"{lfn}")
        lfns = [i + "%" for i in lfn]
        retVal = self.dbR_.executeStoredProcedure(
            packageName="BOOKKEEPINGORACLEDB.getDirectoryMetadata_new", parameters=[], output=True, array=lfns
        )
        if not retVal["OK"]:
            return retVal

        records = {}
        failed = []
        for i in retVal["Value"]:
            fileName = i[0][:-1]
            if fileName in records:
                records[fileName] += [
                    dict(
                        zip(
                            (
                                "Production",
                                "ConfigName",
                                "ConfigVersion",
                                "EventType",
                                "FileType",
                                "ProcessingPass",
                                "ConditionDescription",
                                "VisibilityFlag",
                            ),
                            i[1:],
                        )
                    )
                ]
            else:
                records[fileName] = [
                    dict(
                        zip(
                            (
                                "Production",
                                "ConfigName",
                                "ConfigVersion",
                                "EventType",
                                "FileType",
                                "ProcessingPass",
                                "ConditionDescription",
                                "VisibilityFlag",
                            ),
                            i[1:],
                        )
                    )
                ]
        failed = [i[:-1] for i in lfns if i[:-1] not in records]
        return S_OK({"Successful": records, "Failed": failed})

    #############################################################################
    def getFilesForGUID(self, guid):
        """For retrieving the file for a given guid.

        :param str guid: file GUID
        :return: the file for a given GUID
        """
        return self.dbW_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getFilesForGUID", str, [guid])

    #############################################################################
    def getRunsGroupedByDataTaking(self):
        """For retrieving all runs grouped by data taking description.

        :return: the runs data taking description and production
        """
        command = "SELECT d.description, r.runnumber, r.production FROM \
prodrunview r, productionoutputfiles p, data_taking_conditions d, productionscontainer cont WHERE \
d.daqperiodid=cont.daqperiodid AND p.production=r.production AND cont.production=p.production \
GROUP BY d.description, r.runnumber, r.production ORDER BY r.runnumber"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        values = {}
        for i in retVal["Value"]:
            rnb = i[1]
            desc = i[0]
            prod = i[2]
            if desc in values:
                if rnb in values[desc]:
                    if prod > 0:
                        values[desc][rnb] += [prod]
                else:
                    if prod > 0:
                        values[desc].update({rnb: [prod]})
                    else:
                        values[desc].update({rnb: []})
            else:
                if prod > 0:
                    values[desc] = {rnb: [prod]}
                else:
                    values[desc] = {rnb: []}
        return S_OK(values)

    #############################################################################
    def getListOfFills(self, configName=default, configVersion=default, conddescription=default):
        """It returns a list of fills for a given condition.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: data taking condition
        :return: run numbers
        """
        condition = ""
        if configName != default:
            condition += f" and c.configname='{configName}' "

        if configVersion != default:
            condition += f" and c.configversion='{configVersion}' "

        if conddescription != default:
            retVal = self.__getConditionString(conddescription, "prod")
            if not retVal["OK"]:
                return retVal
            condition += retVal["Value"]

        command = (
            "SELECT DISTINCT j.FillNumber FROM jobs j, productionscontainer prod, \
configurations c WHERE j.configurationid=c.configurationid %s AND prod.production=j.production AND j.production<0"
            % (condition)
        )
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        return S_OK([i[0] for i in retVal["Value"]])

    #############################################################################
    def getRunsForFill(self, fillid):
        """It returns a list of runs for a given FILL.

        :param int fillid: fill number
        :return: runs
        """

        command = "SELECT DISTINCT j.runnumber FROM jobs j WHERE j.production<0 AND j.fillnumber=%d" % (fillid)
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        return S_OK([i[0] for i in retVal["Value"]])

    #############################################################################
    def getListOfRuns(
        self,
        configName=default,
        configVersion=default,
        conddescription=default,
        processing=default,
        evt=default,
        quality=default,
    ):
        """For retriecing run numbers.

        :param str configName: configuration name
        :param str configVersion: configuration version
        :param str conddescription: simulation or data taking condition
        :param str processing: processing pass
        :param int evt: event type
        :param str quality: data quality
        :return: runs
        """

        return self.getFilesWithMetadata(
            configName=configName,
            configVersion=configVersion,
            conddescription=conddescription,
            processing=processing,
            evt=evt,
            quality=quality,
            visible="Y",
            replicaflag="Yes",
            selection=" DISTINCT j.runnumber ",
        )

    #############################################################################
    def getSimulationConditions(self, in_dict):
        """For retrieving the simulation conditions for a given BKQuery.

        :param dict in_dict: bookkeeping query
        :return: simulation conditions
        """
        condition = ""
        tables = " simulationconditions sim"
        paging = False
        start = in_dict.get("StartItem", default)
        maximum = in_dict.get("MaxItem", default)

        simid = in_dict.get("SimId", default)
        if simid != default:
            condition += " AND sim.simid=%d " % int(simid)

        simdesc = in_dict.get("SimDescription", default)
        if simdesc != default:
            condition += " AND sim.simdescription like '%" + simdesc + "%'"

        visible = in_dict.get("Visible", default)
        if visible != default:
            condition += f" AND sim.visible='{visible}'"

        if start != default and maximum != default:
            paging = True

        sort = in_dict.get("Sort", default)
        if sort != default:
            condition += "Order by "
            order = sort.get("Order", "Asc")
            if order.upper() not in ["ASC", "DESC"]:
                return S_ERROR("wrong sorting order!")
            items = sort.get("Items", default)
            if isinstance(items, list):
                order = ""
                for item in items:
                    order += f"sim.{item},"
                condition += f" {order[:-1]}"
            elif isinstance(items, str):
                condition += f" sim.{items} {order}"
            else:
                result = S_ERROR("SortItems is not properly defined!")
        else:
            condition += " ORDER BY sim.inserttimestamps desc"

        if paging:
            command = (
                "SELECT sim_simid, sim_simdescription, sim_beamcond, sim_beamenergy, sim_generator, \
sim_magneticfield, sim_detectorcond, sim_luminosity, sim_g4settings, sim_visible FROM \
(SELECT ROWNUM r , sim_simid, sim_simdescription, sim_beamcond, sim_beamenergy, sim_generator, \
sim_magneticfield, sim_detectorcond, sim_luminosity, sim_g4settings, sim_visible FROM \
(SELECT ROWNUM r, sim.simid sim_simid, sim.simdescription sim_simdescription, sim.beamcond \
sim_beamcond, sim.beamenergy sim_beamenergy, sim.generator sim_generator, \
sim.magneticfield sim_magneticfield, sim.detectorcond sim_detectorcond, sim.luminosity \
sim_luminosity, sim.g4settings sim_g4settings, sim.visible sim_visible \
FROM %s WHERE sim.simid=sim.simid %s) WHERE rownum <=%d) WHERE r >%d"
                % (tables, condition, maximum, start)
            )
            retVal = self.dbR_.query(command)
        else:
            command = (
                "SELECT sim.simid sim_simid, sim.simdescription sim_simdescription, sim.beamcond sim_beamcond, \
sim.beamenergy sim_beamenergy, sim.generator sim_generator, \
sim.magneticfield sim_magneticfield, sim.detectorcond sim_detectorcond, sim.luminosity sim_luminosity, \
sim.g4settings sim_g4settings, sim.visible sim_visible FROM %s WHERE sim.simid=sim.simid %s"
                % (tables, condition)
            )
            retVal = self.dbR_.query(command)

        if not retVal["OK"]:
            return retVal
        command = "SELECT COUNT(*) FROM simulationconditions"

        parameterNames = [
            "SimId",
            "SimDescription",
            "BeamCond",
            "BeamEnergy",
            "Generator",
            "MagneticField",
            "DetectorCond",
            "Luminosity",
            "G4settings",
            "Visible",
        ]
        records = [list(record) for record in retVal["Value"]]
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        totalRecords = retVal["Value"][0][0]
        return S_OK({"ParameterNames": parameterNames, "Records": records, "TotalRecords": totalRecords})

    #############################################################################
    def updateSimulationConditions(self, in_dict):
        """it updates a given simulation condition.

        :param dict in_dict: dictionary which contains the simulation conditions attributes.
        """
        simid = in_dict.get("SimId", default)
        if simid != default:
            condition = ""
            for cond in in_dict:
                if cond != "SimId":
                    condition += f"{cond}='{in_dict[cond]}',"
            condition = condition[:-1]
            command = "UPDATE simulationconditions SET %s WHERE simid=%d" % (condition, int(simid))
            return self.dbW_.query(command)
        return S_ERROR("SimId is missing!")

    #############################################################################
    def deleteSimulationConditions(self, simid):
        """it deletes a given simulation condition.

        :param int simid: simulation condition id
        """
        return self.dbW_.query("DELETE simulationconditions WHERE simid=%d" % simid)

    #############################################################################
    def getProductionSummaryFromView(self, in_dict):
        """Data set summary.

        :param dict in_dict: bookkeeping query dictionary
        """
        evt = in_dict.get("EventType", default)
        prod = in_dict.get("Production", default)
        configName = in_dict.get("ConfigName", default)
        configVersion = in_dict.get("ConfigVersion", default)

        tables = " productionoutputfiles prod, productionscontainer cont, simulationconditions sim, \
data_taking_conditions daq, configurations c "
        condition = (
            " cont.production=prod.production AND c.configurationid=cont.configurationid %s "
            % self.__buildVisible(visible="Y", replicaFlag="Yes")
        )

        condition, tables = self.__buildConfiguration(configName, configVersion, condition, tables)

        condition, tables = self.__buildProduction(prod, condition, tables, useMainTables=False)

        condition, tables = self.__buildEventType(evt, condition, tables, useMainTables=False)

        command = (
            "SELECT prod.production, prod.eventtypeid, c.configname, c.configversion, \
BOOKKEEPINGORACLEDB.getProductionProcessingPass(prod.production), sim.simdescription, daq.description \
FROM %s WHERE sim.simid(+)=cont.simid AND DAQ.daqperiodid(+)=cont.daqperiodid AND %s"
            % (tables, condition)
        )
        parameterNames = [
            "Production",
            "EventType",
            "ConfigName",
            "ConfigVersion",
            "ProcessingPass",
            "ConditionDescription",
        ]
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal
        rows = []
        for record in retVal["Value"]:
            i = dict(zip(parameterNames[:-1], record[:-2]))
            i["ConditionDescription"] = (
                record[len(parameterNames)]
                if record[len(parameterNames)] is not None
                else record[len(parameterNames) - 1]
            )
            rows += [i]
        return S_OK(rows)

    #############################################################################
    def getJobInputOutputFiles(self, diracjobids):
        """For retrieving the input and output files for jobs by a given list of
        DIRAC jobid.

        :param list diracjobids: list of DIRAC jobid
        :return: Successful: DIRAC job which has input/output
          Failed: DIRAC job which does not exists in the db.
        """
        result = {"Failed": {}, "Successful": {}}
        for diracJobid in diracjobids:
            command = (
                "SELECT j.jobid, f.filename FROM inputfiles i, files f, jobs j WHERE f.fileid=i.fileid AND \
i.jobid=j.jobid and j.diracjobid=%d ORDER BY j.jobid, f.filename"
                % int(diracJobid)
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                result["Failed"][diracJobid] = retVal["Message"]
            result["Successful"][diracJobid] = {}
            result["Successful"][diracJobid]["InputFiles"] = []
            for i in retVal["Value"]:
                result["Successful"][diracJobid]["InputFiles"] += [i[1]]

            command = (
                "SELECT j.jobid, f.filename FROM jobs j, files f WHERE j.jobid=f.jobid AND \
diracjobid=%d ORDER BY j.jobid, f.filename"
                % int(diracJobid)
            )
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                result["Failed"][diracJobid] = retVal["Message"]
            result["Successful"][diracJobid]["OutputFiles"] = []
            for i in retVal["Value"]:
                result["Successful"][diracJobid]["OutputFiles"] += [i[1]]
        return S_OK(result)

    #############################################################################
    def insertRunStatus(self, runnumber, jobId, isFinished="N"):
        """inserts the run status of a give run.

        :param int runnumber: run number
        :param int jobId: internal bookkeeping job id
        :param str isFinished: the run is not finished by default
        """
        return self.dbW_.executeStoredProcedure(
            "BOOKKEEPINGORACLEDB.insertRunStatus", [runnumber, jobId, isFinished], False
        )

    #############################################################################
    def setRunStatusFinished(self, runnumber, isFinished):
        """Set the run status.

        :param int runnumber: run number
        :param str isFinished: 'Y' if it is finished otherwise 'N'
        """
        result = self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.setRunFinished", [runnumber, isFinished], False)
        if not result["OK"]:
            return result
        return S_OK(f"{str(runnumber)} is finished")

    #############################################################################
    def getRunStatus(self, runnumbers):
        """For retrieving the run status.

        :param list runnumbers: list of runs.
        :return: dictionary which contains the failed runs with the result and sucessful run
        """
        status = {}
        params = ["Finished"]
        status["Successful"] = {}
        status["Failed"] = []
        for i in runnumbers:
            command = "SELECT Finished FROM runstatus WHERE runnumber=%d" % i
            retVal = self.dbR_.query(command)
            if not retVal["OK"]:
                self.log.error(i, retVal["Message"])
                status["Failed"] += [i]
            else:
                if len(retVal["Value"]) > 0:
                    status["Successful"][i] = dict(zip(params, retVal["Value"][0]))
                else:
                    status["Failed"] += [i]
        return S_OK(status)

    #############################################################################
    def fixRunLuminosity(self, runnumbers):
        """Fix run luminosity for run filea and also all descendants.

        :param list runnumbers: list of run numbers
        :return: Successful runs and Failed runs
        """
        status = {"Failed": [], "Successful": []}
        for run in runnumbers:
            result = self.dbW_.executeStoredProcedure("BOOKKEEPINGORACLEDB.updateLuminosity", [run], False)
            if result["OK"]:
                status["Successful"] += [run]
            else:
                status["Failed"] += [run]
        return S_OK(status)

    #############################################################################
    def getProductionProducedEvents(self, prodid):
        """the produced event by a production taking into account the step.

        :param int prodid: production number
        :return: produced events
        """
        return self.dbR_.executeStoredFunctions("BOOKKEEPINGORACLEDB.getProducedEvents", int, [prodid])

    #############################################################################
    def bulkinsertEventType(self, eventtypes):
        """It inserts a list of event types to the db.

        :param list eventtypes it inserts a list of event types. For example: the list elements are the following:

          .. code-block:: python

            {'EVTTYPEID': '12265021',
             'DESCRIPTION': 'Bu_D0pipipi,Kpi-withf2=DecProdCut_pCut1600MeV',
             'PRIMARY': '[B+ -> (D~0 -> K+ pi-) pi+ pi- pi+]cc'}

        :returns: S_ERROR S_OK({'Failed':[],'Successful':[]})
        """
        failed = []
        for evt in eventtypes:
            evtId = evt.get("EVTTYPEID")
            evtDesc = evt.get("DESCRIPTION")
            evtPrimary = evt.get("PRIMARY")
            retVal = self.insertEventTypes(evtId, evtDesc, evtPrimary)
            if not retVal["OK"]:
                failed.append({evtId: {"Error": retVal["Message"], "EvtentType": evt}})

        successful = list({evt["EVTTYPEID"] for evt in eventtypes} - {list(i)[0] for i in failed})
        return S_OK({"Failed": failed, "Successful": successful})

    #############################################################################
    def bulkupdateEventType(self, eventtypes):
        """It updates a list of event types which are exist in the db.

        :param list eventtypes it is a list of event types. For example: the list elements are the following:

          .. code-block:: python

            {'EVTTYPEID': '12265021',
             'DESCRIPTION': 'Bu_D0pipipi,Kpi-withf2=DecProdCut_pCut1600MeV',
             'PRIMARY': '[B+ -> (D~0 -> K+ pi-) pi+ pi- pi+]cc'}

        :returns: S_ERROR S_OK({'Failed':[],'Successful':[]})
        """
        failed = []
        for evt in eventtypes:
            evtId = evt.get("EVTTYPEID")
            evtDesc = evt.get("DESCRIPTION")
            evtPrimary = evt.get("PRIMARY")
            retVal = self.updateEventType(evtId, evtDesc, evtPrimary)
            if not retVal["OK"]:
                failed.append({evtId: {"Error": retVal["Message"], "EvtentType": evt}})

        successful = list({evt["EVTTYPEID"] for evt in eventtypes} - {list(i)[0] for i in failed})
        return S_OK({"Failed": failed, "Successful": successful})

    def getRunConfigurationsAndDataTakingCondition(self, runnumber):
        """For retrieving the run configuration name and version and the data
        taking condition.

        :param: int runnumber
        :return: S_OK()/S_ERROR ConfigName, ConfigVersion and DataTakingDescription
        """
        command = (
            "SELECT c.configname, c.configversion FROM jobs j, configurations c \
WHERE j.configurationid=c.configurationid AND j.production<0 AND j.runnumber=%d"
            % runnumber
        )

        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        if not retVal["Value"]:
            return S_ERROR("Run does not exists in the db")

        result = {"ConfigName": retVal["Value"][0][0], "ConfigVersion": retVal["Value"][0][1]}

        command = (
            "SELECT d.description FROM jobs j, productionscontainer prod, data_taking_conditions d \
WHERE j.production=prod.production AND j.production<0 AND prod.daqperiodid=d.daqperiodid AND j.runnumber=%d"
            % runnumber
        )

        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        if not retVal["Value"]:
            return S_ERROR("Data taking description does not exists")
        result["ConditionDescription"] = retVal["Value"][0][0]

        return S_OK(result)

    def deleteCertificationData(self):
        """It destroy the data used by the integration test."""
        return self.dbR_.executeStoredProcedure("BKUTILITIES.destroyDatasets", [], False)

    def updateProductionOutputfiles(self):
        """It is used to trigger an update of the productionoutputfiles table."""
        return self.dbR_.executeStoredProcedure("BKUTILITIES.updateProdOutputFiles", [], False)

    #############################################################################
    def getAvailableTagsFromSteps(self):
        """Availabe database tags.

        :returns: S_OK/S_ERROR a list of db tags
        """

        command = "SELECT DISTINCT DDDB,CONDDB,DQTAG FROM steps WHERE Usable='Yes'"
        retVal = self.dbR_.query(command)
        if not retVal["OK"]:
            return retVal

        records = []
        for record in retVal["Value"]:
            if record[0] is not None:
                records.append(["DDDB", record[0]])
            if record[1] is not None:
                records.append(["CONDDB", record[1]])
            if record[2] is not None:
                records.append(["DQTAG", record[2]])

        return S_OK({"ParameterNames": ["TagName", "TagValue"], "Records": records})

    #############################################################################
    def bulkgetIDsFromFilesTable(self, lfns):
        """This method used to retreive the JobId, FileId and FiletypeId for a
        given list of lfns.

        :param list lfns: list of lfns
        :returns: S_OK/S_ERROR {"FileId:1","JobId":22, "FileTypeId":3}
        """
        retVal = self.dbR_.executeStoredProcedure(
            packageName="BOOKKEEPINGORACLEDB.bulkgetIdsFromFiles", parameters=[], output=True, array=lfns
        )
        if not retVal["OK"]:
            return retVal

        fileParams = ["JobId", "FileId", "FileTypeId"]
        result = {}
        for record in retVal["Value"]:
            result[record[0]] = dict(zip(fileParams, record[1:]))

        failed = list(set(lfns) - set(result))
        return S_OK({"Successful": result, "Failed": failed})
