##
# File:    PdbxLoader.py
# Author:  J. Westbrook
# Date:    14-Mar-2018
# Version: 0.001
#
# Updates:
#     20-Mar-2018 jdw  adding prdcc within chemical component collection
#     21-Mar-2018 jdw  content filtering options added from documents
#     25-Mar-2018 jdw  add support for loading multiple collections for a content type.
#     26-Mar-2018 jdw  improve how successful loads are tracked accross collections -
#      9-Apr-2018 jdw  folding in improved schema path utility apis
#     19-Jun-2018 jdw  integrate with dynamic schema / status object must be unit cardinality.
#     22-Jun-2018 jdw  change collection attribute specification to dot notation
#     22-Jun-2018 jdw  separate cases where the loading success can be easily mapped to the source data object.
#     24-Jun-2018 jdw  rename and specialize function
#     14-Jul-2018 jdw  add methods to return data exchange status objects for load operations
#     25-Jul-2018 jdw  fixed bazaar bad function references blocking failure handling
#     25-Jul-2018 jdw  restore pruning operations
#     14-Aug-2018 jdw  primaryIndexD from self.__schU.getSchemaInfo(schemaName) updated to list and i
#                      in __createCollection(self, databaseName, collectionName, indexAttributeNames=None) make indexAttributeNames a list
#     10-Sep-2018 jdw  Adjust error handling and reporting across multiple collections
#     24-Oct-2018 jdw  update for new configuration organization
#     11-Nov-2018 jdw  add DrugBank and CCDC mapping path details.
#     21-Nov-2018 jdw  add addDocumentPrivateAttributes(dList, collectionName) to inject private document keys
#      3-Dec-2018 jdw  generalize the creation of collection indices.
#     16-Feb-2019 jdw  Add argument mergeContentTypes to load() method. Generalize the handling of path lists to
#                      support locator object lists.
#      6-Aug-2019 jdw  Add schema generation option and move dictionary API instantiation into load() method.
#     18-May-2020 jdw  Add brute force document purging for loadType=replace
#
##
"""
Worker methods for loading primary data content following mapping conventions in external schema definitions.

"""

__docformat__ = "restructuredtext en"
__author__ = "John Westbrook"
__email__ = "jwest@rcsb.rutgers.edu"
__license__ = "Apache 2.0"

# pylint: disable=too-many-lines

import logging
import operator
import os
import sys
import time

import bson

from jsonschema import Draft4Validator
from jsonschema import FormatChecker
from mmcif.api.DictMethodRunner import DictMethodRunner
from rcsb.db.define.DictionaryApiProviderWrapper import DictionaryApiProviderWrapper
from rcsb.db.helpers.DictMethodResourceProvider import DictMethodResourceProvider
from rcsb.db.mongo.Connection import Connection
from rcsb.db.mongo.MongoDbUtil import MongoDbUtil
from rcsb.db.processors.DataExchangeStatus import DataExchangeStatus
from rcsb.db.processors.DataTransformFactory import DataTransformFactory
from rcsb.db.processors.SchemaDefDataPrep import SchemaDefDataPrep
from rcsb.db.utils.RepositoryProvider import RepositoryProvider
from rcsb.db.utils.SchemaProvider import SchemaProvider
from rcsb.utils.multiproc.MultiProcUtil import MultiProcUtil

logger = logging.getLogger(__name__)


class PdbxLoaderWorker(object):
    def __init__(self, cfgOb, repositoryProviderObj, dictionaryMethodRunnerObj, resourceName):
        self.__verbose = False
        self.__cfgOb = cfgOb
        self.__rpP = repositoryProviderObj
        self.__dmh = dictionaryMethodRunnerObj
        self.__resourceName = resourceName

    def loadWorker(self, dataList, procName, optionsD, workingDir):
        """ Multi-proc worker method for MongoDb loading -

        successList, resultList, diagList=workerFunc(runList=nextList,procName, optionsD, workingDir)

        locatorObjList -> containerList -> docList  ->|LOAD|<-  .... return success locatorObjList

        """
        try:
            startTime = self.__begin(message=procName)
            # Recover common options
            styleType = optionsD["styleType"]
            filterType = optionsD["filterType"]
            readBackCheck = optionsD["readBackCheck"]
            logSize = "logSize" in optionsD and optionsD["logSize"]
            dataSelectors = optionsD["dataSelectors"]
            loadType = optionsD["loadType"]
            databaseName = optionsD["databaseName"]
            pruneDocumentSize = optionsD["pruneDocumentSize"]
            sd = optionsD["schemaDefAccess"]
            dtf = optionsD["dataTransformFactory"]
            collectionNameList = optionsD["collectionNameList"]
            useNameFlag = optionsD["useNameFlag"]
            sdp = SchemaDefDataPrep(schemaDefAccessObj=sd, dtObj=dtf, workPath=workingDir, verbose=self.__verbose)
            # -------------------------------------------
            # -- Create map of  cIdD{ container identifier} =  locatorObj
            #
            collectionName = None
            cIdD = {}
            cNameL = []
            containerList = []
            for locatorObj in dataList:
                # JDW
                cL = self.__rpP.getContainerList([locatorObj])
                if cL:
                    cNameL.append(cL[0].getName().upper().strip())
                    cId = cL[0].getName() if useNameFlag else cL[0].getProp("uid")
                    cIdD[cId] = locatorObj
                    containerList.extend(cL)
            # -- Apply methods to each container -
            for container in containerList:
                if self.__dmh:
                    self.__dmh.apply(container)
                else:
                    logger.debug("%s No dynamic method handler for ", procName)
            # -----
            if loadType != "full":
                for collectionName in collectionNameList:
                    ok = self.__purgeDocuments(databaseName, collectionName, cNameL)
                    logger.debug("%s %s - loadType %r cNameL %r (%r)", databaseName, collectionName, loadType, cNameL, ok)
            # --
            failContainerIdS = set()
            rejectContainerIdS = set()
            # -----
            for collectionName in collectionNameList:
                ok = True
                # ---------------- - ---------------- - ---------------- - ---------------- - ---------------- -
                docIdL = sd.getDocumentKeyAttributeNames(collectionName)
                replaceIdL = sd.getDocumentReplaceAttributeNames(collectionName)
                #
                tableIdExcludeList = sd.getCollectionExcluded(collectionName)
                tableIdIncludeList = sd.getCollectionSelected(collectionName)
                sliceFilter = sd.getCollectionSliceFilter(collectionName)
                sdp.setSchemaIdExcludeList(tableIdExcludeList)
                sdp.setSchemaIdIncludeList(tableIdIncludeList)
                #
                logger.debug("%s databaseName %s collectionName %s slice filter %s", procName, databaseName, collectionName, sliceFilter)
                logger.debug("%s databaseName %s include list %r", procName, databaseName, tableIdIncludeList)
                logger.debug("%s databaseName %s exclude list %r", procName, databaseName, tableIdExcludeList)
                #
                dList, containerIdList, rejectIdList = sdp.processDocuments(
                    containerList,
                    styleType=styleType,
                    filterType=filterType,
                    dataSelectors=dataSelectors,
                    sliceFilter=sliceFilter,
                    useNameFlag=useNameFlag,
                    collectionName=collectionName,
                )
                #
                # ------
                # Collect the container identifiers for the rejected containers (paths for logging only)
                # Note that rejections are NOT treated as failures!
                #
                rejectPathList = []
                for cId in rejectIdList:
                    rejectContainerIdS.add(cId)
                    locObj = cIdD[cId]
                    rejectPathList.extend(self.__rpP.getLocatorPaths([locObj], locatorIndex=0))
                rejectPathList = list(set(rejectPathList))
                #
                if logSize:
                    self.__logDocumentSize(procName, dList, docIdL)

                dList = sdp.addDocumentPrivateAttributes(dList, collectionName)
                dList = sdp.addDocumentSubCategoryAggregates(dList, collectionName)
                #
                # --- And after adjustments create index
                #     to map dList -> containerNamList  using dList(uniqId) -> containterName
                #
                indexDoc = {}
                try:
                    for dD, cId in zip(dList, containerIdList):
                        dIdTup = self.__getKeyValues(dD, docIdL)
                        indexDoc[dIdTup] = cId
                except Exception as e:
                    logger.exception("Failing cN %r  dD %r with %s", cId, dD, str(e))

                #
                if dList:
                    ok, _, failDocIdS = self.__loadDocuments(
                        databaseName, collectionName, dList, docIdL, replaceIdL=replaceIdL, loadType=loadType, readBackCheck=readBackCheck, pruneDocumentSize=pruneDocumentSize
                    )
                # ------
                # Collect the container identifiers for the successful loads (paths for logging only)
                #
                failPathList = []
                for dId in failDocIdS:
                    cId = indexDoc[dId]
                    failContainerIdS.add(cId)
                    locObj = cIdD[cId]
                    failPathList.extend(self.__rpP.getLocatorPaths([locObj], locatorIndex=0))
                failPathList = list(set(failPathList))
                #
                if failPathList:
                    logger.debug("%s %s/%s worker load failures %r", procName, databaseName, collectionName, [os.path.basename(pth) for pth in failPathList if pth is not None])
                if rejectPathList:
                    logger.debug("%s %s/%s worker load rejected %r", procName, databaseName, collectionName, [os.path.basename(pth) for pth in rejectPathList])
            #
            # -------------------------
            #  failContainerIdS = set()
            #  rejectContainerIdS = set()
            #
            #  cIdD[cId] = locatorObj
            # ----
            retList = [locatorObj for cId, locatorObj in cIdD.items() if cId not in failContainerIdS]
            logger.debug("%s %s load worker returns  successes %d rejects %d failures %d", procName, databaseName, len(retList), len(rejectContainerIdS), len(failContainerIdS))

            ok = len(failContainerIdS) == 0
            self.__end(startTime, procName + " with status " + str(ok))
            return retList, [], []

        except Exception as e:
            # logger.error("Failing for dataList %r" % dataList)
            logger.exception("Failing with %s", str(e))

        return [], [], []

    # -------------- -------------- -------------- -------------- -------------- -------------- --------------
    #                                        ---  Supporting code follows ---
    #
    def __logDocumentSize(self, procName, dList, docIdL):
        maxDocumentMegaBytes = -1
        thresholdMB = 15.8
        # thresholdMB = 5.0
        for tD in dList:
            cN = self.__getKeyValues(tD, docIdL)
            # documentMegaBytes = float(sys.getsizeof(pickle.dumps(tD, protocol=0))) / 1000000.0
            documentMegaBytes = float(sys.getsizeof(bson.BSON.encode(tD))) / 1000000.0
            logger.debug("%s Document %s %.4f MB", procName, cN, documentMegaBytes)
            maxDocumentMegaBytes = max(maxDocumentMegaBytes, documentMegaBytes)
            if documentMegaBytes > thresholdMB:
                logger.info("Large document %r  %.4f MB", cN, documentMegaBytes)
                for ky in tD:
                    try:
                        sMB = float(sys.getsizeof(bson.BSON.encode({"t": tD[ky]}))) / 1000000.0
                    except Exception:
                        sMB = -1
                    logger.info("Sub-document length %s sizeMB %.4f  %8d", ky, sMB, len(tD[ky]))
                #
        logger.info("%s maximum document size loaded %.4f MB", procName, maxDocumentMegaBytes)
        return True

    def __purgeDocuments(self, databaseName, collectionName, cardinalIdL):
        """Purge documents from collection within database with cardinal identifiers in cardinalIdL.

        """
        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                for cardId in cardinalIdL:
                    # selectD = {"rcsb_id": "/%s/" % cardId} # this filter did not work
                    selectD = {"rcsb_id": {"$regex": "^%s" % cardId.upper(), "$options": "i"}}  # case-insensitive
                    dCount = mg.delete(databaseName, collectionName, selectD)
                    logger.debug("Remove %d objects in database %s collection %s selection %r", dCount, databaseName, collectionName, selectD)
            return True
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    #

    def __writePathList(self, filePath, pathList):
        try:
            with open(filePath, "w") as ofh:
                for pth in pathList:
                    ofh.write("%s\n" % pth)
            return True
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __begin(self, message=""):
        startTime = time.time()
        ts = time.strftime("%Y %m %d %H:%M:%S", time.localtime())
        logger.debug("Starting %s at %s", message, ts)
        return startTime

    def __end(self, startTime, message=""):
        endTime = time.time()
        ts = time.strftime("%Y %m %d %H:%M:%S", time.localtime())
        delta = endTime - startTime
        logger.debug("Completed %s at %s (%.4f seconds)", message, ts, delta)

    def __createCollection(self, databaseName, collectionName, indexDL=None, bsonSchema=None):
        """Create database and collection and optionally a set of indices -
        """
        try:
            logger.debug("Create database %s collection %s", databaseName, collectionName)
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                ok1 = mg.createCollection(databaseName, collectionName, bsonSchema=bsonSchema)
                ok2 = mg.databaseExists(databaseName)
                ok3 = mg.collectionExists(databaseName, collectionName)
                okI = True
                if indexDL:
                    for indexD in indexDL:
                        okI = mg.createIndex(databaseName, collectionName, indexD["ATTRIBUTE_NAMES"], indexName=indexD["INDEX_NAME"], indexType="DESCENDING", uniqueFlag=False)

            return ok1 and ok2 and ok3 and okI
            #
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __removeCollection(self, databaseName, collectionName):
        """Drop collection within database

        """
        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                #
                logger.debug("Remove collection database %s collection %s", databaseName, collectionName)
                logger.debug("Starting databases = %r", mg.getDatabaseNames())
                logger.debug("Starting collections = %r", mg.getCollectionNames(databaseName))
                ok = mg.dropCollection(databaseName, collectionName)
                logger.debug("Databases = %r", mg.getDatabaseNames())
                logger.debug("Post drop collections = %r", mg.getCollectionNames(databaseName))
                ok = mg.collectionExists(databaseName, collectionName)
                logger.debug("Post drop collections = %r", mg.getCollectionNames(databaseName))
            return ok
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __pruneBySize(self, dList, limitMB=15.9):
        """ For the input list of objects (dictionaries).objects
            Return a pruned list satisfying the input total object size limit -

        """
        oL = []
        try:
            for dD in dList:
                sD = {}
                sumMB = 0.0
                for ky in dD:
                    dMB = 0.0
                    try:
                        dMB = float(sys.getsizeof(bson.BSON.encode({ky: dD[ky]}))) / 1000000.0
                    except Exception as e:
                        logger.error("ky %r d[ky] %r with %s", ky, dD[ky], str(e))

                    sumMB += dMB
                    sD[ky] = dMB
                if sumMB < limitMB:
                    oL.append(dD)
                    continue
                #
                sortedSd = sorted(sD.items(), key=operator.itemgetter(1))
                prunedSum = 0.0
                for ky, sMB in sortedSd:
                    prunedSum += sMB
                    if prunedSum > limitMB:
                        dD.pop(ky, None)
                        logger.debug("Pruning ky %s size(MB) %.2f", ky, sMB)
                oL.append(dD)
        except Exception as e:
            logger.exception("Failing with %s", str(e))
            #
        logger.debug("Pruning returns document list length %d", len(dList))
        return oL

    def __loadDocuments(self, databaseName, collectionName, dList, docIdL, replaceIdL=None, loadType="full", readBackCheck=False, pruneDocumentSize=None):
        #
        # Load database/collection with input document list -
        #
        rIdL = []

        logger.debug("databaseName %s collectionName %s docIdL %r", databaseName, collectionName, docIdL)
        inputDocIdS = {self.__getKeyValues(dD, docIdL) for dD in dList}
        failDocIdS = set()
        successDocIdS = set()

        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                #
                if loadType == "replace" and replaceIdL:
                    deleteTupL = mg.deleteList(databaseName, collectionName, dList, replaceIdL)
                    logger.debug("Deleted document status %r", deleteTupL)
                if pruneDocumentSize:
                    dList = self.__pruneBySize(dList, limitMB=pruneDocumentSize)
                #
                rIdL.extend(mg.insertList(databaseName, collectionName, dList, keyNames=docIdL, salvage=True))
                # ---
                #  If there is a failure then determine the specific successes and failures -
                #
                successDocIdS = inputDocIdS
                if len(rIdL) != len(dList):
                    sIdS = set()
                    try:
                        for rId in rIdL:
                            rObj = mg.fetchOne(databaseName, collectionName, "_id", rId)
                            dIdTup = self.__getKeyValues(rObj, docIdL)
                            sIdS.add(dIdTup)
                    except Exception as e:
                        logger.exception("Failing with %s", str(e))
                    successDocIdS = sIdS
                # enumerate the failures
                failDocIdS = inputDocIdS - successDocIdS
                #
                if readBackCheck:
                    # build an index of the input document list
                    indD = {}
                    try:
                        for ii, dD in enumerate(dList):
                            dIdTup = self.__getKeyValues(dD, docIdL)
                            indD[dIdTup] = ii
                    except Exception as e:
                        logger.exception("Failing ii %d d %r with %s", ii, dD, str(e))
                    #
                    # Note that objects in dList are mutated by the insert operation with the additional key '_id',
                    # hence, it is possible to compare the fetched object with the input object.
                    #
                    rbStatus = True
                    for ii, rId in enumerate(rIdL):
                        rObj = mg.fetchOne(databaseName, collectionName, "_id", rId)
                        dIdTup = self.__getKeyValues(rObj, docIdL)
                        jj = indD[dIdTup]
                        if rObj != dList[jj]:
                            rbStatus = False
                            break
                #
                if readBackCheck and not rbStatus:
                    return False, successDocIdS, failDocIdS
                #
            return len(rIdL) == len(dList), successDocIdS, failDocIdS
        except Exception as e:
            logger.exception("Failing with %s", str(e))

        return False, [], inputDocIdS

    def __getKeyValues(self, dct, keyNames):
        """Return the tuple of values of corresponding to the input dictionary key names expressed in dot notation.

        Args:
            dct (dict): source dictionary object (nested)
            keyNames (list): list of dictionary keys in dot notatoin

        Returns:
            tuple: tuple of values corresponding to the input key names

        """
        rL = []
        try:
            for keyName in keyNames:
                rL.append(self.__getKeyValue(dct, keyName))
        except Exception as e:
            logger.exception("Failing for key names %r with %s", keyNames, str(e))

        return tuple(rL)

    def __getKeyValue(self, dct, keyName):
        """  Return the value of the corresponding key expressed in dot notation in the input dictionary object (nested).
        """
        try:
            kys = keyName.split(".")
            for key in kys:
                try:
                    dct = dct[key]
                except Exception:
                    logger.warning("Missing document key %r in %r", key, list(dct.keys()))
                    return None
            return dct
        except Exception as e:
            logger.error("Failing for key %r with %s", keyName, str(e))

        return None


class PdbxLoader(object):
    def __init__(
        self,
        cfgOb,
        cachePath=None,
        resourceName="MONGO_DB",
        numProc=4,
        chunkSize=15,
        fileLimit=None,
        verbose=False,
        readBackCheck=False,
        maxStepLength=2000,
        useSchemaCache=True,
        rebuildSchemaFlag=False,
    ):
        """  Worker methods for loading primary data content following mapping conventions in external schema definitions.

        Args:
            cfgOb (object): ConfigInfo() instance
            cachePath (str, optional): path to cache directories
            resourceName (str, optional): server resource name
            numProc (int, optional): number of processes to launch (MultiProcUtil)
            chunkSize (int, optional): partition list of load paths into chunks of this size
            fileLimit (int, optional): maximum number of files to process (no limit = None)
            verbose (bool, optional): Description
            readBackCheck (bool, optional): read back and check each loaded object
            maxStepLength (int, optional): limit multiprocessing steps

        """
        self.__verbose = verbose
        #
        # Limit the load length of each file type for testing  -  Set to None to remove -
        self.__fileLimit = fileLimit
        self.__maxStepLength = maxStepLength
        #
        # Controls for multiprocessing execution -
        self.__numProc = max(numProc, 1)
        self.__chunkSize = max(chunkSize, 1)
        #
        self.__cfgOb = cfgOb
        self.__resourceName = resourceName
        #
        self.__readBackCheck = readBackCheck
        self.__cachePath = cachePath
        self.__useSchemaCache = useSchemaCache
        self.__rebuildSchemaFlag = rebuildSchemaFlag
        self.__mpFormat = "[%(levelname)s] %(asctime)s %(processName)s-%(module)s.%(funcName)s: %(message)s"
        #
        self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=self.__useSchemaCache, rebuildFlag=self.__rebuildSchemaFlag)
        self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__cachePath)
        #
        self.__statusList = []
        #
        self.__sectionName = "site_info_configuration"
        self.__dmh = None
        #

    def load(
        self,
        databaseName,
        collectionLoadList=None,
        loadType="full",
        inputPathList=None,
        styleType="rowwise_by_name",
        dataSelectors=None,
        failedFilePath=None,
        saveInputFileListPath=None,
        pruneDocumentSize=None,
        logSize=False,
        validationLevel="min",
        mergeContentTypes=None,
        useNameFlag=True,
        updateSchemaOnReplace=True,
        validateFailures=True,
        reloadPartial=True,
    ):
        """Driver method for loading PDBx/mmCIF content into the Mongo document store.

        Args:
            databaseName (str): A content datbase schema (e.g. 'bird','bird_family','bird_chem_comp', chem_comp', 'pdbx', 'pdbx_core')
            collectionLoadList (list, optional): list of collection names in this schema to load (default is load all collections)
            loadType (str, optional): mode of loading 'full' (bulk delete then bulk insert) or 'replace'
            inputPathList (list, optional): Data file path list (if not provided the full repository will be scanned)
            styleType (str, optional): one of 'rowwise_by_name', 'columnwise_by_name', 'rowwise_no_name', 'rowwise_by_name_with_cardinality'
            dataSelectors (list, optional): selector names defined for this schema (e.g. PUBLIC_RELEASE)
            failedFilePath (str, optional): Path to hold file paths for load failures
            saveInputFileListPath (list, optional): List of files
            pruneDocumentSize (bool, optional): iteratively remove large elements from a collection to satisfy size limits
            logSize (bool, optional): Compute and log bson serialized object size
            validationLevel (str, optional): Completeness of json/bson metadata schema bound to each collection (e.g. 'min', 'full' or None)
            useNameFlag (bool, optional): Use container name as unique identifier otherwise use UID property.
            updateSchemaOnReplace (bool, optional): Update validation schema for loadType == 'replace'
            validateFailures (bool, optional): output validation report on load failures
            reloadPartial (bool, optional): on load failures attempt reload of partial objects.
        Returns:
            bool: True on success or False otherwise

        """
        try:
            #
            self.__statusList = []
            desp = DataExchangeStatus()
            statusStartTimestamp = desp.setStartTime()
            #
            logger.info("Beginning load operation (%r) for database %s", loadType, databaseName)
            startTime = self.__begin(message="loading operation")
            #
            modulePathMap = self.__cfgOb.get("DICT_METHOD_HELPER_MODULE_PATH_MAP", sectionName=self.__sectionName)
            dP = DictionaryApiProviderWrapper(self.__cfgOb, self.__cachePath, useCache=True)
            dictApi = dP.getApiByName(databaseName)
            # ---
            dmrP = DictMethodResourceProvider(self.__cfgOb, cachePath=self.__cachePath)
            # Cache dependencies in serial mode.
            ok = dmrP.cacheResources(useCache=True)
            if not ok:
                logger.error("Checking cached resource dependencies failed - %s load (%r) aborted", databaseName, loadType)
                return ok
            # ---
            self.__dmh = DictMethodRunner(dictApi, modulePathMap=modulePathMap, resourceProvider=dmrP)
            locatorObjList = self.__rpP.getLocatorObjList(contentType=databaseName, inputPathList=inputPathList, mergeContentTypes=mergeContentTypes)
            logger.info("Loading database %s (%r) with path length %d", databaseName, loadType, len(locatorObjList))
            #
            if saveInputFileListPath:
                self.__writePathList(saveInputFileListPath, self.__rpP.getLocatorPaths(locatorObjList))
                logger.info("Saving %d paths in %s", len(locatorObjList), saveInputFileListPath)
            #
            filterType = "drop-empty-attributes|drop-empty-tables|skip-max-width|assign-dates|convert-iterables|normalize-enums|translateXMLCharRefs"
            if styleType in ["columnwise_by_name", "rowwise_no_name"]:
                filterType = "drop-empty-tables|skip-max-width|assign-dates|convert-iterables|normalize-enums|translateXMLCharRefs"
            #
            optD = {}
            optD["databaseName"] = databaseName
            optD["styleType"] = styleType
            optD["filterType"] = filterType
            optD["readBackCheck"] = self.__readBackCheck
            optD["dataSelectors"] = dataSelectors
            optD["loadType"] = loadType
            optD["logSize"] = logSize
            optD["pruneDocumentSize"] = pruneDocumentSize
            optD["useNameFlag"] = useNameFlag
            optD["validationLevel"] = validationLevel
            optD["validateFailures"] = validateFailures
            optD["reloadPartial"] = reloadPartial
            # ---------------- - ---------------- - ---------------- - ---------------- - ---------------- -
            #

            numProc = self.__numProc
            chunkSize = self.__chunkSize if locatorObjList and self.__chunkSize < len(locatorObjList) else 0
            #
            sd, _, fullCollectionNameList, docIndexD = self.__schP.getSchemaInfo(databaseName, dataTyping="ANY")

            collectionNameList = collectionLoadList if collectionLoadList else fullCollectionNameList

            for collectionName in collectionNameList:
                if loadType == "full":
                    self.__removeCollection(databaseName, collectionName)
                    indexDL = docIndexD[collectionName] if collectionName in docIndexD else []
                    bsonSchema = None
                    if validationLevel and validationLevel in ["min", "full"]:
                        bsonSchema = self.__schP.getJsonSchema(databaseName, collectionName, encodingType="BSON", level=validationLevel)
                    ok = self.__createCollection(databaseName, collectionName, indexDL=indexDL, bsonSchema=bsonSchema)
                    logger.debug("Collection create return status %r", ok)
                elif loadType == "replace" and updateSchemaOnReplace:
                    bsonSchema = None
                    if validationLevel and validationLevel in ["min", "full"]:
                        bsonSchema = self.__schP.getJsonSchema(databaseName, collectionName, encodingType="BSON", level=validationLevel)
                    if bsonSchema:
                        ok = self.__updateCollectionSchema(databaseName, collectionName, bsonSchema=bsonSchema)
                        if not ok:
                            logger.info("Schema update failing for %s (%s)", databaseName, collectionName)
            #
            dtf = DataTransformFactory(schemaDefAccessObj=sd, filterType=filterType)
            optD["schemaDefAccess"] = sd
            optD["dataTransformFactory"] = dtf
            optD["collectionNameList"] = collectionNameList

            # ---------------- - ---------------- - ---------------- - ---------------- - ---------------- -
            numPaths = len(locatorObjList)
            logger.debug("Processing %d total paths", numPaths)
            numProc = min(numProc, numPaths)
            maxStepLength = self.__maxStepLength
            if numPaths > maxStepLength:
                numLists = int(numPaths / maxStepLength)
                # JDW always fill numProc
                numLists = max(numLists, numProc)
                subLists = [locatorObjList[i::numLists] for i in range(numLists)]
            else:
                subLists = [locatorObjList]
            #
            if subLists:
                logger.info("Starting load of %s (%r) using numProc %d outer subtask count %d subtask length %d", databaseName, loadType, numProc, len(subLists), len(subLists[0]))
            else:
                logger.error("Path partitioning fails for %s (%r) using numProc %d", databaseName, loadType, numProc)
            #
            failList = []
            for ii, subList in enumerate(subLists):
                logger.info("Running outer subtask %d of %d length %d", ii + 1, len(subLists), len(subList))
                #
                # pdbxLoaderWorker = PdbxLoaderWorker(self.__cfgOb, self.__rpP, self.__dmh, self.__resourceName)
                mpu = MultiProcUtil(verbose=True)
                mpu.setWorkingDir(self.__cachePath)
                mpu.setOptions(optionsD=optD)
                mpu.set(workerObj=self, workerMethod="loadWorker")
                ok, failListT, _, _ = mpu.runMulti(dataList=subList, numProc=numProc, numResults=1, chunkSize=chunkSize)
                logger.info("Completed outer subtask %d of %d length %d with failure count %d status %r", ii + 1, len(subLists), len(subList), len(failListT), ok)
                failList.extend(failListT)
            failList = list(set(failList))
            logger.debug("Failing path list %r", failList)
            #
            failedPathList = self.__rpP.getLocatorPaths(failList, locatorIndex=0)
            if failedFilePath and failedPathList:
                wOk = self.__writePathList(failedFilePath, failedPathList)
                logger.info("Writing failure path %s length %d status %r", failedFilePath, len(failList), wOk)
            #
            ok = len(failList) == 0
            self.__end(startTime, "Loading operation completed with status " + str(ok))
            #
            # Create the status objects for the current operations
            # ----
            sFlag = "Y" if ok else "N"
            for collectionName in collectionNameList:
                desp.setStartTime(tS=statusStartTimestamp)
                desp.setObject(databaseName, collectionName)
                desp.setStatus(updateId=None, successFlag=sFlag)
                desp.setEndTime()
                self.__statusList.append(desp.getStatus())
            #
            if ok:
                logger.info("Completed loading %s with status %r loaded %d paths", databaseName, ok, numPaths)
            else:
                logger.info(
                    "Completed loading %s with status %r failure count %d of %d paths: %r", databaseName, ok, len(failList), numPaths, [os.path.basename(pth) for pth in failedPathList],
                )
            #
            return ok
        except Exception as e:
            logger.exception("Failing with %s", str(e))

        return False

    def getLoadStatus(self):
        return self.__statusList

    def loadWorker(self, dataList, procName, optionsD, workingDir):
        """ Multi-proc worker method for MongoDb loading -

        successList, resultList, diagList=workerFunc(runList=nextList,procName, optionsD, workingDir)

        locatorObjList -> containerList -> docList  ->|LOAD|<-  .... return success locatorObjList

        """
        try:
            startTime = self.__begin(message=procName)
            # Recover common options
            styleType = optionsD["styleType"]
            filterType = optionsD["filterType"]
            readBackCheck = optionsD["readBackCheck"]
            logSize = "logSize" in optionsD and optionsD["logSize"]
            dataSelectors = optionsD["dataSelectors"]
            loadType = optionsD["loadType"]
            databaseName = optionsD["databaseName"]
            pruneDocumentSize = optionsD["pruneDocumentSize"]
            sd = optionsD["schemaDefAccess"]
            dtf = optionsD["dataTransformFactory"]
            collectionNameList = optionsD["collectionNameList"]
            useNameFlag = optionsD["useNameFlag"]
            validationLevel = optionsD["validationLevel"]
            validateFailures = optionsD["validateFailures"]
            reloadPartial = optionsD["reloadPartial"]
            #
            sdp = SchemaDefDataPrep(schemaDefAccessObj=sd, dtObj=dtf, workPath=workingDir, verbose=self.__verbose)
            # -------------------------------------------
            # -- Create map of  cIdD{ container identifier} =  locatorObj
            #
            collectionName = None
            cIdD = {}
            cNameL = []
            containerList = []
            for locatorObj in dataList:
                # JDW
                cL = self.__rpP.getContainerList([locatorObj])
                if cL:
                    cNameL.append(cL[0].getName().upper().strip())
                    cId = cL[0].getName() if useNameFlag else cL[0].getProp("uid")
                    cIdD[cId] = locatorObj
                    containerList.extend(cL)
            # -- Apply methods to each container -
            for container in containerList:
                if self.__dmh:
                    self.__dmh.apply(container)
                else:
                    logger.debug("%s No dynamic method handler for ", procName)
            # -----
            if loadType != "full":
                for collectionName in collectionNameList:
                    logger.debug("Purging objects from %s for %d containers", collectionName, len(cNameL))
                    ok = self.__purgeDocuments(databaseName, collectionName, cNameL)
                    logger.debug("%s %s - loadType %r cNameL %r (%r)", databaseName, collectionName, loadType, cNameL, ok)
                    # --
            # -----
            failContainerIdS = set()
            rejectContainerIdS = set()
            cardinalIdFailS = set()
            # -----
            for collectionName in collectionNameList:
                ok = True
                # ---------------- - ---------------- - ---------------- - ---------------- - ---------------- -
                docIdL = sd.getDocumentKeyAttributeNames(collectionName)
                replaceIdL = sd.getDocumentReplaceAttributeNames(collectionName)
                #
                tableIdExcludeList = sd.getCollectionExcluded(collectionName)
                tableIdIncludeList = sd.getCollectionSelected(collectionName)
                sliceFilter = sd.getCollectionSliceFilter(collectionName)
                sdp.setSchemaIdExcludeList(tableIdExcludeList)
                sdp.setSchemaIdIncludeList(tableIdIncludeList)
                #
                logger.debug("%s databaseName %s collectionName %s slice filter %s", procName, databaseName, collectionName, sliceFilter)
                logger.debug("%s databaseName %s include list %r", procName, databaseName, tableIdIncludeList)
                logger.debug("%s databaseName %s exclude list %r", procName, databaseName, tableIdExcludeList)
                #
                dList, containerIdList, rejectIdList = sdp.processDocuments(
                    containerList,
                    styleType=styleType,
                    filterType=filterType,
                    dataSelectors=dataSelectors,
                    sliceFilter=sliceFilter,
                    useNameFlag=useNameFlag,
                    collectionName=collectionName,
                )
                #
                # -- JDWJDW
                # logger.info("loadType %r collectionName %r replaceIdL %r idList %r", loadType, collectionName, replaceIdL, containerIdList)
                # --
                # ------
                # Collect the container identifiers for the rejected containers (paths for logging only)
                # Note that rejections are NOT treated as failures!
                #
                rejectPathList = []
                for cId in rejectIdList:
                    rejectContainerIdS.add(cId)
                    locObj = cIdD[cId]
                    rejectPathList.extend(self.__rpP.getLocatorPaths([locObj], locatorIndex=0))
                rejectPathList = list(set(rejectPathList))
                #
                if logSize:
                    self.__logDocumentSize(procName, dList, docIdL)

                dList = sdp.addDocumentPrivateAttributes(dList, collectionName)
                dList = sdp.addDocumentSubCategoryAggregates(dList, collectionName)
                #
                # --- And after adjustments create index
                #     to map dList -> containerNamList  using dList(uniqId) -> containterName
                #
                indexDoc = {}
                try:
                    for dD, cId in zip(dList, containerIdList):
                        dIdTup = self.__getKeyValues(dD, docIdL)
                        indexDoc[dIdTup] = cId
                except Exception as e:
                    logger.exception("Failing cN %r  dD %r with %s", cId, dD, str(e))

                #
                if dList:
                    ok, _, failDocIdS = self.__loadDocuments(
                        databaseName, collectionName, dList, docIdL, replaceIdL=replaceIdL, loadType=loadType, readBackCheck=readBackCheck, pruneDocumentSize=pruneDocumentSize
                    )
                #
                if failDocIdS:

                    logger.info("Initial load failures: %r", failDocIdS)
                    fList = []
                    for dD in dList:
                        tId = self.__getKeyValues(dD, docIdL)
                        if tId in failDocIdS:
                            fList.append(dD)
                            if validateFailures:
                                logger.info("Validating document %r", tId)
                                self.__validateDocuments(databaseName, collectionName, [dD], docIdL, schemaLevel=validationLevel)
                    #
                    #  -- Try and repair failDocIdS --
                    #
                    if reloadPartial:
                        logger.debug("Attempting corrections on documents %r", failDocIdS)
                        fList = self.__validateAndFix(databaseName, collectionName, fList, docIdL, schemaLevel=validationLevel)

                        fOk, _, failDocIdS = self.__loadDocuments(
                            databaseName, collectionName, fList, docIdL, replaceIdL=replaceIdL, loadType=loadType, readBackCheck=readBackCheck, pruneDocumentSize=pruneDocumentSize
                        )
                        logger.info("Final load (%r) failures: %r", fOk, failDocIdS)

                # ------
                # Collect the container identifiers for the successful loads (paths for logging only)
                #
                failPathList = []
                for dId in failDocIdS:
                    cId = indexDoc[dId]
                    cardinalIdFailS.add(dId[0])
                    failContainerIdS.add(cId)
                    locObj = cIdD[cId]
                    failPathList.extend(self.__rpP.getLocatorPaths([locObj], locatorIndex=0))
                failPathList = list(set(failPathList))
                #
                if failPathList:
                    logger.debug("%s %s/%s worker load failures %r", procName, databaseName, collectionName, [os.path.basename(pth) for pth in failPathList if pth is not None])
                if rejectPathList:
                    logger.debug("%s %s/%s worker load rejected %r", procName, databaseName, collectionName, [os.path.basename(pth) for pth in rejectPathList])
            #
            # -------------------------
            #  failContainerIdS = set()
            #  rejectContainerIdS = set()
            #
            #  cIdD[cId] = locatorObj
            # ----
            retList = [locatorObj for cId, locatorObj in cIdD.items() if cId not in failContainerIdS]
            logger.debug("%s %s load worker returns  successes %d rejects %d failures %d", procName, databaseName, len(retList), len(rejectContainerIdS), len(failContainerIdS))
            #
            if cardinalIdFailS:
                # remove all collection objects related to a load failure
                for collectionName in collectionNameList:
                    logger.info("Purging all objects from %s for failed ids: %r", collectionName, cardinalIdFailS)
                    ok = self.__purgeDocuments(databaseName, collectionName, list(cardinalIdFailS))
            #
            ok = len(failContainerIdS) == 0
            self.__end(startTime, procName + " with status " + str(ok))
            return retList, [], []

        except Exception as e:
            # logger.error("Failing for dataList %r" % dataList)
            logger.exception("Failing with %s", str(e))

        return [], [], []

    # -------------- -------------- -------------- -------------- -------------- -------------- --------------
    #                                        ---  Supporting code follows ---
    #
    def __validateAndFix(self, databaseName, collectionName, dList, docIdL, schemaLevel="full"):
        """[summary]

        Args:
            databaseName ([type]): [description]
            collectionName ([type]): [description]
            dList ([type]): [description]
            docIdL ([type]): [description]
            schemaLevel (str, optional): [description]. Defaults to "full".

        Returns:
            (list):  updated document list (remediated for validation issues)
        """
        #
        rList = []
        logger.info("Validating and fixing objects in databaseName %s collectionName %s numObject %d docIdL %r", databaseName, collectionName, len(dList), docIdL)
        cD = self.__schP.getJsonSchema(databaseName, collectionName, encodingType="JSON", level=schemaLevel)
        # --
        try:
            Draft4Validator.check_schema(cD)
        except Exception as e:
            logger.error("%s %s schema validation fails with %s", databaseName, collectionName, str(e))
        # --
        filterArtifactErrors = True
        valInfo = Draft4Validator(cD, format_checker=FormatChecker())
        for ii, dD in enumerate(dList):
            cN = self.__getKeyValues(dD, docIdL)
            logger.info("Checking %r with schema %s collection %s document (%d)", cN, databaseName, collectionName, ii + 1)
            updL = []
            try:
                for error in sorted(valInfo.iter_errors(dD), key=str):
                    #
                    # Filter and cleanup artifacts -
                    #
                    if filterArtifactErrors and "properties are not allowed ('_id' was unexpected)" in error.message:
                        dD.pop("_id")
                        continue
                    if filterArtifactErrors and "datetime.datetime" in error.message and "is not of type 'string'" in error.message:
                        continue
                    #
                    logger.info("Document issues with schema %s collection %s (%s) path %s error: %s", databaseName, collectionName, cN, error.path, error.message)
                    logger.debug("Failing document %d : %r", ii + 1, list(dD.items()))
                    #
                    pLen = len(error.path)
                    if pLen >= 1:
                        subObjName = str(error.path[0])
                        if subObjName in dD:
                            logger.info("Found subobject %r", subObjName)
                            logger.info("subObject %r", dD[subObjName])
                            dD.pop(subObjName)
                            updL.append(cN)
                    #
            except Exception as e:
                logger.exception("Validation updating processing error %s", str(e))
            #
            if updL:
                rList.append(dD)
        #
        logger.info("Corrected document count (%d) %r", len(updL), updL)
        return rList

    #
    def __validateDocuments(self, databaseName, collectionName, dList, docIdL, schemaLevel="full"):
        #
        logger.info("Validating databaseName %s collectionName %s numObject %d docIdL %r", databaseName, collectionName, len(dList), docIdL)
        eCount = 0
        cD = self.__schP.getJsonSchema(databaseName, collectionName, encodingType="JSON", level=schemaLevel)
        # cD = self.__schP.makeSchema(databaseName, collectionName, encodingType="JSON", level=schemaLevel, saveSchema=True, extraOpts=self.__extraOpts)
        # Raises exceptions for schema compliance.
        try:
            Draft4Validator.check_schema(cD)
        except Exception as e:
            logger.error("%s %s schema validation fails with %s", databaseName, collectionName, str(e))
        #
        filterErrors = True
        valInfo = Draft4Validator(cD, format_checker=FormatChecker())
        logger.info("Validating %d documents from %s %s", len(dList), databaseName, collectionName)
        for ii, dD in enumerate(dList):
            cN = self.__getKeyValues(dD, docIdL)
            logger.info("Checking with schema %s collection %s document (%d) %r", databaseName, collectionName, ii + 1, cN)
            try:
                cCount = 0
                for error in sorted(valInfo.iter_errors(dD), key=str):
                    # filter artifacts -
                    #
                    if filterErrors and "properties are not allowed ('_id' was unexpected)" in error.message:
                        continue
                    if filterErrors and "datetime.datetime" in error.message and "is not of type 'string'" in error.message:
                        continue
                    logger.info("Document issues with schema %s collection %s (%s) path %s error: %s", databaseName, collectionName, cN, error.path, error.message)
                    logger.debug("Failing document %d : %r", ii + 1, list(dD.items()))
                    eCount += 1
                    cCount += 1
                if cCount > 0:
                    logger.info("For schema %s collection %s container %s error count %d", databaseName, collectionName, cN, cCount)
            except Exception as e:
                logger.exception("Validation processing error %s", str(e))
        return eCount

    def __logDocumentSize(self, procName, dList, docIdL):
        maxDocumentMegaBytes = -1
        thresholdMB = 15.8
        # thresholdMB = 5.0
        for tD in dList:
            cN = self.__getKeyValues(tD, docIdL)
            # documentMegaBytes = float(sys.getsizeof(pickle.dumps(tD, protocol=0))) / 1000000.0
            documentMegaBytes = float(sys.getsizeof(bson.BSON.encode(tD))) / 1000000.0
            logger.debug("%s Document %s %.4f MB", procName, cN, documentMegaBytes)
            maxDocumentMegaBytes = max(maxDocumentMegaBytes, documentMegaBytes)
            if documentMegaBytes > thresholdMB:
                logger.info("Large document %r  %.4f MB", cN, documentMegaBytes)
                for ky in tD:
                    try:
                        sMB = float(sys.getsizeof(bson.BSON.encode({"t": tD[ky]}))) / 1000000.0
                    except Exception:
                        sMB = -1
                    logger.info("Sub-document length %s sizeMB %.4f  %8d", ky, sMB, len(tD[ky]))
                #
        logger.info("%s maximum document size loaded %.4f MB", procName, maxDocumentMegaBytes)
        return True

    #

    def __writePathList(self, filePath, pathList):
        try:
            with open(filePath, "w") as ofh:
                for pth in pathList:
                    ofh.write("%s\n" % pth)
            return True
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __begin(self, message=""):
        startTime = time.time()
        ts = time.strftime("%Y %m %d %H:%M:%S", time.localtime())
        logger.debug("Starting %s at %s", message, ts)
        return startTime

    def __end(self, startTime, message=""):
        endTime = time.time()
        ts = time.strftime("%Y %m %d %H:%M:%S", time.localtime())
        delta = endTime - startTime
        logger.debug("Completed %s at %s (%.4f seconds)", message, ts, delta)

    def __createCollection(self, databaseName, collectionName, indexDL=None, bsonSchema=None):
        """Create database and collection and optionally a set of indices -
        """
        try:
            logger.debug("Create database %s collection %s", databaseName, collectionName)
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                ok1 = mg.createCollection(databaseName, collectionName, bsonSchema=bsonSchema)
                ok2 = mg.databaseExists(databaseName)
                ok3 = mg.collectionExists(databaseName, collectionName)
                okI = True
                if indexDL:
                    for indexD in indexDL:
                        okI = mg.createIndex(databaseName, collectionName, indexD["ATTRIBUTE_NAMES"], indexName=indexD["INDEX_NAME"], indexType="DESCENDING", uniqueFlag=False)

            return ok1 and ok2 and ok3 and okI
            #
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __updateCollectionSchema(self, databaseName, collectionName, bsonSchema=None, validationLevel="strict", validationAction="error"):
        """Update validation schema for the input collection -
        """
        try:
            logger.debug("Updating validatio for schema database %s collection %s", databaseName, collectionName)
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                ok1 = mg.databaseExists(databaseName)
                ok2 = mg.collectionExists(databaseName, collectionName)
                if ok1 and ok2:
                    ok3 = mg.updateCollection(databaseName, collectionName, bsonSchema=bsonSchema, validationLevel=validationLevel, validationAction=validationAction)
                    logger.info("Updated %r %r validation schema", databaseName, collectionName)
            return ok1 and ok2 and ok3
            #
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __removeCollection(self, databaseName, collectionName):
        """Drop collection within database

        """
        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                #
                logger.debug("Remove collection database %s collection %s", databaseName, collectionName)
                logger.debug("Starting databases = %r", mg.getDatabaseNames())
                logger.debug("Starting collections = %r", mg.getCollectionNames(databaseName))
                ok = mg.dropCollection(databaseName, collectionName)
                logger.debug("Databases = %r", mg.getDatabaseNames())
                logger.debug("Post drop collections = %r", mg.getCollectionNames(databaseName))
                ok = mg.collectionExists(databaseName, collectionName)
                logger.debug("Post drop collections = %r", mg.getCollectionNames(databaseName))
            return ok
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __purgeDocuments(self, databaseName, collectionName, cardinalIdL):
        """Purge documents from collection within database with cardinal identifiers in cardinalIdL.

        """
        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                for cardId in cardinalIdL:
                    selectD = {"rcsb_id": {"$regex": "^%s" % cardId.upper(), "$options": "i"}}  # case-insensitive
                    dCount = mg.delete(databaseName, collectionName, selectD)
                    logger.debug("Remove %d objects in database %s collection %s selection %r", dCount, databaseName, collectionName, selectD)
            return True
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return False

    def __pruneBySize(self, dList, limitMB=15.9):
        """ For the input list of objects (dictionaries).objects
            Return a pruned list satisfying the input total object size limit -

        """
        oL = []
        try:
            for dD in dList:
                sD = {}
                sumMB = 0.0
                for ky in dD:
                    dMB = 0.0
                    try:
                        dMB = float(sys.getsizeof(bson.BSON.encode({ky: dD[ky]}))) / 1000000.0
                    except Exception as e:
                        logger.error("ky %r d[ky] %r with %s", ky, dD[ky], str(e))

                    sumMB += dMB
                    sD[ky] = dMB
                if sumMB < limitMB:
                    oL.append(dD)
                    continue
                #
                sortedSd = sorted(sD.items(), key=operator.itemgetter(1))
                prunedSum = 0.0
                for ky, sMB in sortedSd:
                    prunedSum += sMB
                    if prunedSum > limitMB:
                        dD.pop(ky, None)
                        logger.debug("Pruning ky %s size(MB) %.2f", ky, sMB)
                oL.append(dD)
        except Exception as e:
            logger.exception("Failing with %s", str(e))
            #
        logger.debug("Pruning returns document list length %d", len(dList))
        return oL

    def __loadDocuments(self, databaseName, collectionName, dList, docIdL, replaceIdL=None, loadType="full", readBackCheck=False, pruneDocumentSize=None):
        #
        # Load database/collection with input document list -
        #
        rIdL = []

        logger.debug("databaseName %s collectionName %s docIdL %r", databaseName, collectionName, docIdL)
        inputDocIdS = {self.__getKeyValues(dD, docIdL) for dD in dList}
        failDocIdS = set()
        successDocIdS = set()

        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                #
                if loadType == "replace" and replaceIdL:
                    deleteTupL = mg.deleteList(databaseName, collectionName, dList, replaceIdL)
                    logger.debug("Deleted document status %r", deleteTupL)
                if pruneDocumentSize:
                    dList = self.__pruneBySize(dList, limitMB=pruneDocumentSize)
                #
                rIdL.extend(mg.insertList(databaseName, collectionName, dList, keyNames=docIdL, salvage=True))
                # ---
                #  If there is a failure then determine the specific successes and failures -
                #
                successDocIdS = inputDocIdS
                if len(rIdL) != len(dList):
                    sIdS = set()
                    try:
                        for rId in rIdL:
                            rObj = mg.fetchOne(databaseName, collectionName, "_id", rId)
                            dIdTup = self.__getKeyValues(rObj, docIdL)
                            sIdS.add(dIdTup)
                    except Exception as e:
                        logger.exception("Failing with %s", str(e))
                    successDocIdS = sIdS
                # enumerate the failures
                failDocIdS = inputDocIdS - successDocIdS
                #

                #
                if readBackCheck:
                    # build an index of the input document list
                    indD = {}
                    try:
                        for ii, dD in enumerate(dList):
                            dIdTup = self.__getKeyValues(dD, docIdL)
                            indD[dIdTup] = ii
                    except Exception as e:
                        logger.exception("Failing ii %d d %r with %s", ii, dD, str(e))
                    #
                    # Note that objects in dList are mutated by the insert operation with the additional key '_id',
                    # hence, it is possible to compare the fetched object with the input object.
                    #
                    rbStatus = True
                    for ii, rId in enumerate(rIdL):
                        rObj = mg.fetchOne(databaseName, collectionName, "_id", rId)
                        dIdTup = self.__getKeyValues(rObj, docIdL)
                        jj = indD[dIdTup]
                        if rObj != dList[jj]:
                            rbStatus = False
                            break
                #
                if readBackCheck and not rbStatus:
                    return False, successDocIdS, failDocIdS
                #
            return len(rIdL) == len(dList), successDocIdS, failDocIdS
        except Exception as e:
            logger.exception("Failing with %s", str(e))

        return False, [], inputDocIdS

    def __getKeyValues(self, dct, keyNames):
        """Return the tuple of values corresponding to the input dictionary of key names expressed in dot notation.

        Args:
            dct (dict): source dictionary object (nested)
            keyNames (list): list of dictionary keys in dot notation

        Returns:
            tuple: tuple of values corresponding to the input key names

        """
        rL = []
        try:
            for keyName in keyNames:
                rL.append(self.__getKeyValue(dct, keyName))
        except Exception as e:
            logger.exception("Failing for key names %r with %s", keyNames, str(e))

        return tuple(rL)

    def __getKeyValue(self, dct, keyName):
        """  Return the value of the corresponding key expressed in dot notation in the input dictionary object (nested).
        """
        try:
            kys = keyName.split(".")
            for key in kys:
                try:
                    dct = dct[key]
                except Exception:
                    logger.warning("Missing document key %r in %r", key, list(dct.keys()))
                    return None
            return dct
        except Exception as e:
            logger.error("Failing for key %r with %s", keyName, str(e))

        return None
