#----------------------------------------------------------------------------
# Name:         ldmStorageFolder.py
# Purpose:      storage folder content
#
# Author:       Walter Obweger
#
# Created:      20200322
# CVS-ID:       $Id$
# Copyright:    Walter Obweger
# Licence:      MIT
#----------------------------------------------------------------------------

import logging
import traceback

import os
from datetime import datetime as dt

from optparse import OptionParser

from lindworm import __version__
from lindworm.ldmStorage import ldmStorage
from lindworm.ldmOS import getSha

class ldmStorageFolder(ldmStorage):
    """folder storage class
    """
    def __init__(self,iModeRev=0,sLogger='srgFld',iLv=0,iVerbose=0):
        """constructor
        
        Args:
            iModeRev (int): reverse processing
                    - 1 : FILO, first in last out
                    - 0 : FIFO, first in first out
            sLogger (str): log origin
            iLv (int): logging level
            iVerbose (int): higher values add more logs
        """
        ldmStorage.__init__(self,iModeRev=iModeRev,
                            sLogger=sLogger,iLv=iLv,
                            iVerbose=iVerbose)
    def __initCfg__(self):
        """initialize configuration properties
        """
        try:
            ldmStorage.__initCfg__(self)
            # +++++ beg:initialize
            sOrg='ldmStorage::__initCfg__'
            self.logDbg('beg:%s',sOrg)
            # ----- end:initialize
            # +++++ beg:default configuration
            self.dCfgDft['lSkipDN']=['CVS','.git']
            self.dCfgDft['sTimePrecision']='seconds'
            self.dCfgDft['sStatNew']='zNew'
            self.dCfgDft['sStatUse']='zUse'
            self.dCfgDft['sStatMod']='zMod'
            #self.dCfgDft['zPrecision']='minutes'
            # ----- end:default configuration
            self.logDbg('end:%s',sOrg)
            return 1
        except:
            self.logTB()
            return -1
    def __initPrc__(self):
        """initialize processing properties
        """
        try:
            ldmStorage.__initPrc__(self)
            # +++++ beg:initialize
            sOrg='ldmStorageFolder::__initDat__'
            self.logDbg('beg:%s',sOrg)
            # ----- end:initialize
            # +++++ beg:initialize data
            self.iCnt=0             # count of directories
            self.iAct=0             # 
            self.iSrcLen=0          # source DN length, used to build relative
            self.lCnt=[]            # files in relative DN
            self.lFdr=[]            # folder list as they arrived, relative DN
            self.dFolder={}         # content cache, relative DN
            # ----- end:initialize data
            self.logDbg('end:%s',sOrg)
            return 1
        except:
            self.logTB()
            return -1
    def clrEnd(self):
        """clear stored file names to process
        """
        iRet=ldmStorage.clrEnd(self)
        self.dCntEnd=None
        return iRet
    def getDatRel(self,sRelDN,sBaseDN=None):
        """get relative content from data
        
        Args:
            sRelDN (str): relative DN
            sBaseDN (str): base DN
        
        Returns:
            tuple
                - return code
                    - >0 : okay processing done
                    - =0 : okay nop
                    - <0 : error

                - dDN (dict): directory content
                    - dict : {'.':...}
                    - None : invalid content
        """
        try:
            # +++++ beg:get relative path list
            iR,dDN=self.getDat(sDef=sBaseDN)
            if iR<1:
                self.oLog.error('getDatRel: sFN:%s iR:%d dDat:%r'%(sBaseDN,iR,self.dDat))
                return -1,None
            if sRelDN=='./':
                lRelDN=['.']
            else:
                lRelDN=sRelDN.split('/')
            #if self.iVerbose>0:
            #    self.logDbg('            lRelDN:%r',lRelDN)
            # ----- end:get relative path list
            # +++++ beg:get dDN storage
            for sDN in lRelDN[1:]:
                if sDN not in dDN:
                    return 0,None
                dDN=dDN[sDN]
            return 1,dDN
            # ----- end:get dDN storage
        except:
            self.logTB()
            return -1,None
    def getDatContent(self,sRoot):
        """get content from data
        
        Args:
            sRoot (str): directory
        
        Returns:
            tuple
                - sCurDN (str): current directory name, relative
                
                - sDN (str): directory content
                
                - sFN (str): file content
        """
        try:
            if self.iVerbose>90:
                self.logDbg('        sRoot:%s',sRoot)
            # +++++ beg:calc relative DN posix style
            sCurDN=sRoot[self.iSrcLen:].replace(os.sep,'/')
            if sCurDN=='':
                sCurDN='./'
            elif sCurDN[0]=='/':
                sCurDN='.'+sCurDN
            else:
                sCurDN='./'+sCurDN
            if self.iVerbose>0:
                self.logDbg('        sCurDN:%s',sCurDN)
            # ----- end:calc relative DN posix style
            # +++++ beg:prepare dDN storage
            iR,dDN=self.getDat()
            if iR<1:
                self.oLog.error('getDat() iR:%d dDN:%r'%(iR,dDN))
                dDN={}
            else:
                if self.iVerbose>0:
                    lKey=list(dDN.keys())
                    lKey.sort()
                    self.logDbg('            getDat() iR:%d dDN keys:%r',iR,lKey)
                if self.iVerbose>95:
                    self.logDbg('            getDat() iR:%d dDN:%r',iR,dDN)
            if sCurDN=='./':
                lCurDN=['.']
                if '.' not in dDN:
                    dDN['.']={}
            else:
                lCurDN=sCurDN.split('/')
            if self.iVerbose>0:
                self.logDbg('            lCurDN:%r',lCurDN)
            # ----- end:prepare dDN storage
            # +++++ beg:get dDN storage
            for sDN in lCurDN[1:]:
                if sDN not in dDN:
                    dDN[sDN]={'.':{}}
                dDN=dDN[sDN]
            dFN=dDN['.']
            # ----- end:get dDN storage
            # +++++ beg:build cache
            self.iCnt+=1
            self.lFdr.append(sCurDN)
            self.dFolder[sCurDN]=dDN
            # ----- end:build cache
            return sCurDN,dDN,dFN
        except:
            self.logTB()
            return None,None,None
    def prcBeg(self,sDef,oRef=None,**kwargs):
        """processing begin, add file name and objects to stack
        
        Args:
            sSrcDN (str): source directory name : str
            oDat (obj): data object
            oRef (obj): reference object
            kwargs : flexible keyword argument
        
        Returns:
            >0  ... okay processing done
            =0  ... okay nop
            <0  ... error
        """
        try:
            # +++++ beg:initialize
            iRet=0
            sOrg='ldmStorageFolder::prcBeg'
            # ----- end:initialize
            # +++++ beg:processing begin
            self.logDbg('beg:%s sDef:%s',sOrg,sDef)
            if self.iVerbose>0:
                self.logDbg('    oRef:%r kwargs:%r',oRef,kwargs)
            dRef={
                'iAct':self.iAct,
                'oRef':oRef,
            }
            iRet=ldmStorage.prcBeg(self,sDef,oRef=dRef,**kwargs)
            if iRet>0:
                self.iAct=0
            self.logDbg('end:%s iRet:%d',sOrg,iRet)
            # ----- end:processing begin
            return iRet
        except:
            self.logTB()
            return -1
    def prcExc(self,oNty=None,**kwargs):
        """processing execution
        
        Args:
            kwargs : flexible keyword argument
        
        Returns:
            return code
                - >0 : okay processing done
                - =0 : okay nop
                - <0 : error
        """
        try:
            # +++++ beg:initialize
            iRet=0
            sOrg='ldmStorageFolder::prcExc'
            # ----- end:initialize
            # +++++ beg:get definition
            iRet,sDef=self.getDef()
            if iRet<1:
                self.logDbg('skp:%s iRet:%d empty stack',sOrg,iRet)
                return iRet
            self.logDbg('beg:%s sDef:%s',sOrg,sDef)
            if self.iVerbose>0:
                self.logDbg('    kwargs:%r',kwargs)
            self.iSrcLen=len(sDef)
            # ----- end:get definition
            # +++++ beg:get folder list to skip
            iR,lSkipDN=self.getCfg('lSkipDN')
            if self.iVerbose>0:
                self.logDbg('    iR:%d lSkipDN:%r',iR,lSkipDN)
            # ----- end:get folder list to skip
            # +++++ beg:get configuration
            iR,sTimePrecision=self.getCfg('sTimePrecision')
            iR,sStatNew=self.getCfg('sStatNew',oDft=0)
            iR,sStatMod=self.getCfg('sStatMod',oDft=0)
            iR,sStatUse=self.getCfg('sStatUse',oDft=0)
            # ----- end:get configuration
            # +++++ beg:walk through directory recursive
            for sRoot,lDN,lFN in os.walk(sDef):
                if self.iVerbose>90:
                    self.logDbg('        sRoot:%s',sRoot)
                sCurDN,dDN,dFN=self.getDatContent(sRoot)
                if oNty is not None:
                    oNty.SetStatus(sCurDN)
                if self.iVerbose>0:
                    self.logDbg('        sCurDN:%s',sCurDN)
                #dFiles={}
                #dContent={'.':dFiles}
                #self.lFdr.append(sCurDN)
                #self.dFolder[sCurDN]=dContent
                if dFN is not None:
                    for sFN in lFN:
                        iR,oRef=self.getRef()
                        iR,dStat=self.getStat(sRoot,sFN,oRef,
                                        sTimePrecision=sTimePrecision,
                                        sStatNew=sStatNew,
                                        sStatMod=sStatMod,
                                        sStatUse=sStatUse,
                                        **kwargs)
                        if iR>0:
                            if self.iVerbose>0:
                                self.logDbg('          sFN:%s dStat:%r',sFN,dStat)
                            dFN[sFN]=dStat
                if self.iVerbose>5:
                    iLenFN=len(dFN)
                    self.logDbg('        sCurDN:%s iLenFN:%r',sCurDN,iLenFN)
                for sSkipDN in lSkipDN:
                    if sSkipDN in lDN:
                        lDN.remove(sSkipDN)
            # ----- end:walk through directory recursive
            # +++++ beg:finalize
            if self.iVerbose>90:
                self.logDbg('   :%s sDef:%s dDat:%r',sOrg,sDef,self.dDat)
            self.logDbg('end:%s iRet:%d',sOrg,iRet)
            # ----- end:finalize
            return iRet
        except:
            self.logTB()
            return -1
    def prcEnd(self,**kwargs):
        """processing end

        use updated properties to access current properties to be finalized
            - self.sDefEnd : definition
            - self.oArgEnd : keyword arguments passed at prcBeg
            - self.oDatEnd : data object
            - self.oRefEnd : reference object

        Args:
            kwargs : flexible keyword argument
            
        Returns:
            return code
                - >0 : okay processing done
                - =0 : okay nop
                - <0 : error
        """
        try:
            # +++++ beg:initialize
            iRet=0
            sOrg='ldmStorageFolder::prcEnd'
            # ----- end:initialize
            # +++++ beg:processing end
            self.logDbg('beg:%s len(lFdr):%d kwargs:%r',sOrg,
                                len(self.lFdr),kwargs)
            iRet=ldmStorage.prcEnd(self,**kwargs)
            if iRet>0:
                # +++++ beg:revert current file info
                #self.iAct=iMarkerAct    # revert current file line back
                # ----- end:revert current file info
                if self.iVerbose>5:
                    self.logDbg('    %s iAct:%5d iCnt:%6d sDefEnd:%s',sOrg,
                                    self.iAct,self.iCnt,self.sDefEnd)
                if self.iVerbose>95:
                    self.logDbg('   :%s oDatEnd:%r',sOrg,
                                    self.oDatEnd)
                # +++++ beg:loop through folders
                iOfsFdr=0
                lCntKey=['sz','iDN','iFN',]
                dCntDef={
                    'sz':0,
                    'iDN':0,
                    'iFN':0,
                }
                for sRelDN in self.lFdr:
                    iR,dDN=self.getDatRel(sRelDN,sBaseDN=self.sDefEnd)
                    if self.iVerbose>5:
                        self.logDbg('   :%s sRelDN:%s iR:%d sDefEnd:%r',sOrg,
                                        sRelDN,iR,self.sDefEnd)
                    iSz=0
                    iLenFN=0
                    iLenDN=0
                    if iR>0:
                        if self.iVerbose>5:
                            self.logDbg('   :%s sRelDN:%s iR:%d len(dDN):%d',sOrg,
                                            sRelDN,iR,len(dDN))
                        if self.iVerbose>95:
                            self.logDbg('   :%s sRelDN:%s iR:%d dDN:%r',sOrg,
                                            sRelDN,iR,dDN)
                        dFN=dDN.get('.',None)
                        iLenDN=len(dDN)-1
                        if dFN is not None:
                            for sFN,dStat in dFN.items():
                                iSz+=dStat.get('sz',0)
                                iLenFN+=1
                    dCnt={
                        'sRelDN':sRelDN,
                        'sz':iSz,
                        'iDN':iLenDN,
                        'iFN':iLenFN,
                        }
                    self.lCnt.append(dCnt)
                    # +++++ beg:update count total
                    for sK in lCntKey:
                        dCntDef[sK]=dCntDef[sK]+dCnt[sK]
                    # ----- end:update count total
                    iOfsFdr+=1
                # ----- end:loop through folders
                # +++++ beg:post processing
                self.dCntEnd=dCntDef
                iR=self.prcPost(**kwargs)
                # ----- end:post processing
                if self.iVerbose>5:
                    self.logDbg('   :%s lCnt:%r',sOrg,self.lCnt)
            self.logDbg('end:%s iRet:%d',sOrg,iRet)
            # ----- end:processing end
            return iRet
        except:
            self.logTB()
            return -1
    def prcPost(self,**kwargs):
        """processing end

        Args:
            **kwargs ... flexible keyword argument
                iShaMB  ... data to calculate sha in MB 
        Returns:
            return code
                - >0 : okay processing done
                - =0 : okay nop
                - <0 : error
        """
        try:
            # +++++ beg:initialize
            iRet=0
            sOrg='ldmStorageFolder::prcPost'
            iShaMB=kwargs.get('iShaMB',0)
            # ----- end:initialize
            if self.sDefEnd is None:
                self.logDbg('skp:%s iRet:%d empty definition,'
                                'prcEnd has to be call before',sOrg,iRet)
                return iRet
            # +++++ beg:folder post processing
            iOfsFld=-1
            iCntFN=0
            self.logDbg('beg:%s sDefEnd:%s len(lFdr):%d kwargs:%r',sOrg,
                                self.sDefEnd,len(self.lFdr),kwargs)
            oNty=kwargs.get('oNty',None)
            if oNty is not None:
                self.logDbg('    dCntEnd:%r',self.dCntEnd)
                oNty.SetMax(self.dCntEnd['iFN'])
            for sRelDN in self.lFdr:
                iOfsFld+=1
                if sRelDN in self.dFolder:
                    if oNty is not None:
                        oNty.SetStatus(sRelDN)
                        oNty.IncStatus()
                    sDN='/'.join([self.sDefEnd,sRelDN])
                    dDN=self.dFolder[sRelDN]
                    if self.iVerbose>5:
                        self.logDbg('   :%s sRelDN:%s len(dDN):%d',sOrg,
                                        sRelDN,len(dDN))
                    if self.iVerbose>9:
                        self.logDbg('    dDN:%r',dDN)
                    if '.' in dDN:
                        dFN=dDN['.']
                        # +++++ beg:calc sha fingerprint
                        for sFN,dStat in dFN.items():
                            iCntFN+=1
                            if oNty is not None:
                                #oNty.SetStatus(sFN)
                                #oNty.IncStatus()
                                oNty.SetVal(iCntFN)
                            sSha=getSha(sFN,iMB=iShaMB,
                                        sDN=sDN,
                                        oLog=self.oLog)
                            if self.iVerbose>5:
                                self.logDbg('      sFN:%s sha:%s',sFN,
                                                sSha)
                            dStat['sha']=sSha
                        # ----- end:calc sha fingerprint
            # ----- end:folder post processing
            #if oNty is not None:
            #S    oNty.clrStatus()
            self.logDbg('end:%s iRet:%d',sOrg,iRet)
            return iRet
        except:
            self.logTB()
            return -1
    def getStatDct(self,sFN,oStat,dStat,oRef,**kwargs):
        return 0
    def getStatTimeStamp(self,sFN,oStat,dStat,oRef,
                    sTimePrecision='minutes',
                    sStatNew=0,
                    sStatMod=0,
                    sStatUse=0,
                    **kwargs):
        try:
            iRet=0
            if sStatNew:
                oTmp=dt.fromtimestamp(oStat.st_ctime)
                dStat[sStatNew]=oTmp.isoformat(timespec=sTimePrecision)
                iRet+=1
            if sStatMod:
                oTmp=dt.fromtimestamp(oStat.st_mtime)
                dStat[sStatMod]=oTmp.isoformat(timespec=sTimePrecision)
            if sStatUse:
                oAccess=dt.fromtimestamp(oStat.st_atime)
                dStat[sStatUse]=oTmp.isoformat(timespec=sTimePrecision)
            return iRet
        except:
            self.logTB()
        return 0
    def getStat(self,sDN,sFN,oRef,**kwargs):
        try:
            if sDN is not None:
                sTmpFN=os.path.join(sDN,sFN)
            else:
                sTmpFN=sFN
            if os.path.exists(sTmpFN):
                # +++++ beg:read file status information
                oStat=os.stat(sTmpFN)
                # ----- end:read file status information
                # +++++ beg:
                dStat={
                    'sz':oStat.st_size,
                }
                iR=self.getStatTimeStamp(sTmpFN,oStat,dStat,oRef,**kwargs)
                iR=self.getStatDct(sTmpFN,oStat,dStat,oRef,**kwargs)
                return 1,dStat
            return 0,None
        except:
            self.logTB()
            return -1,None

def execMain(sSrcDN='.',
                sBldDN=None,
                sBldFN=None,
                sCfgFN=None,
                iShaMB=1,
                iVerbose=0):
    """main function to perform conversion
    """
    try:
        # +++++ beg:
        # ----- end:
        # +++++ beg:
        logging.debug('beg:execMain')
        iRet=0
        # ----- end:
        # +++++ beg:
        oFld=ldmStorageFolder(iVerbose=iVerbose)
        iRet=oFld.loadCfg(sCfgFN)
        # ----- end:
        # +++++ beg:
        oFld.prcBeg(sSrcDN,oRef=None)
        oFld.prcExc()#(oGtrMD=self)
        iRetFdr=oFld.prcEnd(iShaMB=iShaMB)
        if iRetFdr>0:
            iRet+=1
        # ----- end:
        # +++++ beg:
        if iRetFdr>0:
            iRetSave=oFld.saveDat(sBldFN,sDN=sBldDN,
                                lKey=[oFld.sDefEnd],
                                sAtr='dDat')
        # ----- end:
        # +++++ beg:
        logging.debug('end:execMain iRet:%d'%(iRet))
        # ----- end:
        return iRet
    except:
        logging.error(traceback.format_exc())
        return -1

def main(args=None):
    # +++++ beg:
    # ----- end:
    
    # +++++ beg:init
    iVerbose=5                                          # 20190624 wro:set default verbose level
    # ----- end:init
    # +++++ beg:define CLI arguments
    usage = "usage: %prog [options]"
    oParser=OptionParser(usage,version="%prog "+__version__)
    oParser.add_option('-c','--cfgFN',dest='sCfgFN',
            default='ldmStorageFolderCfg.json',
            help='configuration file',metavar='pyGatherMDCfg.json',
            )
    oParser.add_option('','--srcDN',dest='sSrcDN',
            default='.',
            help='source folder',metavar='path/to/folder/to/read',
            )
    oParser.add_option('','--bldDN',dest='sBldDN',
            default=None,
            help='build directory',metavar='path/to/output/folder',
            )
    oParser.add_option('','--otBldFN',dest='sBldFN',
            default=None,
            help='build file',metavar='sng.json',
            )
    oParser.add_option('-l','--log',dest='sLogFN',
            default='./log/ldmStorageFolder.log',
            help='log filename',metavar='./log/ldmStorageFolder.log')
    oParser.add_option("-v", action="store_true", dest="verbose", default=True)
    oParser.add_option("-q", action="store_false", dest="verbose", default=True)
    # ----- end:define CLI arguments
    # +++++ beg:parse command line
    (opt,args)=oParser.parse_args(args=args)
    if opt.verbose:
        print("config FN:     %r"%(opt.sCfgFN))
        print("source DN:     %r"%(opt.sSrcDN))
        print(" build DN:     %r"%(opt.sBldDN))
        print(" build FN:     %r"%(opt.sBldFN))
        iVerbose=20
    # ----- end:parse command line
    # +++++ beg:prepare logging
    import lindworm.logUtil as logUtil
    logUtil.logInit(opt.sLogFN,iLevel=logging.DEBUG)
    # ----- end:prepare logging
    # +++++ beg:status
    print('logging file:%r'%(opt.sLogFN))
    # ----- end:status
    # +++++ beg:perform main action
    iRet=execMain(sSrcDN=opt.sSrcDN,
                sBldDN=opt.sBldDN,
                sBldFN=opt.sBldFN,
                sCfgFN=opt.sCfgFN,
                iVerbose=iVerbose)
    # ----- end:perform main action
    # +++++ beg:finished
    print('\nend processing')
    print('   iRet:%r'%(iRet))
    # ----- end:finished
    return iRet

if __name__=='__main__':
    # +++++ beg:call entry point
    main(args=None)
    # ----- end:call entry point
