#!/usr/bin/python
#backups.py
#
#    Copyright DataHaven.NET LTD. of Anguilla, 2006-2013
#    Use of this software constitutes acceptance of the Terms of Use
#      http://datahaven.net/terms_of_use.html
#    All rights reserved.
#

import os
import sys
import time
import random
import gc


try:
    from twisted.internet import reactor
except:
    sys.exit('Error initializing twisted.internet.reactor in backups.py')

from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet import threads


import lib.dhnio as dhnio
import lib.misc as misc
import lib.nameurl as nameurl
import lib.transport_control as transport_control
import lib.settings as settings
import lib.contacts as contacts
import lib.eccmap as eccmap
import lib.tmpfile as tmpfile
import lib.diskspace as diskspace
import lib.packetid as packetid
from lib.automat import Automat


import backup_monitor
import backup_rebuilder  
import fire_hire
import list_files_orator
import lib.automats as automats

import raidread
import p2p_service
import io_throttle
import backup_db
import contact_status

#------------------------------------------------------------------------------ 

_RemoteFiles = {}
_LocalFiles = {}
_RemoteMaxBlockNumbers = {}
_LocalMaxBlockNumbers = {}
_LocalBackupSize = {}
_BackupsInProcess = []
_SuppliersSet = None
_BackupStatusNotifyCallback = None
_StatusCallBackForGuiBackup = None
_LocalFilesNotifyCallback = None
_UpdatedBackupIDs = set()
_RepaintingTask = None   
_RepaintingTaskDelay = 2.0

#------------------------------------------------------------------------------ 

def init():
    dhnio.Dprint(4, 'backups.init')
    RepaintingProcess(True)
    ReadLocalFiles()
    ReadLatestRawListFiles()


def shutdown():
    dhnio.Dprint(4, 'backups.shutdown')
    RepaintingProcess(False)

#------------------------------------------------------------------------------ 

# index for all remote files (on suppliers HDD's) stored in dictionary
# values are -1, 0 or 1 - this mean file is missing, no info yet, or its existed 
# all entries are indexed by [backupID][blockNum]['D' or 'P'][supplierNum]
def remote_files():
    global _RemoteFiles
    return _RemoteFiles


# max block number for every remote backup ID
def remote_max_block_numbers():
    global _RemoteMaxBlockNumbers
    return _RemoteMaxBlockNumbers


# index for all local files (on our HDD) stored in dictionary
# values are 0 or 1 - this mean file exist or not 
# all entries are indexed by [backupID][blockNum]['D' or 'P'][supplierNum]
def local_files():
    global _LocalFiles
    return _LocalFiles


# max block number for every local backup ID
def local_max_block_numbers():
    global _LocalMaxBlockNumbers
    return _LocalMaxBlockNumbers


def local_backup_size():
    global _LocalBackupSize
    return _LocalBackupSize


# currently working backups (started from dobackup.py)
def backups_in_process():
    global _BackupsInProcess
    return _BackupsInProcess


def suppliers_set():
    global _SuppliersSet
    if _SuppliersSet is None:
        _SuppliersSet = SuppliersSet(contacts.getSupplierIDs())
    return _SuppliersSet

#------------------------------------------------------------------------------ 
# this should represent the set of suppliers for either the user or for a customer the user
# is acting as a scrubber for
class SuppliersSet:
    def __init__(self, supplierList):
        self.suppliers = [] 
        self.supplierCount = 0 
        self.UpdateSuppliers(supplierList)

    def GetActiveArray(self):
        activeArray = [0] * self.supplierCount
        for i in range(self.supplierCount):
            if not self.suppliers[i]:
                continue
            if contact_status.isOnline(self.suppliers[i]):
                activeArray[i] = 1
            else:
                activeArray[i] = 0
        return activeArray

    def ChangedArray(self, supplierList):
        changedArray = [0] * self.supplierCount
        changedIdentities = []
        for i in range(self.supplierCount):
            if not self.suppliers[i]:
                continue
            if supplierList[i] != self.suppliers[i]:
                changedArray[i] = 1
                changedIdentities.append(supplierList[i])
        return changedArray, changedIdentities

    def SuppliersChanged(self, supplierList):
        if len(supplierList) != self.supplierCount:
            return True
        for i in range(self.supplierCount):
            if not self.suppliers[i]:
                continue
            if supplierList[i] != self.suppliers[i]:
                return True
        return False

    # if suppliers 1 and 3 changed, return [1,3]
    def SuppliersChangedNumbers(self, supplierList):
        changedList = []
        for i in range(self.supplierCount):
            if not self.suppliers[i]:
                continue
            if supplierList[i] != self.suppliers[i]:
                changedList.append(i)
        return changedList

    def SupplierCountChanged(self, supplierList):
        if len(supplierList) != self.supplierCount:
            return True
        else:
            return False

    def UpdateSuppliers(self, supplierList):
        self.suppliers = supplierList
        self.supplierCount = len(self.suppliers)

#------------------------------------------------------------------------------ 

def IncomingListFiles(supplierNum, packet, listFileText):
    supplierIdentity = packet.OwnerID
    #self.requestedListFilesPacketIDs.discard(supplierIdentity)
    #dhnio.Dprint(8, 'backups.IncomingListFiles for supplier %d other requests: %s' % (supplierNum, self.requestedListFilesPacketIDs))

    if supplierNum >= suppliers_set().supplierCount:
        dhnio.Dprint(2, 'backups.IncomingListFiles WARNING incorrect supplier number %d, number of suppliers is %d' % (supplierNum, suppliers_set().supplierCount))
        return

    backups2remove = ReadRawListFiles(supplierNum, listFileText)
    list_files_orator.IncommingListFiles(supplierNum, packet, listFileText)
    SaveLatestRawListFiles(supplierIdentity, listFileText)
    if len(backups2remove) > 0:
        p2p_service.RequestDeleteListBackups(backups2remove)


def ReadRawListFiles(supplierNum, listFileText):
    backups2remove = []
    # new code            
    # examples:
    # F20090709034221PM-0-Data from 0-1000
    # F20090709034221PM-0-Data from 0-1000 missing 1,3,
    for line in listFileText.splitlines():
        line = line.strip()
        # comment lines in Files reports start with blank,
        if line == '':
            continue
        # also don't consider the identity a backup,
        if line.find('http://') != -1 or line.find('.xml') != -1:
            continue
        # nor backup_info.xml, backup_db 
        if line in [ settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName() ]:
            continue
        words = line.split(' ')
        # minimum is 3 words: "F20090709034221PM-0-Data", "from", "0-1000"
        if len(words) < 3:
            dhnio.Dprint(2, 'backups.ReadRawListFiles WARNING incorrect line:[%s]' % line)
            continue
        try:
            backupID, lineSupplierNum, dataORparity = words[0].split('-')
            minBlockNum, maxBlockNum = words[2].split('-')
            lineSupplierNum = int(lineSupplierNum)
            maxBlockNum = int(maxBlockNum)
        except:
            dhnio.Dprint(2, 'backups.ReadRawListFiles WARNING incorrect line:[%s]' % line)
            continue
        if backupID == '':
            dhnio.Dprint(2, 'backups.ReadRawListFiles WARNING incorrect line:[%s]' % line)
            continue
        if dataORparity not in ['Data', 'Parity']:
            dhnio.Dprint(2, 'backups.ReadRawListFiles WARNING incorrect line:[%s]' % line)
            continue
        if lineSupplierNum != supplierNum:
            # this mean supplier have old files and we do not need it 
            # TODO what to do...  remove the files? Send "DeleteFile" packet?
            backups2remove.append(backupID)
            continue
        if backup_db.IsDeleted(backupID):
            backups2remove.append(backupID)
            continue
        missingBlocksSet = set()
        if len(words) ==  5:
            if words[3].strip() != 'missing':
                dhnio.Dprint(2, 'backups.ReadRawListFiles WARNING incorrect line:[%s]' % line)
                continue
            missingBlocksSet = set(words[4].split(','))
        if not remote_files().has_key(backupID):
            remote_files()[backupID] = {}
            dhnio.Dprint(6, 'backups.ReadRawListFiles new remote entry for %s created in the memory' % backupID)
        # +1 because range(2) give us [0,1] but we want [0,1,2]
        for blockNum in range(maxBlockNum+1):
            if not remote_files()[backupID].has_key(blockNum):
                remote_files()[backupID][blockNum] = {
                    'D': [0] * suppliers_set().supplierCount,
                    'P': [0] * suppliers_set().supplierCount,}
            # we set -1 if the file is missing and 1 if exist, so 0 mean "no info yet" ... smart!
            bit = -1 if str(blockNum) in missingBlocksSet else 1 
            if dataORparity == 'Data':
                remote_files()[backupID][blockNum]['D'][supplierNum] = bit
            elif dataORparity == 'Parity':
                remote_files()[backupID][blockNum]['P'][supplierNum] = bit
        # save max block number for this backup
        if not remote_max_block_numbers().has_key(backupID):
            remote_max_block_numbers()[backupID] = -1 
        if maxBlockNum > remote_max_block_numbers()[backupID]:
            remote_max_block_numbers()[backupID] = maxBlockNum
        
        # mark this backup to be repainted
        RepaintBackup(backupID)
    # return list of backupID's which is too old but stored on suppliers machines 
    return backups2remove
            

def SaveLatestRawListFiles(idurl, listFileText):
    supplierPath = settings.SupplierPath(idurl)
    if not os.path.isdir(supplierPath):
        try:
            os.makedirs(supplierPath)
        except:
            dhnio.DprintException()
    dhnio.WriteFile(settings.SupplierListFilesFilename(idurl), listFileText)


def ReadLatestRawListFiles():
    dhnio.Dprint(4, 'backups.ReadLatestRawListFiles')
    for idurl in contacts.getSupplierIDs():
        if idurl:
            filename = os.path.join(settings.SupplierPath(idurl, 'listfiles'))
            if os.path.isfile(filename):
                listFileText = dhnio.ReadTextFile(filename)
                if listFileText.strip() != '':
                    ReadRawListFiles(contacts.numberForSupplier(idurl), listFileText)


def ScanMissingBlocks(backupID):
    missingBlocks = set()
    localMaxBlockNum = local_max_block_numbers().get(backupID, -1)
    remoteMaxBlockNum = remote_max_block_numbers().get(backupID, -1)
    supplierActiveArray = suppliers_set().GetActiveArray()
    #dhnio.Dprint(6, 'backups.ScanMissingBlocks supplierActiveArray=%s' % supplierActiveArray)

    if not remote_files().has_key(backupID):
        if not local_files().has_key(backupID):
            # we have no local and no remote info for this backup
            # no chance to do some rebuilds...
            # TODO but how we get here ?! 
            dhnio.Dprint(4, 'backups.ScanMissingBlocks no local and no remote info for %s' % backupID)
        else:
            # we have no remote info, but some local files exists
            # so let's try to sent all of them
            # need to scan all block numbers 
            for blockNum in range(localMaxBlockNum):
                # we check for Data and Parity packets
                localData = GetLocalDataArray(backupID, blockNum)
                localParity = GetLocalParityArray(backupID, blockNum)  
                for supplierNum in range(len(supplierActiveArray)):
                    # if supplier is not alive we can not send to him
                    # so no need to scan for missing blocks 
                    if supplierActiveArray[supplierNum] != 1:
                        continue
                    if localData[supplierNum] == 1:
                        missingBlocks.add(blockNum)
                    if localParity[supplierNum] == 1:
                        missingBlocks.add(blockNum)
    else:
        # now we have some remote info
        # we take max block number from local and remote
        maxBlockNum = max(remoteMaxBlockNum, localMaxBlockNum)
        # dhnio.Dprint(6, 'backups.ScanMissingBlocks maxBlockNum=%d' % maxBlockNum)
        # and increase by one because range(3) give us [0, 1, 2], but we want [0, 1, 2, 3]
        for blockNum in range(maxBlockNum + 1):
            # if we have few remote files, but many locals - we want to send all missed 
            if not remote_files()[backupID].has_key(blockNum):
                missingBlocks.add(blockNum)
                continue
            # take remote info for this block
            remoteData = GetRemoteDataArray(backupID, blockNum)
            remoteParity = GetRemoteParityArray(backupID, blockNum)  
            # now check every our supplier for every block
            for supplierNum in range(len(supplierActiveArray)):
                # if supplier is not alive we can not send to him
                # so no need to scan for missing blocks 
                if supplierActiveArray[supplierNum] != 1:
                    continue
                if remoteData[supplierNum] != 1:    # -1 means missing
                    missingBlocks.add(blockNum)     # 0 - no info yet
                if remoteParity[supplierNum] != 1:  # 1 - file exist on remote supplier 
                    missingBlocks.add(blockNum)
                
    # dhnio.Dprint(6, 'backups.ScanMissingBlocks %s' % missingBlocks)
    return list(missingBlocks)

def ScanBlocksToRemove(backupID, check_all_suppliers=True):
    packets = []
    localMaxBlockNum = local_max_block_numbers().get(backupID, -1)
    if not remote_files().has_key(backupID) or not local_files().has_key(backupID):
        return packets
    for blockNum in range(localMaxBlockNum + 1):
        localArray = {'Data': GetLocalDataArray(backupID, blockNum),
                      'Parity': GetLocalParityArray(backupID, blockNum)}  
        remoteArray = {'Data': GetRemoteDataArray(backupID, blockNum),
                       'Parity': GetRemoteParityArray(backupID, blockNum)}  
        dhnio.Dprint(6, 'backups.ScanBlocksToRemove %s [%d] remote=%s local=%s' % (backupID, blockNum, str(remoteArray), str(localArray)))
        for supplierNum in range(suppliers_set().supplierCount):
            supplierIDURL = suppliers_set().suppliers[supplierNum]
            if not supplierIDURL:
                continue
            for dataORparity in ['Data', 'Parity']:
                packetID = packetid.MakePacketID(backupID, blockNum, supplierNum, dataORparity)
                if io_throttle.HasPacketInSendQueue(supplierIDURL, packetID):
                    continue
                if check_all_suppliers:
                    if ( 0 not in remoteArray[dataORparity] ) and localArray[dataORparity][supplierNum] == 1:  
                        packets.append(packetID)
                else:
                    if remoteArray[dataORparity][supplierNum] == 1 and localArray[dataORparity][supplierNum] == 1:  
                        packets.append(packetID)
    return packets

def ScanBlocksToSend(backupID):
    if '' in suppliers_set().suppliers:
        return {} 
    localMaxBlockNum = local_max_block_numbers().get(backupID, -1)
    supplierActiveArray = suppliers_set().GetActiveArray()
    bySupplier = {}
    for supplierNum in range(len(supplierActiveArray)):
        bySupplier[supplierNum] = set()
    if not remote_files().has_key(backupID):
        for blockNum in range(localMaxBlockNum + 1):
            localData = GetLocalDataArray(backupID, blockNum)
            localParity = GetLocalParityArray(backupID, blockNum)  
            for supplierNum in range(len(supplierActiveArray)):
                if supplierActiveArray[supplierNum] != 1:
                    continue
                if localData[supplierNum] == 1:
                    bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, 'Data'))
                if localParity[supplierNum] == 1:
                    bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, 'Parity'))
    else:
        for blockNum in range(localMaxBlockNum + 1):
            remoteData = GetRemoteDataArray(backupID, blockNum)
            remoteParity = GetRemoteParityArray(backupID, blockNum)  
            localData = GetLocalDataArray(backupID, blockNum)
            localParity = GetLocalParityArray(backupID, blockNum)  
            for supplierNum in range(len(supplierActiveArray)):
                if supplierActiveArray[supplierNum] != 1:
                    continue
                if remoteData[supplierNum] != 1 and localData[supplierNum] == 1:    
                    bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, 'Data'))   
                if remoteParity[supplierNum] != 1 and localParity[supplierNum] == 1:   
                    bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, 'Parity'))
    return bySupplier


def ReadLocalFiles():
    global _LocalFilesNotifyCallback
    local_files().clear()
    local_max_block_numbers().clear()
    local_backup_size().clear()
    counter = 0
    # for filename in os.listdir(tmpfile.subdir('data-par')):
    for filename in os.listdir(settings.getLocalBackupsDir()):
        if filename.startswith('newblock-'):
            continue
        LocalFileReport(packetID=filename)
        counter += 1
    dhnio.Dprint(6, 'backups.ReadLocalFiles  %d files indexed' % counter)
    if dhnio.Debug(6):
        try:
            if sys.version_info >= (2, 6):
                #localSZ = sys.getsizeof(local_files())
                #remoteSZ = sys.getsizeof(remote_files())
                import lib.getsizeof
                localSZ = lib.getsizeof.total_size(local_files())
                remoteSZ = lib.getsizeof.total_size(remote_files())
                dhnio.Dprint(6, '    all local info uses %d bytes in the memory' % localSZ)
                dhnio.Dprint(6, '    all remote info uses %d bytes in the memory' % remoteSZ)
        except:
            dhnio.DprintException()
    if _LocalFilesNotifyCallback is not None:
        _LocalFilesNotifyCallback()


def SetSupplierList(supplierList):
    # going from 2 to 4 suppliers (or whatever) invalidates all backups
    # all suppliers was changed because its number was changed
    # so we lost everything!
    if len(supplierList) != suppliers_set().supplierCount:
        dhnio.Dprint(2, "backups.SetSupplierList got list of %d suppliers, but we have %d now!" % (len(supplierList), suppliers_set().supplierCount))
        # remove all local files and all backups
        DeleteAllBackups()
        # erase all remote info
        ClearRemoteInfo()
        # also erase local info
        ClearLocalInfo()
        # restart backup_monitor
        backup_monitor.Restart()
    # only single suppliers changed
    # need to erase info only for them 
    elif suppliers_set().SuppliersChanged(supplierList):
        # take a list of suppliers positions that was changed
        changedSupplierNums = suppliers_set().SuppliersChangedNumbers(supplierList)
        # notify io_throttle that we do not neeed already this suppliers
        for supplierNum in changedSupplierNums:
            dhnio.Dprint(2, "backups.SetSupplierList supplier %d changed: [%s]->[%s]" % (
                supplierNum, nameurl.GetName(suppliers_set().suppliers[supplierNum]), nameurl.GetName(supplierList[supplierNum])))
            io_throttle.DeleteSuppliers([suppliers_set().suppliers[supplierNum]])
        # remove remote info for this guys
        for backupID in remote_files().keys():
            for blockNum in remote_files()[backupID].keys():
                for supplierNum in changedSupplierNums:
                    remote_files()[backupID][blockNum]['D'][supplierNum] = 0
                    remote_files()[backupID][blockNum]['P'][supplierNum] = 0
        # restart backup_monitor
        backup_monitor.Restart()
    # finally save the list of current suppliers and clear all stats 
    suppliers_set().UpdateSuppliers(supplierList)


#def RemoteBlockReport(backupID, blockNum, remoteData, remoteParity, sentReports):
#    if not remote_files().has_key(backupID):
#        remote_files()[backupID] = {}
#        dhnio.Dprint(6, 'backups.RebuildReport new remote entry for %s created in the memory' % backupID)
#    if not remote_files()[backupID].has_key(blockNum):
#        remote_files()[backupID][blockNum] = {
#            'D': [0] * suppliers_set().supplierCount,
#            'P': [0] * suppliers_set().supplierCount,}
#    # save reconstructed block info into remote info structure, synchronize
#    for supplierNum in range(suppliers_set().supplierCount):
#        remote_files()[backupID][blockNum]['D'][supplierNum] = remoteData[supplierNum]
#        remote_files()[backupID][blockNum]['P'][supplierNum] = remoteParity[supplierNum]
#    # if we know only 5 blocks stored on remote machine
#    # but we got reconstructed 6th block - remember this  
#    remote_max_block_numbers()[backupID] = max(
#        remote_max_block_numbers().get(backupID, -1),
#        blockNum)
#    
##    # remember sent results
##    for packetID, result in sentReports.items():
##        try:
##            backupID, blockNum, supplierNum, dataORparity = packetID.split("-")
##            blockNum = int(blockNum)
##            supplierNum = int(supplierNum)
##            supplierIDURL = suppliers_set().suppliers[supplierNum]
##            if result is 'acked':
##                suppliers_set().successPackets[supplierNum] += 1
##            else:
##                suppliers_set().failedPackets[supplierNum] += 1
##                if suppliers_set().failedPackets[supplierNum] >= 6:
##                    supplierIdentity = contacts.getContact(supplierIDURL)
##                    supplierContact = '' if supplierIdentity is None else supplierIdentity.getContact() 
##                    if not supplierContact.startswith('cspace://'):
##                        # mark this guy as failed if 6 times we were failed to sent to him
##                        suppliers_set().failedSuppliers.add(supplierIDURL)
##                        dhnio.Dprint(4, 'backups.RebuildReport %s have %d fails - mark him as failed' % (
##                            nameurl.GetName(supplierIDURL), suppliers_set().failedPackets[supplierNum]))
##                    else:
##                        # but if he is 'cspace' user - give him more chances - 24 times
##                        if suppliers_set().failedPackets[supplierNum] >= 24:
##                            suppliers_set().failedSuppliers.add(supplierIDURL)
##                            dhnio.Dprint(4, 'backups.RebuildReport %s is "cspace user" and have %d fails - mark him as failed' % (
##                                nameurl.GetName(supplierIDURL), suppliers_set().failedPackets[supplierNum]))
##                        
##        except:
##            dhnio.DprintException()
#
#    # mark to repaint this backup in gui
#    RepaintBackup(backupID)

#------------------------------------------------------------------------------ 

def RemoteFileReport(backupID, blockNum, supplierNum, dataORparity, result):
    blockNum = int(blockNum)
    supplierNum = int(supplierNum)
    if not remote_files().has_key(backupID):
        remote_files()[backupID] = {}
        dhnio.Dprint(6, 'backups.BackupReport new remote entry for %s created in the memory' % backupID)
    if not remote_files()[backupID].has_key(blockNum):
        remote_files()[backupID][blockNum] = {
            'D': [0] * suppliers_set().supplierCount,
            'P': [0] * suppliers_set().supplierCount,}
    # save backed up block info into remote info structure, synchronize on hand info
    flag = 1 if result else 0
    if dataORparity == 'Data':
        remote_files()[backupID][blockNum]['D'][supplierNum] = flag 
    elif dataORparity == 'Parity':
        remote_files()[backupID][blockNum]['P'][supplierNum] = flag
    else:
        dhnio.Dprint(4, 'backups.BackupReport WARNING incorrect packet ID: %s' % packetID)
    # if we know only 5 blocks stored on remote machine
    # but we have backed up 6th block - remember this  
    remote_max_block_numbers()[backupID] = max(remote_max_block_numbers().get(backupID, -1), blockNum)
    # mark to repaint this backup in gui
    RepaintBackup(backupID)


def LocalFileReport(packetID=None, backupID_=None, blockNum_=None, supplierNum_=None, dataORparity_=None):
    if packetID is not None: 
        try:
            backupID, blockNum, supplierNum, dataORparity = packetID.split('-')
            blockNum = int(blockNum)
            supplierNum = int(supplierNum)
        except:
            dhnio.Dprint(4, 'backups.LocalFileReport WARNING incorrect filename: ' + packetID)
            return
    else:
        backupID = backupID_
        blockNum = int(blockNum_)
        supplierNum = int(supplierNum_)
        dataORparity = dataORparity_
        # packetID = backupID + '-' + str(blockNum) + '-' + str(supplierNum) + '-' + dataORparity
        packetID = packetid.MakePacketID(backupID, blockNum, supplierNum, dataORparity)
    filename = packetID
    if dataORparity not in ['Data', 'Parity']:
        dhnio.Dprint(4, 'backups.LocalFileReport WARNING Data or Parity? ' + filename)
        return
    if supplierNum >= suppliers_set().supplierCount:
        dhnio.Dprint(4, 'backups.LocalFileReport WARNING supplier number %d > %d %s' % (supplierNum, suppliers_set().supplierCount, filename))
        return
    if not local_files().has_key(backupID):
        local_files()[backupID] = {}
        dhnio.Dprint(6, 'backups.LocalFileReport new local entry for %s created in the memory' % backupID)
    if not local_files()[backupID].has_key(blockNum):
        local_files()[backupID][blockNum] = {
            'D': [0] * suppliers_set().supplierCount,
            'P': [0] * suppliers_set().supplierCount}
    local_files()[backupID][blockNum][dataORparity[0]][supplierNum] = 1
    if not local_max_block_numbers().has_key(backupID):
        local_max_block_numbers()[backupID] = -1
    if local_max_block_numbers()[backupID] < blockNum:
        local_max_block_numbers()[backupID] = blockNum
    if not local_backup_size().has_key(backupID):
        local_backup_size()[backupID] = 0
    try:
        local_backup_size()[backupID] += os.path.getsize(os.path.join(settings.getLocalBackupsDir(), filename))
    except:
        dhnio.DprintException()
    RepaintBackup(backupID)

#------------------------------------------------------------------------------ 

def RepaintBackup(backupID): 
    global _UpdatedBackupIDs
    _UpdatedBackupIDs.add(backupID)


def RepaintingProcess(on_off):
    global _UpdatedBackupIDs
    global _BackupStatusNotifyCallback
    global _RepaintingTask
    global _RepaintingTaskDelay
    if on_off is False:
        _RepaintingTaskDelay = 2.0
        if _RepaintingTask is not None:
            if _RepaintingTask.active():
                _RepaintingTask.cancel()
                _RepaintingTask = None
                _UpdatedBackupIDs.clear()
                return
    for backupID in _UpdatedBackupIDs:
        if _BackupStatusNotifyCallback is not None:
            _BackupStatusNotifyCallback(backupID)
    _RepaintingTaskDelay = misc.LoopAttenuation(_RepaintingTaskDelay, len(_UpdatedBackupIDs) > 0, 2.0, 8.0)
    _UpdatedBackupIDs.clear()
#        _RepaintingTaskDelay = 0.5
#    else:
#        if _RepaintingTaskDelay < 2.0:
#            _RepaintingTaskDelay *= 2.0
    # attenuation
    _RepaintingTask = reactor.callLater(_RepaintingTaskDelay, RepaintingProcess, True)

#------------------------------------------------------------------------------ 
# if there is a backup in process, one supplier may list more files in the backup than another,
# leading the backups to try to fix a backup in process.  We need to tell the backups
# when a backup is in process and when it is finished
def AddBackupInProcess(BackupName):
    backups_in_process().append(BackupName)


def RemoveBackupInProcess(BackupName):
    if BackupName in backups_in_process():
        backups_in_process().remove(BackupName)


def IsBackupInProcess(BackupID):
    return BackupID in backups_in_process()

#------------------------------------------------------------------------------ 
          
def DeleteAllBackups():
    all = list(set(local_files().keys() + remote_files().keys() + backup_db.GetBackupIds()))
    dhnio.Dprint(4, 'backups.DeleteAllBackups ' + str(all))
    for backupId in all:
        DeleteBackup(backupId, saveDB=False)
    DeleteAllLocalBackups()
    backup_db.Save()


# if the user deletes a backup, make sure we remove any work we're doing on it
def DeleteBackup(backupID, removeLocalFilesToo=True, saveDB=True):
    dhnio.Dprint(4, 'backups.DeleteBackup ' + backupID)
    # abort backup if it just started and is running at the moment
    backup_db.AbortRunningBackup(backupID)
    # also remove from list of running backups  
    if backupID in backups_in_process():
        backups_in_process().remove(backupID)
    # if we have several copies of same ID in working queue - remove them all
    backup_rebuilder.RemoveBackupToWork(backupID)
    # if we requested for files for this backup - we do not need it anymore
    io_throttle.DeleteBackupRequests(backupID)
    # remove interests in transport_control
    transport_control.DeleteBackupInterest(backupID)
    # mark it as being deleted in the db
    backup_db.DeleteDirBackup(backupID)
    # if we need to remove only one ID - lets save the DB right now
    if saveDB:
        backup_db.Save()
    # finally remove local files for this backupID
    if removeLocalFilesToo:
        DeleteLocalBackupFiles(backupID)
    # remove all remote info for this backup from the memory 
    if remote_files().has_key(backupID):
        #del remote_files()[backupID]
        remote_files().pop(backupID)
        dhnio.Dprint(4, '  remote info for %s were removed from memory, other remote files:' % backupID)
        dhnio.Dprint(4, '  %s' % str(remote_files().keys()))
    if remote_max_block_numbers().has_key(backupID):
        del remote_max_block_numbers()[backupID]
    # also remove local info
    if local_files().has_key(backupID):
        #del local_files()[backupID]
        local_files().pop(backupID)
        dhnio.Dprint(4, '  local info for %s were removed from memory, other local files:' % backupID)
        dhnio.Dprint(4, '  %s' % str(local_files().keys()))
    if local_max_block_numbers().has_key(backupID):
        del local_max_block_numbers()[backupID]
    if local_backup_size().has_key(backupID):
        del local_backup_size()[backupID]


def DeleteAllSupplierData(supplierNum):
    io_throttle.DeleteSuppliers([suppliers_set().suppliers[supplierNum]])


def DeleteLocalBackupFiles(backupID):
    dhnio.Dprint(4, 'backups.DeleteLocalBackupFiles ' + backupID)
    num = 0
    sz = 0
    # localDir = tmpfile.subdir('data-par')
    localDir = settings.getLocalBackupsDir()
#    if backupID not in local_files():
#        local_files()[backupID] = {}
    for filename in os.listdir(localDir):
        if filename.startswith(backupID):
#            try:
#                backupID_, blockNum, supplierNum, dataORparity = filename.split("-")
#                blockNum = int(blockNum)
#                supplierNum = int(supplierNum)
#            except:
#                dhnio.DprintException()
#                continue
#            if backupID_ != backupID:
#                dhnio.Dprint(4, 'backups.DeleteLocalBackupFiles WARNING incorrect file name: %s' % filename)
#                continue
            filepath = os.path.join(localDir, filename)
            try:
                sz += os.path.getsize(filepath) 
                os.remove(filepath)
                num += 1
#                if dataORparity == 'Data':
#                    local_files()[backupID][blockNum]['D'][supplierNum] = 0
#                elif dataORparity == 'Parity':
#                    local_files()[backupID][blockNum]['P'][supplierNum] = 0
#                else:
#                    dhnio.Dprint(4, 'backups.DeleteLocalBackupFiles WARNING incorrect file name: %s' % filename)
            except:
                dhnio.Dprint(2, 'backups.DeleteLocalBackupFiles ERROR can not remove ' + filepath)
                dhnio.DprintException()
    local_files()[backupID] = []
    local_max_block_numbers()[backupID] = -1
    local_backup_size()[backupID] = 0
    return num, sz


def DeleteAllLocalBackups():
    dhnio.Dprint(4, 'backups.DeleteAllLocalBackups')
    num = 0
    sz = 0
    backupIDs = set()
    # localDir = tmpfile.subdir('data-par')
    localDir = settings.getLocalBackupsDir()
    for filename in os.listdir(localDir):
        filepath = os.path.join(localDir, filename)
        try:
            backupID, blockNum, supplierNum, dataORparity = filename.split("-")
            sz += os.path.getsize(filepath) 
            os.remove(filepath)
            num += 1
            backupIDs.add(backupID)
        except:
            dhnio.Dprint(1, 'backups.DeleteAllLocalBackups ERROR can not remove ' + filepath)
            dhnio.DprintException()
    dhnio.Dprint(4, 'backups.DeleteAllLocalBackups %d backups removed in %d files of total size %s' % (
        len(backupIDs), num, diskspace.MakeStringFromBytes(sz)))

#------------------------------------------------------------------------------ 

def ClearLocalInfo():
    local_files().clear()
    local_max_block_numbers().clear()
    local_backup_size().clear()

def ClearRemoteInfo():
    remote_files().clear()
    remote_max_block_numbers().clear()

#------------------------------------------------------------------------------ 

def GetListFilesBackupIds(supplierNum, supplierId, listFileText):
    rawlist = listFileText.splitlines()
    resultlist = []
    for line in rawlist:
        # comment lines in Files reports start with blank,
        # also don't consider the identity a backup,
        # nor the backup_info.xml or backup_db
        if len(line) > 0 and (line[0] == ' ' or line.strip() == supplierId or line.strip() in [ settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName() ] ):
            continue
        dashindex = line.find("-")
        backupId = ''
        if dashindex>0: 
            # if this is supplier 3 make sure we have a -3-
            if line.find("-" + str(supplierNum) + "-") > 0:   
                # if there is a dash use only part before the dash
                backupId=line[0:dashindex]                    
        if backupId != '' and backupId not in resultlist:
            resultlist.append(backupId)
    return misc.sorted_backup_ids(resultlist)


# if we get a line "F20090709034221PM-0-Data from 0-1000"
# or "F20090709034221PM-0-Data from 0-1000 missing 1,3,"
# this will return 1000
def GetLineMaxBlockNum(line):
    try:
        maxBlock = -1
        workLine = line
        if line.find(" missing ") > 0:
            workLine = line[0:line.find(" missing ")]
        lineMax = int(workLine[workLine.rfind("-")+1:])
        if lineMax > maxBlock:
            maxBlock = lineMax
        return maxBlock
    except:
        return -1


def GetLineMissingBlocks(line, lineMax, backupMax):
    missingArray = []
    missingindex = line.find(" missing ") 
    if missingindex != -1:
        missingArray = line[missingindex+9:].split(",")
    if '' in missingArray:
        missingArray.remove('')
    if backupMax > lineMax:
        intMissingArray = range(lineMax+1,backupMax+1)
        for i in intMissingArray:
            missingArray.append(str(i))
    return missingArray


def GetBackupStats(backupID):
    if not remote_files().has_key(backupID):
        return 0, 0, [(0, 0)] * suppliers_set().supplierCount
    percentPerSupplier = 100.0 / suppliers_set().supplierCount
    # ??? maxBlockNum = remote_max_block_numbers().get(backupID, -1)
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    fileNumbers = [0] * suppliers_set().supplierCount
    totalNumberOfFiles = 0
    for blockNum in remote_files()[backupID].keys():
        for supplierNum in range(len(fileNumbers)):
            if supplierNum < suppliers_set().supplierCount:
                if remote_files()[backupID][blockNum]['D'][supplierNum] == 1:
                    fileNumbers[supplierNum] += 1
                    totalNumberOfFiles += 1
                if remote_files()[backupID][blockNum]['P'][supplierNum] == 1:
                    fileNumbers[supplierNum] += 1
                    totalNumberOfFiles += 1
    statsArray = []
    for supplierNum in range(suppliers_set().supplierCount):
        if maxBlockNum > -1:
            # 0.5 because we count both Parity and Data.
            percent = percentPerSupplier * 0.5 * fileNumbers[supplierNum] / ( maxBlockNum + 1 )
        else:
            percent = 0.0
        statsArray.append(( percent, fileNumbers[supplierNum] ))
    del fileNumbers 
    return totalNumberOfFiles, maxBlockNum, statsArray


# return totalPercent, totalNumberOfFiles, totalSize, maxBlockNum, statsArray
def GetBackupLocalStats(backupID):
    # ??? maxBlockNum = local_max_block_numbers().get(backupID, -1)
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    if not local_files().has_key(backupID):
        return 0, 0, 0, maxBlockNum, [(0, 0)] * suppliers_set().supplierCount
    percentPerSupplier = 100.0 / suppliers_set().supplierCount
    totalNumberOfFiles = 0
    fileNumbers = [0] * suppliers_set().supplierCount
    for blockNum in range(maxBlockNum + 1):
        if blockNum not in local_files()[backupID].keys():
            continue
#    for blockNum in local_files()[backupID].keys():
        for supplierNum in range(len(fileNumbers)):
            if supplierNum < suppliers_set().supplierCount:
                if local_files()[backupID][blockNum]['D'][supplierNum] == 1:
                    fileNumbers[supplierNum] += 1
                    totalNumberOfFiles += 1
                if local_files()[backupID][blockNum]['P'][supplierNum] == 1:
                    fileNumbers[supplierNum] += 1
                    totalNumberOfFiles += 1
    statsArray = []
    for supplierNum in range(suppliers_set().supplierCount):
        if maxBlockNum > -1:
            # 0.5 because we count both Parity and Data.
            percent = percentPerSupplier * 0.5 * fileNumbers[supplierNum] / ( maxBlockNum + 1 )
        else:
            percent = 0.0
        statsArray.append(( percent, fileNumbers[supplierNum] ))
    del fileNumbers 
    totalPercent = 100.0 * 0.5 * totalNumberOfFiles / ((maxBlockNum + 1) * suppliers_set().supplierCount)
    return totalPercent, totalNumberOfFiles, local_backup_size().get(backupID, 0), maxBlockNum, statsArray


def GetBackupBlocksAndPercent(backupID):
    if not remote_files().has_key(backupID):
        return 0, 0
    # get max block number
    # ??? maxBlockNum = remote_max_block_numbers().get(backupID, -1)
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    if maxBlockNum == -1:
        return 0, 0
    # we count all remote files for this backup
    fileCounter = 0
    for blockNum in remote_files()[backupID].keys():
        for supplierNum in range(suppliers_set().supplierCount):
            if remote_files()[backupID][blockNum]['D'][supplierNum] == 1:
                fileCounter += 1
            if remote_files()[backupID][blockNum]['P'][supplierNum] == 1:
                fileCounter += 1
    # +1 since zero based and *0.5 because Data and Parity
    return maxBlockNum + 1, 100.0 * 0.5 * fileCounter / ((maxBlockNum + 1) * suppliers_set().supplierCount)

# return : blocks, percent, weak block, weak percent
def GetBackupRemoteStats(backupID, only_available_files=True):
    if not remote_files().has_key(backupID):
        return 0, 0, 0, 0
    # get max block number
    # ??? maxBlockNum = remote_max_block_numbers().get(backupID, -1)
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    if maxBlockNum == -1:
        return 0, 0, 0, 0
    supplierCount = suppliers_set().supplierCount
    fileCounter = 0
    weakBlockNum = -1
    lessSuppliers = supplierCount
    activeArray = suppliers_set().GetActiveArray()
    # we count all remote files for this backup - scan all blocks
    for blockNum in range(maxBlockNum + 1):
        if blockNum not in remote_files()[backupID].keys():
            lessSuppliers = 0
            weakBlockNum = blockNum
            continue
        goodSuppliers = supplierCount
        for supplierNum in range(supplierCount):
            if activeArray[supplierNum] != 1 and only_available_files:
                goodSuppliers -= 1
                continue
            if remote_files()[backupID][blockNum]['D'][supplierNum] != 1 or remote_files()[backupID][blockNum]['P'][supplierNum] != 1:
                goodSuppliers -= 1
            if remote_files()[backupID][blockNum]['D'][supplierNum] == 1:
                fileCounter += 1
            if remote_files()[backupID][blockNum]['P'][supplierNum] == 1:
                fileCounter += 1
        if goodSuppliers < lessSuppliers:
            lessSuppliers = goodSuppliers
            weakBlockNum = blockNum
    # +1 since zero based and *0.5 because Data and Parity
    return (maxBlockNum + 1, 100.0 * 0.5 * fileCounter / ((maxBlockNum + 1) * supplierCount),
            weakBlockNum, 100.0 * float(lessSuppliers) / float(supplierCount))


def GetBackupRemoteArray(backupID):
    if not remote_files().has_key(backupID):
        return None
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    if maxBlockNum == -1:
        return None
    return remote_files()[backupID]


def GetBackupLocalArray(backupID):
    if not local_files().has_key(backupID):
        return None
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    if maxBlockNum == -1:
        return None
    return local_files()[backupID]
        
    
def GetBackupIds():
    return misc.sorted_backup_ids(remote_files().keys())


def GetKnownMaxBlockNum(backupID):
    return max(remote_max_block_numbers().get(backupID, -1), 
               local_max_block_numbers().get(backupID, -1))


def GetLocalDataArray(backupID, blockNum):
    if not local_files().has_key(backupID):
        return [0] * suppliers_set().supplierCount
    if not local_files()[backupID].has_key(blockNum):
        return [0] * suppliers_set().supplierCount
    return local_files()[backupID][blockNum]['D']


def GetLocalParityArray(backupID, blockNum):
    if not local_files().has_key(backupID):
        return [0] * suppliers_set().supplierCount
    if not local_files()[backupID].has_key(blockNum):
        return [0] * suppliers_set().supplierCount
    return local_files()[backupID][blockNum]['P']
    

def GetRemoteDataArray(backupID, blockNum):
    if not remote_files().has_key(backupID):
        return [0] * suppliers_set().supplierCount
    if not remote_files()[backupID].has_key(blockNum):
        return [0] * suppliers_set().supplierCount
    return remote_files()[backupID][blockNum]['D']

    
def GetRemoteParityArray(backupID, blockNum):
    if not remote_files().has_key(backupID):
        return [0] * suppliers_set().supplierCount
    if not remote_files()[backupID].has_key(blockNum):
        return [0] * suppliers_set().supplierCount
    return remote_files()[backupID][blockNum]['P']


def GetSupplierStats(supplierNum):
    result = {}
    files = total = 0
    for backupID in remote_files().keys():
        result[backupID] = [0, 0]
        for blockNum in remote_files()[backupID].keys():
            if remote_files()[backupID][blockNum]['D'][supplierNum] == 1:
                result[backupID][0] += 1
                files += 1 
            if remote_files()[backupID][blockNum]['P'][supplierNum] == 1:
                result[backupID][0] += 1
                files += 1
            result[backupID][1] += 2
            total += 2
    return files, total, result

#def GetSupplierLocalFiles(supplierNum):
#    result = []
#    files = total = 0
#    for backupID in local_files().keys():
#        for blockNum

#def GetFailedSuppliers():
#    return list(suppliers_set().failedSuppliers)

def GetWeakLocalBlock(backupID):
    # scan all local blocks for this backup 
    # and find the worst block 
    supplierCount = suppliers_set().supplierCount
    if not local_files().has_key(backupID):
        return -1, 0, supplierCount
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    weakBlockNum = -1
    lessSuppliers = supplierCount
    for blockNum in range(maxBlockNum+1):
        if blockNum not in local_files()[backupID].keys():
            return blockNum, 0, supplierCount
        goodSuppliers = supplierCount
        for supplierNum in range(supplierCount):
            if  local_files()[backupID][blockNum]['D'][supplierNum] != 1 or local_files()[backupID][blockNum]['P'][supplierNum] != 1:
                goodSuppliers -= 1
        if goodSuppliers < lessSuppliers:
            lessSuppliers = goodSuppliers
            weakBlockNum = blockNum
    return weakBlockNum, lessSuppliers, supplierCount
 
    
def GetWeakRemoteBlock(backupID):
    # scan all remote blocks for this backup 
    # and find the worst block - less suppliers keeps the data and stay online 
    supplierCount = suppliers_set().supplierCount
    if not remote_files().has_key(backupID):
        return -1, 0, supplierCount
    maxBlockNum = GetKnownMaxBlockNum(backupID)
    weakBlockNum = -1
    lessSuppliers = supplierCount
    activeArray = suppliers_set().GetActiveArray()
    for blockNum in range(maxBlockNum+1):
        if blockNum not in remote_files()[backupID].keys():
            return blockNum, 0, supplierCount
        goodSuppliers = supplierCount
        for supplierNum in range(supplierCount):
            if activeArray[supplierNum] != 1:
                goodSuppliers -= 1
                continue
            if  remote_files()[backupID][blockNum]['D'][supplierNum] != 1 or remote_files()[backupID][blockNum]['P'][supplierNum] != 1:
                goodSuppliers -= 1
        if goodSuppliers < lessSuppliers:
            lessSuppliers = goodSuppliers
            weakBlockNum = blockNum
    return weakBlockNum, lessSuppliers, supplierCount

#------------------------------------------------------------------------------ 

def SetBackupStatusNotifyCallback(callBack):
    global _BackupStatusNotifyCallback
    _BackupStatusNotifyCallback = callBack

def SetLocalFilesNotifyCallback(callback):
    global _LocalFilesNotifyCallback
    _LocalFilesNotifyCallback = callback

#def SetStatusCallBackForGuiBackup(callBack):
#    global _StatusCallBackForGuiBackup
#    StatusCallBackForGuiBackup = callBack

#------------------------------------------------------------------------------ 


if __name__ == "__main__":
    init()
    import pprint
    pprint.pprint(GetBackupIds())











