# validated: 2017-10-08 DS 8a37b81f4e10 cpp/Storage.cpp cpp/Storage.h cpp/Storage_load.cpp cpp/Storage_save.cpp
'''----------------------------------------------------------------------------'''
''' Copyright (c) FIRST 2017. All Rights Reserved.                             '''
''' Open Source Software - may be modified and shared by FRC teams. The code   '''
''' must be accompanied by the FIRST BSD license file in the root directory of '''
''' the project.                                                               '''
'''----------------------------------------------------------------------------'''

import contextlib
import os
import threading

from .message import Message
from .network_connection import NetworkConnection
from .storage_load import load_entries
from .storage_save import save_entries
from .structs import EntryInfo, ConnectionInfo
from .value import Value

from .support.compat import monotonic, file_replace
from .support.lists import ensure_id_exists

from .constants import (
    kEntryAssign,
    kEntryUpdate,
    kFlagsUpdate,
    kEntryDelete,
    kClearEntries,
    kExecuteRpc,
    kRpcResponse,
    
    NT_UNASSIGNED,
    NT_PERSISTENT,
    
    NT_RPC,
    
    NT_NOTIFY_IMMEDIATE,
    NT_NOTIFY_LOCAL,
    NT_NOTIFY_NEW,
    NT_NOTIFY_DELETE,
    NT_NOTIFY_UPDATE,
    NT_NOTIFY_FLAGS
)

import logging
logger = logging.getLogger('nt')



class _Entry(object):
    __slots__ = ['name', '_value', 'flags',
                 'id', 'local_id', 'seq_num', 'local_write',
                 'rpc_uid', 'rpc_call_uid', 'user_entry']
    
    def __init__(self, name, local_id, user_entry):
        
        # We redundantly store the name so that it's available when accessing the
        # raw Entry* via the ID map.
        self.name = name
        
        # The current value and flags.
        self._value = None
        self.flags = 0
        
        # Unique ID for self entry as used in network messages.  The value is
        # assigned by the server, on the client this is 0xffff until an
        # entry assignment is received back from the server.
        self.id = 0xffff
        
        # Local ID
        self.local_id = local_id
        
        # Sequence number for update resolution.
        self.seq_num = 0
        
        # If value has been written locally.  Used during initial handshake
        # on client to determine whether or not to accept remote changes.
        self.local_write = False
        
        # RPC handle
        self.rpc_uid = None
        
        # Last UID used when calling self RPC (primarily for client use).  This
        # is incremented for each call.
        self.rpc_call_uid = 0
        
        # python-specific: User-visible entry for optimized value retrieval
        self.user_entry = user_entry
        
    @property
    def value(self):
        return self._value
    
    @value.setter
    def value(self, value):
        self._value = value
        self.user_entry._value = value
        
    def isPersistent(self):
        return (self.flags & NT_PERSISTENT) != 0 
    
    def increment_seqnum(self):
        self.seq_num += 1
        self.seq_num &= 0xffff
        
    def isSeqNewerThan(self, other):
        '''
            self > other
        '''
        seq_num = self.seq_num
        if seq_num < other:
            return (other - seq_num) > 32768
        elif seq_num > other:
            return (seq_num - other) < 32768
        else:
            return False
        
    def isSeqNewerOrEqual(self, other):
        '''
            self >= other
        '''
        seq_num = self.seq_num
        if seq_num < other:
            return (other - seq_num) > 32768
        elif seq_num > other:
            return (seq_num - other) < 32768
        else:
            return True
        
    def isRpc(self):
        return self.value.type == NT_RPC
        
    def __repr__(self):
        return "<_Entry name='%s' value=%s flags=%s id=%s local_id=%s seq_num=%s local_write=%s rpc_uid=%s rpc_call_uid=%s" % \
            (self.name, self.value, self.flags, self.id, self.local_id,
             self.seq_num, self.local_write, self.rpc_uid, self.rpc_call_uid)


class Storage(object):
    
    def __init__(self, entry_notifier, rpc_server, user_entry_creator):
        self.m_notifier = entry_notifier
        self.m_rpc_server = rpc_server
        
        # python-specific
        self.m_user_entry_creator = user_entry_creator
        
        self.m_mutex = threading.Lock()
        self.m_entries = {}
        self.m_idmap = []
        self.m_localmap = []
        self.m_rpc_results = {}
        self.m_rpc_blocking_calls = set()
        
        # If any persistent values have changed
        self.m_persistent_dirty = False
        
        # condition variable and termination flag for blocking on a RPC result
        self.m_terminating = False
        self.m_rpc_results_cond = threading.Condition(self.m_mutex)
        
        # configured by dispatcher at startup
        self.m_dispatcher = None
        self.m_server = True
        
        # Differs from ntcore because python doesn't have switch statements...
        self._process_fns = {
            kEntryAssign:   self._processIncomingEntryAssign,
            kEntryUpdate:   self._processIncomingEntryUpdate,
            kFlagsUpdate:   self._processIncomingFlagsUpdate,
            kEntryDelete:   self._processIncomingEntryDelete,
            kClearEntries:  self._processIncomingClearEntries,
            kExecuteRpc:    self._processIncomingExecuteRpc,
            kRpcResponse:   self._processIncomingRpcResponse
        }
    
    def stop(self):
        self.m_terminating = True
        with self.m_mutex:
            self.m_rpc_results_cond.notify_all()
    
    def setDispatcher(self, dispatcher, server):
        with self.m_mutex:
            self.m_dispatcher = dispatcher
            self.m_server = server
    
    def clearDispatcher(self):
        self.m_dispatcher = None
    
    def getMessageEntryType(self, msg_id):
        with self.m_mutex:
            if msg_id >= len(self.m_idmap):
                return NT_UNASSIGNED
        
            entry = self.m_idmap[msg_id]
            if not entry or not entry.value:
                return NT_UNASSIGNED
        
            return entry.value.type
    
    @contextlib.contextmanager
    def _lockAndGetSendQueue(self):
        '''
            Python specific function
            
            This exists because our lock object isn't like the C++ lock
        '''
        with self.m_mutex:
            dispatcher = self.m_dispatcher
            if not dispatcher:
                yield None
                return
            
            queue_outgoing = dispatcher._queueOutgoing
            outgoing = []
            yield outgoing
        
        # This has to happen outside the lock
        for o in outgoing:
            try:
                queue_outgoing(*o)
            except TypeError:
                raise
    
    def processIncoming(self, msg, conn):
        # Note: c++ version takes a third param (weak_conn), but it's
        #       not needed here as conn == weak_conn
        fn = self._process_fns.get(msg.type)
        if fn:
            with self._lockAndGetSendQueue() as outgoing:
                fn(msg, conn, outgoing)
    
    def _processIncomingEntryAssign(self, msg, conn, outgoing):

        msg_id = msg.id
        name = msg.str
        entry = None
        may_need_update = False
        
        if self.m_server:
            # if we're a server, id=0xffff requests are requests for an id
            # to be assigned, we need to send the assignment back to
            # the sender as well as all other connections.
            if msg_id == 0xffff:
                entry = self._getOrNew(name)
                # see if it was already assigned; ignore if so.
                if entry.id != 0xffff:
                    return
                
                entry.flags = msg.flags
                entry.seq_num = msg.seq_num_uid
                self._setEntryValueImpl(entry, msg.value, outgoing, False)
                return
            
            if msg_id >= len(self.m_idmap) or not self.m_idmap[msg_id]:
                # ignore arbitrary entry assignments
                # this can happen due to e.g. assignment to deleted entry
                logger.debug("server: received assignment to unknown entry")
                return

            entry = self.m_idmap[msg_id]
        else:
            # clients simply accept assignments
            if msg_id == 0xffff:
                logger.debug("client: received entry assignment request?")
                return

            ensure_id_exists(self.m_idmap, msg_id)
            
            entry = self.m_idmap[msg_id]
            if entry is None:
                # create local
                entry = self._getOrNew(name)
                entry.id = msg_id
                self.m_idmap[msg_id] = entry
                if entry.value is None:
                    # didn't exist at all (rather than just being a response to a
                    # id assignment request)
                    entry.value = msg.value
                    entry.flags = msg.flags
                    entry.seq_num = msg.seq_num_uid
                    
                    # notify
                    self.m_notifier.notifyEntry(entry.local_id, name, entry.value, NT_NOTIFY_NEW)
                    return

                may_need_update = True  # we may need to send an update message
                
                # if the received flags don't match what we sent, most likely
                # updated flags locally in the interim; send flags update message.
                if msg.flags != entry.flags:
                    outmsg = Message.flagsUpdate(msg_id, entry.flags)
                    outgoing.append((outmsg, None, None))
        
        # common client and server handling

        # already exists; ignore if sequence number not higher than local
        seq_num = msg.seq_num_uid
        if entry.isSeqNewerThan(seq_num):
            if may_need_update:
                outmsg = Message.entryUpdate(entry.id, entry.seq_num,
                                             entry.value)
                outgoing.append((outmsg, None, None))
            
            return

        # sanity check: name should match id
        if msg.str != entry.name:
            logger.debug("entry assignment for same id with different name?")
            return

        notify_flags = NT_NOTIFY_UPDATE

        # don't update flags from a <3.0 remote (not part of message)
        # don't update flags if self is a server response to a client id request
        if not may_need_update and conn.get_proto_rev() >= 0x0300:
            # update persistent dirty flag if persistent flag changed
            if (entry.flags & NT_PERSISTENT) != (msg.flags & NT_PERSISTENT):
                self.m_persistent_dirty = True

            if entry.flags != msg.flags:
                notify_flags |= NT_NOTIFY_FLAGS

            entry.flags = msg.flags

        # update persistent dirty flag if the value changed and it's persistent
        if entry.isPersistent() and entry.value != msg.value:
            self.m_persistent_dirty = True

        # update local
        entry.value = msg.value
        entry.seq_num = seq_num

        # notify
        self.m_notifier.notifyEntry(entry.local_id, name, entry.value, notify_flags)

        # broadcast to all other connections (note for client there won't
        # be any other connections, don't bother)
        if self.m_server and outgoing is not None:
            outmsg = Message.entryAssign(entry.name, msg_id, msg.seq_num_uid,
                                         msg.value, entry.flags)
            outgoing.append((outmsg, None, conn))
    
    def _processIncomingEntryUpdate(self, msg, conn, outgoing):
        msg_id = msg.id
        if msg_id >= len(self.m_idmap) or not self.m_idmap[msg_id]:
            # ignore arbitrary entry updates;
            # this can happen due to deleted entries
            logger.debug("received update to unknown entry")
            return

        entry = self.m_idmap[msg_id]

        # ignore if sequence number not higher than local
        seq_num = msg.seq_num_uid
        if entry.isSeqNewerOrEqual(seq_num):
            return
        
        # update local
        entry.value = msg.value
        entry.seq_num = seq_num

        # update persistent dirty flag if it's a persistent value
        if entry.isPersistent():
            self.m_persistent_dirty = True
        
        # notify
        self.m_notifier.notifyEntry(entry.local_id, entry.name, entry.value,
                                    NT_NOTIFY_UPDATE)

        # broadcast to all other connections (note for client there won't
        # be any other connections, don't bother)
        if self.m_server and outgoing is not None:
            outgoing.append((msg, None, conn))
    
    def _processIncomingFlagsUpdate(self, msg, conn, outgoing):
        msg_id = msg.id
        if msg_id >= len(self.m_idmap) or not self.m_idmap[msg_id]:
            # ignore arbitrary entry updates
            # this can happen due to deleted entries
            logger.debug("received flags update to unknown entry")
            return

        self._setEntryFlagsImpl(self.m_idmap[msg_id], msg.flags, outgoing, False)
        # broadcast to all other connections (note for client there won't
        # be any other connections, don't bother)
        if self.m_server and outgoing is not None:
            outgoing.append((msg, None, conn))
    
    def _processIncomingEntryDelete(self, msg, conn, outgoing):
        msg_id = msg.id
        if msg_id >= len(self.m_idmap) or not self.m_idmap[msg_id]:
            # ignore arbitrary entry updates
            # this can happen due to deleted entries
            logger.debug("received delete to unknown entry")
            return
        
        self._deleteEntryImpl(self.m_idmap[msg_id], outgoing, False)
        
        # broadcast to all other connections (note for client there won't
        # be any other connections, don't bother)
        if self.m_server and outgoing is not None:
            outgoing.append((msg, None, conn))
    
    def _processIncomingClearEntries(self, msg, conn, outgoing):
        # update local
        self._deleteAllEntriesImpl(False)

        # broadcast to all other connections (note for client there won't
        # be any other connections, don't bother)
        if self.m_server and outgoing is not None:
            outgoing.append((msg, None, conn))
    
    def _processIncomingExecuteRpc(self, msg, conn, outgoing):
        if not self.m_server:
            return    # only process on server

        msg_id = msg.id
        if msg_id >= len(self.m_idmap) or not self.m_idmap[msg_id]:
            # ignore call to non-existent RPC
            # this can happen due to deleted entries
            logger.debug("received RPC call to unknown entry")
            return

        entry = self.m_idmap[msg_id]
        if not entry.value or not entry.isRpc():
            logger.debug("received RPC call to non-RPC entry")
            return

        call_uid = msg.seq_num_uid
        # XXX: TODO
        assert False
        self.m_rpc_server.processRpc(entry.local_id, entry.rpc_call_uid,
                                     entry.name, msg, entry.rpc_callback,
                                     conn.uid(), conn.queueOutgoing, conn.info())
    
    def _processIncomingRpcResponse(self, msg, conn, outgoing):
        if self.m_server:
            return    # only process on client
        
        msg_id = msg.id
        if msg_id >= len(self.m_idmap) or not self.m_idmap[msg_id]:
            # ignore call to non-existent RPC
            # this can happen due to deleted entries
            logger.debug("received RPC response to unknown entry")
            return
        
        entry = self.m_idmap[msg_id]
        if not entry.value or not entry.isRpc():
            logger.debug("received RPC call to non-RPC entry")
            return
        
        self.m_rpc_results[(entry.local_id, msg.seq_num_uid)] = msg.str
        self.m_rpc_results_cond.notify_all()
    
    def getInitialAssignments(self, conn, msgs):
        with self.m_mutex:
            conn.set_state(NetworkConnection.State.kSynchronized)
            for entry in self.m_entries.values():
                if entry.value is None:
                    continue
                msgs.append(Message.entryAssign(entry.name, entry.id,
                                                entry.seq_num,
                                                entry.value, entry.flags))
    
    def applyInitialAssignments(self, conn, msgs, new_server, out_msgs):
        with self._lockAndGetSendQueue() as update_msgs:
            if self.m_server:
                return    # should not do this on server
        
            conn.set_state(NetworkConnection.State.kSynchronized)
        
            # clear existing id's
            for entry in self.m_entries.values():
                entry.id = 0xffff
        
            # clear existing idmap
            del self.m_idmap[:]
        
            # apply assignments
            for msg in msgs:
                if msg.type != kEntryAssign:
                    logger.debug("client: received non-entry assignment request?")
                    continue
        
                msg_id = msg.id
                if msg_id == 0xffff:
                    logger.debug("client: received entry assignment request?")
                    continue
        
                seq_num = msg.seq_num_uid
                name = msg.str
        
                entry = self._getOrNew(name)
                if entry.value is None:
                    entry.value = msg.value
                    entry.flags = msg.flags
                    entry.seq_num = msg.seq_num_uid
                
                    # notify
                    self.m_notifier.notifyEntry(entry.local_id, name, entry.value, NT_NOTIFY_NEW)
                else:
                    # if we have written the value locally and the value is not persistent,
                    # then we don't update the local value and instead send it back to the
                    # server as an update message
                    if entry.local_write and not entry.isPersistent():
                        update_msgs.append((Message.entryUpdate(entry.id, entry.seq_num, entry.value), None, None))
                    else:
                        entry.value = msg.value
                        entry.seq_num = seq_num
                        notify_flags = NT_NOTIFY_UPDATE
                        # don't update flags from a <3.0 remote (not part of message)
                        if conn.get_proto_rev() >= 0x0300:
                            if entry.flags != msg.flags:
                                notify_flags |= NT_NOTIFY_FLAGS
        
                            entry.flags = msg.flags
        
                        # notify
                        self.m_notifier.notifyEntry(entry.local_id, name, entry.value, notify_flags)
        
                # set id and save to idmap
                entry.id = msg_id
                
                ensure_id_exists(self.m_idmap, msg_id)
                self.m_idmap[msg_id] = entry
            
            # delete or generate assign messages for unassigned local entries
            def _shouldDelete(entry):
                #  was assigned by the server, don't delete
                if entry.id != 0xffff:
                    return False
                
                # if we have written the value locally, we send an assign message to the
                # server instead of deleting
                if entry.local_write:
                    out_msgs.append(Message.entryAssign(entry.name, entry.id,
                                                        entry.seq_num,
                                                        entry.value, entry.flags))
                    return False
                  
                # otherwise delete
                return True
            
            self._deleteAllEntriesImpl(False, _shouldDelete)
    
    def getEntryValue(self, name):
        with self.m_mutex:
            e = self.m_entries.get(name)
            if e:
                return e.value
    
    def setDefaultEntryValue(self, name, value):
        if not name:
            return False    # can't compare empty name
        
        if value is None:
            return False    # can't compare to a null value
    
        with self._lockAndGetSendQueue() as outgoing:
            entry = self._getOrNew(name)
            
            # We return early if value already exists; if types match return true
            if entry.value is not None:
                return entry.value.type == value.type
            
            self._setEntryValueImpl(entry, value, outgoing, True)
            return True
    
    def setDefaultEntryValueById(self, local_id, value):
        if value is None:
            return False    # can't compare to a null value
    
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return False
            
            # We return early if value already exists; if types match return true
            if entry.value:
                return entry.value.type == value.type
            
            self._setEntryValueImpl(entry, value, outgoing, True)
            return True
    
    def setEntryValue(self, name, value):
        if not name:
            return True
        if value is None:
            return True
        
        with self._lockAndGetSendQueue() as outgoing:
            entry = self._getOrNew(name)
            if entry.value is not None and entry.value.type != value.type:
                return False # error on type mismatch
            
            self._setEntryValueImpl(entry, value, outgoing, True)
            return True
    
    def setEntryValueById(self, local_id, value):
        if value is None:
            return True
        
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return True
            
            if entry.value and entry.value.type != value.type:
                return False # error on type mismatch
            
            self._setEntryValueImpl(entry, value, outgoing, True)
            return True
    
    def _setEntryValueImpl(self, entry, value, outgoing, local):
        
        if value is None:
            return True
        
        old_value = entry.value
        entry.value = value
    
        # if we're the server, assign an id if it doesn't have one
        if self.m_server and entry.id == 0xffff:
            entry.id = len(self.m_idmap)
            self.m_idmap.append(entry)
        
        # update persistent dirty flag if value changed and it's persistent
        if entry.isPersistent() and (old_value is None or old_value != value):
            self.m_persistent_dirty = True
        
        # notify
        nflag = NT_NOTIFY_LOCAL if local else 0
        if old_value is None:
            self.m_notifier.notifyEntry(entry.local_id, entry.name, value, NT_NOTIFY_NEW | nflag)

        elif old_value != value:
            self.m_notifier.notifyEntry(entry.local_id, entry.name, value, NT_NOTIFY_UPDATE | nflag)
        
        # remember local changes
        if local:
            entry.local_write = True
        
        # generate message
        if outgoing is None or (not local and not self.m_server):
            return
        
        if old_value is None or old_value.type != value.type:
            if local:
                entry.increment_seqnum()
            msg = Message.entryAssign(entry.name, entry.id, entry.seq_num,
                                      value, entry.flags)
            outgoing.append((msg, None, None))

        elif old_value != value:
            if local:
                entry.increment_seqnum()
            
            # don't send an update if we don't have an assigned id yet
            if entry.id != 0xffff:
                msg = Message.entryUpdate(entry.id, entry.seq_num, value)
                outgoing.append((msg, None, None))

    def setEntryTypeValue(self, name, value):
        if not name:
            return
        if value is None:
            return
    
        with self._lockAndGetSendQueue() as outgoing:
            entry = self._getOrNew(name)
            self._setEntryValueImpl(entry, value, outgoing, True)
        
    def setEntryTypeValueById(self, local_id, value):
        if value is None:
            return
    
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return
            self._setEntryValueImpl(entry, value, outgoing, True)
    
    def setEntryFlags(self, name, flags):
        if not name:
            return
        
        with self._lockAndGetSendQueue() as outgoing:
            entry = self.m_entries.get(name)
            if entry is None:
                return
            self._setEntryFlagsImpl(entry, flags, outgoing, True)
    
    def setEntryFlagsById(self, local_id, flags):
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return
            self._setEntryFlagsImpl(entry, flags, outgoing, True)
    
    def _setEntryFlagsImpl(self, entry, flags, outgoing, local):
        if entry.value is None or entry.flags == flags:
            return

        # update persistent dirty flag if persistent flag changed
        if (entry.flags & NT_PERSISTENT) != (flags & NT_PERSISTENT):
            self.m_persistent_dirty = True
        
        entry.flags = flags
    
        # notify
        self.m_notifier.notifyEntry(entry.local_id, entry.name, entry.value,
                                    NT_NOTIFY_FLAGS | (NT_NOTIFY_LOCAL if local else 0))
    
        # generate message
        if not local or outgoing is None:
            return
        
        entry_id = entry.id
        
        # don't send an update if we don't have an assigned id yet
        if entry_id != 0xffff:
            outgoing.append((Message.flagsUpdate(entry_id, flags), None, None))
    
    def getEntryFlags(self, name):
        with self.m_mutex:
            entry = self.m_entries.get(name)
            return entry.flags if entry else 0
    
    def getEntryFlagsById(self, local_id):
        with self.m_mutex:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return 0
            else:
                return entry.flags
    
    def deleteEntry(self, name):
        if not name:
            return
        
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_entries[name]
            except KeyError:
                return
            else:
                self._deleteEntryImpl(entry, outgoing, True)
    
    def deleteEntryById(self, local_id):
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return
            else:
                self._deleteEntryImpl(entry, outgoing, True)
            
    def _deleteEntryImpl(self, entry, outgoing, local):
        
        entry_id = entry.id
        if entry_id < len(self.m_idmap):
            self.m_idmap[entry_id] = None
        
        # empty the value and reset id and local_write flag
        old_value = entry.value
        entry.value = None
        entry.id = 0xffff
        entry.local_write = False
            
        # Remove RPC if there was one
        if entry.rpc_uid is not None:
            self.m_rpc_server.removeRpc(entry.rpc_uid)
            entry.rpc_uid = None
        
        # update persistent dirty flag if it's a persistent value
        if entry.isPersistent():
            self.m_persistent_dirty = True
        
        # reset flags
        entry.flags = 0
        
        if old_value is None:
            return  # was not previously assigned
        
        # notify
        self.m_notifier.notifyEntry(entry.local_id, entry.name, old_value,
                                    NT_NOTIFY_DELETE | (NT_NOTIFY_LOCAL if local else 0))
    
        # if it had a value, generate message
        # don't send an update if we don't have an assigned id yet
        if outgoing is not None and local and entry_id != 0xffff:
            outgoing.append((Message.entryDelete(entry_id), None, None))
    
    def _defaultShouldDelete(self, entry):
        return not entry.isPersistent()
    
    def _deleteAllEntriesImpl(self, local, should_delete=None):
        if should_delete is None:
            should_delete = self._defaultShouldDelete
        
        notify_flags = NT_NOTIFY_DELETE | (NT_NOTIFY_LOCAL if local else 0)
        deleted = False
        
        for name, entry in self.m_entries.items():
            if entry.value is not None and should_delete(entry):
                # notify it's being deleted
                self.m_notifier.notifyEntry(entry.local_id, entry.name,
                                            entry.value, notify_flags)
    
                # remove it from idmap
                if entry.id < len(self.m_idmap):
                    self.m_idmap[entry.id] = None
                
                entry.id = 0xffff
                entry.local_write = False
                entry.value = None
                
                deleted = True
        
        return deleted
    
    def deleteAllEntries(self):
        with self._lockAndGetSendQueue() as outgoing:
            deleted = self._deleteAllEntriesImpl(True)
        
            # generate message
            if deleted and outgoing is not None:
                outgoing.append((Message.clearEntries(), None, None))
    
    def _getOrNew(self, name):
        entry = self.m_entries.get(name)
        if not entry:
            local_id = len(self.m_localmap)
            user_entry = self.m_user_entry_creator(name, local_id)
            entry = _Entry(name, local_id, user_entry)
            self.m_entries[name] = entry
            self.m_localmap.append(entry)
        return entry
    
    # ntcore: getEntry
    def getEntryId(self, name):
        if name:
            with self.m_mutex:
                entry = self._getOrNew(name)
                return entry.local_id
    
    # python-specific
    def getEntry(self, name):
        if name:
            with self.m_mutex:
                entry = self._getOrNew(name)
                return entry.user_entry
    
    # python-specific: returns user entries instead of ids
    def getEntries(self, prefix, types):
        with self.m_mutex:
            entries = []
            types = types if isinstance(types, int) else ord(types)
            for k, entry in self.m_entries.items():
                if entry.value is None or not k.startswith(prefix):
                    continue
                if types != 0 and ((types & ord(entry.value.type)) == 0):
                    continue
                entries.append(entry.user_entry)
        return entries
    
    def getEntryInfoById(self, local_id):
        with self.m_mutex:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return EntryInfo(None, NT_UNASSIGNED, 0)
            else:
                return EntryInfo(entry.name, entry.value.type, entry.flags)
    
    def getEntryNameById(self, local_id):
        with self.m_mutex:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return None
            else:
                return entry.name
    
    def getEntryTypeById(self, local_id):
        with self.m_mutex:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return NT_UNASSIGNED
            else:
                if entry.value is None:
                    return NT_UNASSIGNED
                return entry.value.type
    
    def getEntryInfo(self, prefix, types):
        with self.m_mutex:
            infos = []
            types = types if isinstance(types, int) else ord(types)
            for k, entry in self.m_entries.items():
                value = entry.value
                if value is None or not k.startswith(prefix):
                    continue
                
                if types != 0 and (types & ord(value.type)) == 0:
                    continue
        
                info = EntryInfo(entry.name, value.type, entry.flags)
                infos.append(info)
        
            return infos
    
    def addListener(self, prefix, callback, flags):
        with self.m_mutex:
            uid = self.m_notifier.add(callback, prefix, flags)
            # perform immediate notifications
            if (flags & NT_NOTIFY_IMMEDIATE) != 0 and (flags & NT_NOTIFY_NEW) != 0:
                for k, entry in self.m_entries.items():
                    if entry.value is None or not k.startswith(prefix):
                        continue
                    self.m_notifier.notifyEntry(entry.local_id, k, entry.value,
                                                NT_NOTIFY_IMMEDIATE | NT_NOTIFY_NEW, uid)
        return uid
    
    def addListenerById(self, local_id, callback, flags):
        with self.m_mutex:
            uid = self.m_notifier.addById(callback, local_id, flags)
            # perform immediate notifications
            if (flags & NT_NOTIFY_IMMEDIATE) != 0 and (flags & NT_NOTIFY_NEW) != 0:
                try:
                    entry = self.m_localmap[local_id]
                except IndexError:
                    pass
                else:
                    # if no value, don't notify
                    if entry.value is not None:
                        self.m_notifier.notifyEntry(local_id, entry.name, entry.value,
                                                    NT_NOTIFY_IMMEDIATE | NT_NOTIFY_NEW, uid)
        return uid
    
    def addPolledListener(self, poller_uid, prefix, flags):
        with self.m_mutex:
            uid = self.m_notifier.addPolled(poller_uid, prefix, flags)
            # perform immediate notifications
            if (flags & NT_NOTIFY_IMMEDIATE) != 0 and (flags & NT_NOTIFY_NEW) != 0:
                for k, entry in self.m_entries.items():
                    if entry.value is None or not k.startswith(prefix):
                        continue
                    self.m_notifier.notifyEntry(entry.local_id, k, entry.value,
                                                NT_NOTIFY_IMMEDIATE | NT_NOTIFY_NEW, uid)
        return uid
    
    def addPolledListenerById(self, poller_uid, local_id, flags):
        with self.m_mutex:
            uid = self.m_notifier.addPolledById(poller_uid, local_id, flags)
            # perform immediate notifications
            if (flags & NT_NOTIFY_IMMEDIATE) != 0 and (flags & NT_NOTIFY_NEW) != 0:
                try:
                    entry = self.m_localmap[local_id]
                except IndexError:
                    pass
                else:
                    # if no value, don't notify
                    if entry.value is not None:
                        self.m_notifier.notifyEntry(local_id, entry.name, entry.value,
                                                    NT_NOTIFY_IMMEDIATE | NT_NOTIFY_NEW, uid)
        return uid
    
    def _getPersistentEntries(self, periodic):
        # copy values out of storage as quickly as possible so lock isn't held
        with self.m_mutex:
            if periodic and not self.m_persistent_dirty:
                return False
            
            self.m_persistent_dirty = False
            
            entries = [(entry.name, entry.value) for entry in self.m_entries.values() \
                       if entry.value is not None and entry.isPersistent()]
        
        # sort in name order
        entries.sort()
        return entries
    
    # ntcore: called getEntries
    def getEntryValues(self, prefix):
        # copy values out of storage as quickly as possible so lock isn't held
        with self.m_mutex:
            entries = [(entry.name, entry.value) for entry in self.m_entries.values() \
                       if entry.value is not None and entry.name.startswith(prefix)]
        
        # sort in name order
        entries.sort()
        return entries
    
    def createRpc(self, local_id, defn, rpc_uid):
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return
            
            old_value = entry.value
            value = Value.makeRpc(defn)
            entry.value = value
        
            # set up the RPC info
            entry.rpc_uid = rpc_uid
            
            if old_value == value:
                return
            
            # assign an id if it doesn't have one
            if entry.id == 0xffff:
                entry.id = len(self.m_idmap)
                self.m_idmap.append(entry)
            
            # generate message
            if outgoing is not None:
                if old_value is None or old_value.type != value.type:
                    entry.increment_seqnum()
                    msg = Message.entryAssign(entry.name, entry.id, entry.seq_num,
                                              value, entry.flags)
                else:
                    entry.increment_seqnum()
                    msg = Message.entryUpdate(entry.id, entry.seq_num, value)
                
                outgoing.append((msg, None, None))
    
    def callRpc(self, local_id, params):
        with self._lockAndGetSendQueue() as outgoing:
            try:
                entry = self.m_localmap[local_id]
            except IndexError:
                return 0
        
            if entry.value is None or not entry.isRpc():
                return 0
        
            entry.rpc_call_uid += 1
            entry.rpc_call_uid &= 0xffff
            call_uid = entry.rpc_call_uid
            
            msg = Message.executeRpc(entry.id, call_uid, params)
            if not self.m_server:
                outgoing.append((msg, None, None))
                return call_uid
            
            # RPCs are unlikely to be used locally on the server, handle it
            # gracefully anyway.
            rpc_uid = entry.rpc_uid
            name = entry.name
        
        conn_info = ConnectionInfo('Server', 'localhost', 0, monotonic(), 0x0300)
        self.m_rpc_server.processRpc(local_id, call_uid, name, msg, conn_info,
                                     self._process_rpc, rpc_uid)
        return call_uid
    
    def _process_rpc(self, local_id, call_uid, result):
        with self.m_mutex:
            self.m_rpc_results[(local_id, call_uid)] = result
            self.m_rpc_results_cond.notify_all()
    
    def getRpcResult(self, local_id, call_uid, timeout=None):
        # returns timed_out, result
        
        with self.m_mutex:
            call_pair = (local_id, call_uid)
            
            # only allow one blocking call per rpc call uid
            if call_pair in self.m_rpc_blocking_calls:
                return False, None
        
            self.m_rpc_blocking_calls.add(call_pair)
            wait_until = monotonic() + timeout
        
            try:
                while True:
                    result = self.m_rpc_results.pop(call_pair, None) 
                    if result is None:
                        if timeout == 0 or self.m_terminating:
                            return False, None
            
                        if timeout < 0:
                            self.m_rpc_results_cond.wait()
                        else:
                            ttw = monotonic() - wait_until
                            if ttw <= 0:
                                return False, None
                            
                            self.m_rpc_results_cond.wait(ttw)
            
                        # if element does not exist, have been canceled
                        if call_pair not in self.m_rpc_blocking_calls:
                            return False, None
            
                        if self.m_terminating:
                            return False, None
            
                        continue
                    
                    return True, result
            finally:
                try:
                    self.m_rpc_blocking_calls.remove(call_pair)
                except KeyError:
                    pass
    
    def cancelBlockingRpcResult(self, call_uid):
        with self.m_mutex:
            # safe to erase even if id does not exist
            self.m_rpc_blocking_calls.erase(call_uid)
            self.m_rpc_results_cond.notify_all()
    
    #
    # Persistence stuff from Storage_load.cpp/Storage_save.cpp
    #
    
    # from Storage_load.cpp
    def loadPersistent(self, filename=None, fp=None):
        return self._loadFromFile(True, '', filename, fp)
        
    def loadEntries(self, filename=None, fp=None, prefix=''):
        return self._loadFromFile(False, prefix, filename, fp)
    
    def _loadFromFile(self, persistent, prefix, filename, fp):
        try:
            if fp:
                entries = load_entries(fp, filename if filename else '<string>', prefix)
            else:
                with open(filename, 'r') as fp:
                    entries = load_entries(fp, filename, prefix)
        except IOError as e:
            return 'Error reading file: %s' % e
        else:
            self._loadEntries(entries, persistent)
            return
    
    def _loadEntries(self, entries, persistent):
        # entries is a list of (str, Value) tuples
        
        # copy values into storage as quickly as possible so lock isn't held
        with self._lockAndGetSendQueue() as outgoing:
            for name, value in entries:
                entry = self._getOrNew(name)
                old_value = entry.value
                entry.value = value
                was_persist = entry.isPersistent()
                if not was_persist and persistent:
                    entry.flags |= NT_PERSISTENT
                
                # if we're the server, an id if it doesn't have one
                if self.m_server and entry.id == 0xffff:
                    entry.id = len(self.m_idmap)
                    self.m_idmap.append(entry)
                
                # notify (for local listeners)
                if self.m_notifier.m_local_notifiers:
                    if old_value is None:
                        self.m_notifier.notifyEntry(entry.local_id, name, value,
                                                    NT_NOTIFY_NEW | NT_NOTIFY_LOCAL)
                    elif old_value != value:
                        notify_flags = NT_NOTIFY_UPDATE | NT_NOTIFY_LOCAL
                        if not was_persist and persistent:
                            notify_flags |= NT_NOTIFY_FLAGS

                        self.m_notifier.notifyEntry(entry.local_id, name, value, notify_flags)
                    elif not was_persist and persistent:
                        self.m_notifier.notifyEntry(entry.local_id, name, value,
                                                    NT_NOTIFY_FLAGS | NT_NOTIFY_LOCAL)
    
                if outgoing is None:
                    continue    # shortcut
    
                entry.increment_seqnum()
    
                # put on update queue
                if old_value is None or old_value.type != value.type:
                    outgoing.append((Message.entryAssign(name, entry.id,
                                                        entry.seq_num,
                                                        value, entry.flags), None, None))
                elif entry.id != 0xffff:
                    # don't send an update if we don't have an assigned id yet
                    if old_value != value:
                        outgoing.append((Message.entryUpdate(entry.id, entry.seq_num, value), None, None))
                    if not was_persist and persistent:
                        outgoing.append((Message.flagsUpdate(entry.id, entry.flags), None, None))
    
    # from Storage_save.cpp
    def savePersistent(self, filename=None, periodic=False, fp=None):
        entries = self._getPersistentEntries(periodic)
        if entries == False:
            return
        
        err = self._saveEntries(entries, filename, fp)
        if err and periodic:
            self.m_persistent_dirty = True
        return err

    def saveEntries(self, prefix, filename=None, fp=None):
        entries = self.getEntryValues(prefix)
        return self._saveEntries(entries, filename, fp)
        
    def _saveEntries(self, entries, filename, fp):
        # Going to not use tempfile to keep compatibility with ntcore,
        # as having to cleanup temp files on the RIO is probably bad
        if fp:
            try:
                save_entries(fp, entries)
            except IOError as e:
                return 'Error writing file: %s' % e
        else:
            tmp = '%s.tmp' % filename
            bak = '%s.bak' % filename
            
            try:
                with open(tmp, 'w') as fp:
                    save_entries(fp, entries)
                    os.fsync(fp.fileno())
            except IOError as e:
                return 'Error writing file: %s' % e
            
            try:
                file_replace(filename, bak)
            except OSError:
                pass # ignored
                
            try:
                file_replace(tmp, filename)
            except OSError as e:
                try:
                    # try to restore backup
                    file_replace(bak, filename)
                except OSError:
                    pass
                
                return 'Could not rename temp file to real file: %s' % e


