from __future__ import with_statement

import os
import time
import hashlib
import bsddb as bdb
import datetime
import dateutil.tz
import shutil

from urllib import quote, unquote
from urlparse import urlsplit

try:
    import cPickle as pickle
except ImportError:
    import pickle

from dateutil import parser as dateparser

from cheshire3.exceptions import *
from cheshire3.configParser import C3Object
from cheshire3.baseObjects import Session
from cheshire3.utils import gen_uuid


class DeletedObject(object):
    id = ""
    time = ""
    store = ""

    def __nonzero__(self):
        return False

    def __init__(self, store, id, time="unknown"):
        self.store = store
        self.id = id
        self.time = time


class SummaryObject(object):
    """ Allow database to have summary, but no storage """

    totalItems = 0
    totalWordCount = 0
    minWordCount = 0
    maxWordCount = 0
    meanWordCount = 0
    totalByteCount = 0
    minByteCount = 0
    maxByteCount = 0
    meanByteCount = 0
    lastModified = ''

    _possiblePaths = {
        'metadataPath': {
            'docs': "Path to file where the summary metadata will be kept."
        }
    }

    def __init__(self, session, config, parent):
        # Don't want to inherit from database!
        mp = self.paths.get('metadataPath', '')
        if not mp:
            self.paths['metadataPath'] = ''
            return
        if not (urlsplit(mp).scheme or os.path.isabs(mp)):
            # Prepend defaultPath from parents
            dfp = self.get_path(session, 'defaultPath')
            if (not dfp):
                msg = ("Store has relative metadata path, and no visible "
                       "defaultPath.")
                raise ConfigFileException(msg)
            elif urlsplit(dfp).scheme:
                # Default path is a URL
                mp = '/'.join((dfp, mp))
            else:
                mp = os.path.join(dfp, mp)
        self.paths['metadataPath'] = mp
        if (not os.path.exists(mp)):
            # We don't exist, try and instantiate new database
            self._create(session, mp)
        else:
            cxn = bdb.db.DB()
            try:
                cxn.open(mp)
                # Now load values.
                self.totalItems = long(cxn.get("totalItems"))
                self.totalWordCount = long(cxn.get("totalWordCount"))
                self.minWordCount = long(cxn.get("minWordCount"))
                self.maxWordCount = long(cxn.get("maxWordCount"))

                self.totalByteCount = long(cxn.get("totalByteCount"))
                self.minByteCount = long(cxn.get("minByteCount"))
                self.maxByteCount = long(cxn.get("maxByteCount"))

                self.lastModified = str(cxn.get("lastModified"))

                if self.totalItems != 0:
                    self.meanWordCount = self.totalWordCount / self.totalItems
                    self.meanByteCount = self.totalByteCount / self.totalItems
                else:
                    self.meanWordCount = 1
                    self.meanByteCount = 1
                cxn.close()
            except:
                # Doesn't exist in usable form
                self._create(session, mp)

    def _initDb(self, session, dbt):
            dbp = dbt + "Path"
            databasePath = self.get_path(session, dbp, "")
            if (not databasePath):
                databasePath = ''.join([self.id, "_", dbt, ".bdb"])
            if (not os.path.isabs(databasePath)):
                # Prepend defaultPath from parents
                dfp = self.get_path(session, 'defaultPath')
                if not dfp:
                    msg = ("Store has relative path, and no visible "
                           "defaultPath.")
                    raise ConfigFileException(msg)
                databasePath = os.path.join(dfp, databasePath)
            self.paths[dbp] = databasePath

    def _create(self, session, dbPath):
        cxn = bdb.db.DB()
        cxn.set_flags(bdb.db.DB_RECNUM)
        try:
            cxn.open(dbPath, dbtype=bdb.db.DB_BTREE,
                     flags=bdb.db.DB_CREATE, mode=0660)
        except:
            raise ValueError("Could not create %s" % dbPath)
        cxn.put("lastModified", time.strftime('%Y-%m-%d %H:%M:%S'))
        cxn.close()

    def commit_metadata(self, session):
        cxn = bdb.db.DB()
        mp = self.get_path(session, 'metadataPath')
        if mp:
            try:
                self.meanWordCount = self.totalWordCount / self.totalItems
                self.meanByteCount = self.totalByteCount / self.totalItems
            except ZeroDivisionError:
                self.meanWordCount = 1
                self.meanByteCount = 1

            self.lastModified = time.strftime('%Y-%m-%d %H:%M:%S')
            try:
                cxn.open(mp)
                cxn.put("totalItems", str(self.totalItems))
                cxn.put("totalWordCount", str(self.totalWordCount))
                cxn.put("minWordCount", str(self.minWordCount))
                cxn.put("maxWordCount", str(self.maxWordCount))
                cxn.put("totalByteCount", str(self.totalByteCount))
                cxn.put("minByteCount", str(self.minByteCount))
                cxn.put("maxByteCount", str(self.maxByteCount))
                cxn.put("lastModified", self.lastModified)
                cxn.close()
            except:
                # TODO: Nicer failure?
                raise

    def accumulate_metadata(self, session, obj):
        self.totalItems += 1
        self.totalWordCount += obj.wordCount
        if obj.wordCount < self.minWordCount:
            self.minWordCount = obj.wordCount
        if obj.wordCount > self.maxWordCount:
            self.maxWordCount = obj.wordCount

        self.totalByteCount += obj.byteCount
        if obj.byteCount < self.minByteCount:
            self.minByteCount = obj.byteCount
        if obj.byteCount > self.maxByteCount:
            self.maxByteCount = obj.byteCount


class SimpleStore(C3Object, SummaryObject):
    """ Base Store implementation.  Provides non-storage-specific functions """

    # Instantiate some type of simple record store
    idNormalizer = None
    outIdNormalizer = None
    inWorkflow = None
    outWorkflow = None
    storageTypes = []
    reverseMetadataTypes = []
    currentId = -1
    useUUID = 0
    session = None
    switching = 0

    _possiblePaths = {
        'idNormalizer': {
            'docs': ("Identifier for Normalizer to use to turn the data "
                     "object's identifier into a suitable form for storing. "
                     "Eg: StringIntNormalizer")
        },
        'outIdNormalizer': {
            'docs': "Normalizer to reverse the process done by idNormalizer"
        },
        'inWorkflow': {
            'docs': "Workflow with which to process incoming data objects."
        },
        'outWorkflow': {
            'docs': ("Workflow with which to process stored data objects "
                     "when requested.")
        }
    }

    _possibleSettings = {
        'useUUID': {
            'docs': "Each stored data object should be assigned a UUID.",
            'type': int,
            'options': "0|1"
        },
        'digest': {
            'docs': "Type of digest/checksum to use. Defaults to no digest",
            'options': 'sha|md5'
        },
        'expires': {
            'docs': ("Time after ingestion at which to delete the data "
                     "object in number of seconds."),
            'type': int
        },
        'storeDeletions': {
            'docs': "Maintain when an object was deleted from this store.",
            'type': int,
            'options': "0|1"
        },
        'bucketType': {
            'docs': ("Type of 'bucket' to use when splitting an index over "
                     "multiple files."),
        },
        'maxBuckets': {
            'docs': "Maximum number of 'buckets' to split an index into",
            'type': int
        },
        'maxItemsPerBucket': {
            'docs': "Maximum number of items to put into each 'bucket'",
            'type': int
        },
    }

    _possibleDefaults = {
        'expires': {
            "docs": ('Default time after ingestion at which to delete the '
                     'data object in number of seconds. Can be overridden by '
                     'the individual object.'),
            'type': int
        }
    }

    def __init__(self, session, config, parent):
        # Don't inherit metadataPath!
        C3Object.__init__(self, session, config, parent)
        self.switching = (
            self.get_setting(session, 'bucketType', '') or
            self.get_setting(session, 'maxBuckets', 0) or
            self.get_setting(session, 'maxItemsPerBucket', 0)
        )

        SummaryObject.__init__(self, session, config, parent)
        self.idNormalizer = self.get_path(session, 'idNormalizer', None)
        self.outIdNormalizer = self.get_path(session, 'outIdNormalizer', None)
        self.inWorkflow = self.get_path(session, 'inWorkflow', None)
        self.outWorkflow = self.get_path(session, 'outWorkflow', None)

        self.session = session

        dbts = self.get_storageTypes(session)
        self.storageTypes = dbts
        revdbts = self.get_reverseMetadataTypes(session)
        self.reverseMetadataTypes = revdbts

        self.useUUID = self.get_setting(session, 'useUUID', 0)
        self.expires = self.get_default(session, 'expires', 0)
        for dbt in dbts:
            self._initDb(session, dbt)
            self._verifyDb(session, dbt)
            if dbt in revdbts:
                dbt = dbt + "Reverse"
                self._initDb(session, dbt)
                self._verifyDb(session, dbt)

    def _verifyDb(self, session, dbType):
        pass

    def generate_checkSum(self, session, data):
        digest = self.get_setting(session, "digest")
        if (digest):
            if (digest == 'md5'):
                m = hashlib.md5()
            elif (digest == 'sha'):
                m = hashlib.sha1()
            else:
                raise ConfigFileException("Unknown digest type: %s" % digest)

            if type(data) == unicode:
                data = data.encode('utf-8')
            m.update(data)
            digest = m.hexdigest()
            return digest
        else:
            return None

    def generate_expires(self, session, obj=None):
        now = time.time()
        if obj and hasattr(obj, 'expires'):
            return now + obj.expires
        elif self.expires > 0:
            return now + self.expires
        else:
            # Don't expire
            return 0

    def _openAll(self, session):
        for t in self.storageTypes:
            self._openDb(session, t)

    def _closeAll(self, session):
        for t in self.storageTypes:
            self._closeDb(session, t)
        for t in self.reverseMetadataTypes:
            self._closeDb(session, t + "Reverse")
        for (t, cxn) in self.cxns.items():
            if cxn is not None:
                try:
                    cxn.close()
                    self.cxns[t] = None
                except:
                    pass

    def generate_id(self, session):
        """Generate and return a new unique identifier."""
        return self.get_dbSize(session)

    def get_storageTypes(self, session):
        return ['database']

    def get_reverseMetadataTypes(self, session):
        return ['digest', 'expires']

    def get_dbSize(self, session):
        """Return number of items in storage."""
        raise NotImplementedError

    def begin_storing(self, session):
        """Prepare to store objects."""
        self._openAll(session)
        return None

    def commit_storing(self, session):
        """Finalize storing, e.g. commit to persistent storage."""
        self._closeAll(session)
        self.commit_metadata(session)
        return None

    def get_idChunkGenerator(self, session, taskManager=None):
        """Generator to yield chunks of ids from the store.

        Can be used during parallel processing to split processing into
        chunks to be distributed across processes or processing nodes.
        """
        # Defaults
        maxChunkSize = 1000
        maxChunkByteCount = 2048000
        maxChunkWordCount = maxChunkByteCount / 5
        chunkBy = None
        chunkThreshold = 1
        total = self.get_dbSize(session)
        if taskManager is not None:
            maxChunkSize = taskManager.get_setting(session,
                                                   'maxChunkSize',
                                                   maxChunkSize)
            chunkBy = taskManager.get_setting(session, 'chunkBy', chunkBy)
            if (chunkBy is not None) and chunkBy.isalpha():
                # Chunking based on metadata
                cmax = 'maxChunk{0}'.format(chunkBy[0].upper() + chunkBy[1:])
                chunkThreshold = taskManager.get_setting(
                    session,
                    cmax,
                    locals().get(cmax, chunkThreshold)
                )
            else:
                # Generic chunking
                if (chunkBy is not None) and chunkBy.isdigit():
                    # chunkBy is a number of preferred times to use each chunk
                    nIters = int(chunkBy)
                else:
                    nIters = 1
                nTasks = taskManager.nTasks
                maxChunkSize = min(maxChunkSize, (total / (nIters * nTasks)))

        numericId = 0
        iterator = self.__iter__()
        idNormalizer = self.idNormalizer
        outIdNormalizer = self.outIdNormalizer
        while numericId < total:
            chunk = []
            chunkTotal = 0
            while (chunkTotal < chunkThreshold and
                   len(chunk) < maxChunkSize and
                   numericId < total):
                # Get real id and de-normalize if possible
                id = iterator.nextData[0]
                obj = iterator.next()  # Move iterator on
                if outIdNormalizer:
                    id = outIdNormalizer.process_string(session, id)
                # chunkBy
                if chunkBy is not None:
                    b = self.fetch_metadata(session, id, chunkBy)
                    chunkTotal += b
                chunk.append(id)
                numericId += 1
            yield chunk

    def delete_data(self, session, id):
        """Delete data stored against id."""
        raise NotImplementedError

    def fetch_data(self, session, id):
        """Return data stored against id."""
        raise NotImplementedError

    def store_data(self, session, id, data, metadata):
        """Store data against id."""
        raise NotImplementedError

    def fetch_metadata(self, session, id, mType):
        """Return mType metadata stored against id."""
        raise NotImplementedError

    def store_metadata(self, session, id, mType, value):
        """Store value for mType metadata against id."""
        raise NotImplementedError

    def clean(self, session):
        """Delete expired data objects."""
        raise NotImplementedError

    def clear(self, session):
        """This would delete all the data out of self."""
        raise NotImplementedError

    def flush(self, session):
        """Ensure all data is flushed to disk."""
        raise NotImplementedError


class BdbIter(object):
    store = None
    cursor = None
    cxn = None
    nextData = None

    def __init__(self, session, store):
        self.store = store
        self.session = session
        self.cxn = store._openDb(self.session, 'database')
        self.cursor = self.cxn.cursor()
        self.nextData = self.cursor.first()

    def __iter__(self):
        return self

    def next(self):
        try:
            d = self.nextData
            if not d:
                raise StopIteration()
            self.nextData = self.cursor.next()
            return d
        except:
            raise StopIteration()

    def jump(self, position):
        # Jump to this position
        self.nextData = self.cursor.set_range(position)
        return self.nextData[0]


class SwitchingBdbConnection(object):

    def __init__(self, session, parent, path="",
                 maxBuckets=0, maxItemsPerBucket=0, bucketType=''):
        self.session = session
        self.store = parent

        # Allow per call overrides, eg if parent object has multiple types
        if maxBuckets == 0:
            self.maxBuckets = parent.get_setting(session, 'maxBuckets', 0)
        else:
            self.maxBuckets = maxBuckets

        if maxItemsPerBucket == 0:
            self.maxItemsPerBucket = parent.get_setting(session,
                                                        'maxItemsPerBucket',
                                                        0)
        else:
            self.maxItemsPerBucket = maxItemsPerBucket

        if bucketType == '':
            self.bucketType = parent.get_setting(session, 'bucketType', '')
        else:
            self.bucketType = bucketType

        if not self.bucketType:
            if self.maxItemsPerBucket:
                self.bucketType = 'int'
            elif self.maxBuckets:
                self.bucketType = 'hash'
            else:
                self.bucketType = 'term1'

        if os.path.isabs(path):
            self.basePath = path
        else:
            dfp = parent.get_path(session, 'defaultPath')
            basename = parent.id
            self.basePath = os.path.join(dfp, basename)

        self.cxns = {}
        self.createArgs = {}
        self.openArgs = {}
        self.preOpenFlags = 0
        self.bucketFns = {'term1': self.termBucket1,
                          'term-1': self.termBucket_1,
                          'term2': self.termBucket2,
                          'term-2': self.termBucket_2,
                          'hash': self.hashBucket,
                          'int': self.intBucket,
                          'null': self.nullBucket}
        self.listBucketFns = {'term1': self.listTermBucket1,
                              'term-1': self.listTermBucket1,
                              'term2': self.listTermBucket2,
                              'term-2': self.listTermBucket_2,
                              'hash': self.listHashBucket,
                              'int': self.listIntBucket,
                              'null': self.listNullBucket
                              }

    def bucket_exists(self, b):
        return os.path.exists(self.basePath + '_' + b)

    def bucket(self, key):
        # bucket type and settings from parent
        fn = self.bucketFns[self.bucketType]
        return fn(key)

    def listBuckets(self):
        fn = self.listBucketFns[self.bucketType]
        return fn()

    def termBucket1(self, key):
        if not key:
            return "other"
        elif key[0].isalnum():
            return key[0].lower()
        elif key[0] > 'z':
            return 'extended'
        else:
            return "other"

    def listTermBucket1(self):
        l = ['other', 'extended']
        l.extend([str(x) for x in range(10)])
        l.extend([chr(x) for x in range(97, 123)])
        return l

    def termBucket_1(self, key):
        if not key:
            return "other"
        elif key[-1].isalnum():
            return key[-1].lower()
        elif key[-1] > 'z':
            return 'extended'
        else:
            return "other"

    def termBucket2(self, key):
        if not key:
            return "other"
        elif key[0].isdigit():
            return key[0]
        elif key[0].isalpha():
            if len(key) == 1:
                return key + "0"
            elif not key[1].isalnum():
                return key[0].lower() + '_'
            else:
                return key[:2].lower()
        else:
            return "other"

    def termBucket_2(self, key):
        if not key:
            return "other"
        elif key[-1].isdigit():
            return key[-1]
        elif key[-1].isalpha():
            if len(key) == 1:
                return '0' + key
            elif not key[-2].isalnum():
                return '_' + key[-1].lower()
            else:
                return key[-2:].lower()
        else:
            return "other"

    def listTermBucket2(self):
        lets = [chr(x) for x in range(97, 123)]
        nums = [str(x) for x in range(10)]
        all = []
        all.extend(lets)
        all.extend(nums)
        all.append('_')
        l = ['other']
        l.extend(nums)
        for let in lets:
            l.extend([let + x for x in all])
        return l

    def listTermBucket_2(self):
        # reversed order...
        lets = [chr(x) for x in range(97, 123)]
        nums = [str(x) for x in range(10)]
        all = []
        all.extend(lets)
        all.extend(nums)
        all.append('_')
        l = ['other']
        l.extend(nums)
        for let in all:
            l.extend([let + x for x in lets])
        return l

    def hashBucket(self, key):
        # Essentially random division, constrained number of buckets
        return str(hash(key) % self.maxBuckets)

    def listHashBucket(self):
        return [str(x) for x in range(self.maxBuckets)]

    def intBucket(self, key):
        # constrained number of items per bucket
        # must be integer key!  (eg internal record id)
        if not type(key) in [int, long] and not key.isdigit():
            msg = "Cannot use 'int' bucket method if keys are not numbers"
            raise ConfigFileException(msg)
        else:
            return str(long(key) / self.maxItemsPerBucket)

    def listIntBucket(self):
        # find out how many
        x = self.store.get_dbSize(self.session)
        return [str(y) for y in range((x / self.maxItemsPerBucket) + 1)]

    def nullBucket(self, key):
        # all in one bucket
        return '0'

    def listNullBucket(self):
        return ['0']

    def _open(self, b):
        if b in self.cxns and self.cxns[b] is not None:
            return self.cxns[b]
        else:
            cxn = bdb.db.DB()
            if self.preOpenFlags:
                cxn.set_flags(self.preOpenFlags)
            dbp = self.basePath + "_" + b

            if (not os.path.exists(dbp)):
                cxn.open(dbp, **self.store.createArgs[self.basePath])
                cxn.close()
                cxn = bdb.db.DB()

            cxn.open(dbp, **self.openArgs)
            self.cxns[b] = cxn
            return cxn

    def _cursor(self, b):
        cxn = self._open(b)
        return b.cursor()

    def get(self, key, doff=-1, dlen=-1):
        # Find the correct bucket, and look in that cxn
        b = self.bucket(key)
        cxn = self._open(b)
        if doff != -1:
            return cxn.get(key, doff=doff, dlen=dlen)
        else:
            return cxn.get(key)

    def put(self, key, val):
        b = self.bucket(key)
        cxn = self._open(b)
        return cxn.put(key, val)

    def delete(self, key):
        b = self.bucket(key)
        cxn = self._open(b)
        return cxn.delete(key)

    def cursor(self):
        # create a switching cursor!
        l = self.listBuckets()
        return SwitchingBdbCursor(self, l)

    def sync(self):
        # sync all
        for c in self.cxns.itervalues():
            c.sync()
        return None

    def open(self, what, flags=0, dbtype=0, mode=0):
        # delay open until try to write
        self.basePath = what
        if flags == bdb.db.DB_CREATE:
            self.store.createArgs[what] = {'flags': flags,
                                           'dbtype': dbtype,
                                           'mode': mode}
        elif flags:
            self.openArgs = {'flags': flags}
        return None

    def close(self):
        for (k, c) in self.cxns.iteritems():
            c.close()
            self.cxns[k] = None
        return None

    def set_flags(self, f):
        self.preOpenFlags = f
        return None


class SwitchingBdbCursor(object):
    """Cursor to handle switching between multiple BerkeleyDB connections."""

    def __init__(self, cxn, l):
        self.switch = cxn
        nl = []
        for b in l:
            if cxn.bucket_exists(b):
                nl.append(b)
        self.buckets = nl
        self.currCursor = None
        self.currBucketIdx = -1

    def set_cursor(self, b):
        try:
            idx = self.buckets.index(b)
            self.currBucketIdx = idx
        except:
            msg = "%r is not a known bucket (%r)" % (b, self.buckets)
            raise ValueError(msg)
        cxn = self.switch._open(b)
        cur = cxn.cursor()
        if self.currCursor is not None:
            self.currCursor.close()
        self.currCursor = cur
        return cur

    def first(self, doff=-1, dlen=-1):
        if self.currBucketIdx != 0:
            cursor = self.set_cursor(self.buckets[0])
        if dlen != -1:
            return cursor.first(doff=doff, dlen=dlen)
        else:
            return cursor.first()

    def last(self, doff=-1, dlen=-1):
        if self.currBucketIdx != len(self.buckets) - 1:
            cursor = self.set_cursor(self.buckets[-1])
        else:
            cursor = self.currCursor
        if dlen != -1:
            return cursor.last(doff=doff, dlen=dlen)
        else:
            return cursor.last()

    def next(self, doff=-1, dlen=-1):
        # Needs to wrap
        if self.currCursor:
            if dlen != -1:
                x = self.currCursor.next(doff=doff, dlen=dlen)
            else:
                x = self.currCursor.next()
            if x is None and self.currBucketIdx != len(self.buckets) - 1:
                c = self.set_cursor(self.buckets[self.currBucketIdx + 1])
                if dlen != -1:
                    return c.first(doff=doff, dlen=dlen)
                else:
                    return c.first()
            else:
                return x
        else:
            return self.first(doff=doff, dlen=dlen)

    def prev(self, doff=-1, dlen=-1):
        # Needs to wrap
        if self.currCursor:
            if dlen != -1:
                x = self.currCursor.prev(doff=doff, dlen=dlen)
            else:
                x = self.currCursor.prev()
            if x is None and self.currBucketIdx != 0:
                c = self.set_cursor(self.buckets[self.currBucketIdx - 1])
                if dlen != -1:
                    return c.last(doff=doff, dlen=dlen)
                else:
                    return c.last()
            else:
                return x
        else:
            return self.last(doff=doff, dlen=dlen)

    def set_range(self, where, dlen=-1, doff=-1):
        # jump to where bucket
        b = self.switch.bucket(where)
        try:
            cursor = self.set_cursor(b)
        except ValueError:
            # non existant bucket
            tl = self.buckets[:]
            tl.append(b)
            tl.sort()
            idx = tl.index(b)
            if idx != len(self.buckets):
                nb = self.buckets[idx]
            else:
                nb = self.buckets[-1]
            cursor = self.set_cursor(nb)

        if dlen != -1:
            x = cursor.set_range(where, dlen=dlen, doff=doff)
        else:
            x = cursor.set_range(where)
        # at end of where bucket, step to next

        if x is None and self.currBucketIdx != len(self.buckets) - 1:
            c = self.set_cursor(self.buckets[self.currBucketIdx + 1])
            if dlen != -1:
                return c.first(doff=doff, dlen=dlen)
            else:
                return c.first()
        else:
            # End of index
            return x


class BdbStore(SimpleStore):
    """Berkeley DB based storage """
    cxns = {}

    def __init__(self, session, config, parent):
        self.cxns = {}
        self.createArgs = {}
        SimpleStore.__init__(self, session, config, parent)
        self.switchingClass = SwitchingBdbConnection

    def __iter__(self):
        # Return an iterator object to iter through... keys?
        return BdbIter(self.session, self)

    def _create(self, session, dbp):
        if self.switching:
            cxn = self.switchingClass(session, self, dbp)
        else:
            cxn = bdb.db.DB()
        cxn.set_flags(bdb.db.DB_RECNUM)
        try:
            cxn.open(dbp, dbtype=bdb.db.DB_BTREE,
                     flags=bdb.db.DB_CREATE, mode=0660)
        except:
            raise ValueError("Could not create: %s" % dbp)
        cxn.close()

    def _verifyDb(self, session, dbType):
        dbp = self.get_path(session, dbType + "Path")
        if self.switching and dbType in self.get_noSwitchTypes(session):
            self.switching = False
            rv = self._verify(session, dbp)
            self.switching = True
            return rv
        else:
            return self._verify(session, dbp)

    def _verify(self, session, dbp):
        if (not os.path.exists(dbp)):
            # We don't exist, try and instantiate new database
            self._create(session, dbp)
        else:
            cxn = bdb.db.DB()
            try:
                cxn.open(dbp)
                cxn.close()
            except:
                # try to recreate
                self._create(session, dbp)

    def _openDb(self, session, dbType):
        cxn = self.cxns.get(dbType, None)
        if cxn is None:
            dbp = self.get_path(session, dbType + 'Path')
            if dbp is None:
                self._initDb(session, dbType)
                self._verifyDb(session, dbType)
                dbp = self.get_path(session, dbType + 'Path')
            if (os.path.exists(dbp) or
                dbType in self.storageTypes or
                (dbType[-7:] == 'Reverse' and
                 dbType[:-7] in self.reverseMetadataTypes)):
                if (
                    self.switching and
                    dbType in self.get_noSwitchTypes(session)
                ):
                    self.switching = False
                    cxn = self._open(session, dbp)
                    self.switching = True
                else:
                    cxn = self._open(session, dbp)
                self.cxns[dbType] = cxn
                return cxn
            else:
                # Trying to store something we don't care about
                return None
        else:
            return cxn

    def _open(self, session, dbp):
        if self.switching:
            cxn = self.switchingClass(session, self, dbp)
        else:
            cxn = bdb.db.DB()
        cxn.set_flags(bdb.db.DB_RECNUM)
        if session.environment == "apache":
            cxn.open(dbp, flags=bdb.db.DB_NOMMAP)
        else:
            cxn.open(dbp)
        return cxn

    def _closeDb(self, session, dbType):
        cxn = self.cxns.get(dbType, None)
        if cxn is not None:
            try:
                cxn.close()
            except:
                # silently fail, as we're closing anyway
                pass
            self.cxns[dbType] = None

    def _remove(self, session, dbp):
        try:
            cxn = bdb.db.DB()
            cxn.remove(dbp)
        except:
            pass

    def get_noSwitchTypes(self, session):
        return ['digest', 'digestReverse', 'metadata']

    def get_dbSize(self, session):
        cxn = self._openDb(session, 'digest')
        if cxn is None:
            cxn = self._openDb(session, 'database')
        return cxn.stat(bdb.db.DB_FAST_STAT)['nkeys']

    def generate_id(self, session):
        if self.useUUID:
            return gen_uuid()

        cxn = self._openDb(session, 'digest')
        if cxn is None:
            cxn = self._openDb(session, 'database')
        if (self.currentId == -1 or session.environment == "apache"):
            c = cxn.cursor()
            item = c.last()
            if item:
                # Might need to out normalise key
                key = item[0]
                if self.outIdNormalizer:
                    key = self.outIdNormalizer.process_string(session, key)
                    if not type(key) in (int, long):
                        self.useUUID = 1
                        key = gen_uuid()
                    else:
                        key += 1
                else:
                    key = long(key)
                    key += 1
            else:
                key = 0
        else:
            key = self.currentId + 1
        self.currentId = key
        return key

    def store_data(self, session, id, data, metadata={}):
        dig = metadata.get('digest', "")
        if dig:
            cxn = self._openDb(session, 'digestReverse')
            if cxn:
                exists = cxn.get(dig)
                if exists:
                    raise ObjectAlreadyExistsException(exists)

        cxn = self._openDb(session, 'database')
        # Should always have an id by now, but just in case
        if id is None:
            id = self.generate_id(session)
        if (self.idNormalizer is not None):
            id = self.idNormalizer.process_string(session, id)
        elif type(id) == unicode:
            id = id.encode('utf-8')
        else:
            id = str(id)

        if type(data) == unicode:
            data = data.encode('utf-8')
        cxn.put(id, data)

        for (m, val) in metadata.iteritems():
            self.store_metadata(session, id, m, val)
        return None

    def fetch_data(self, session, id):
        cxn = self._openDb(session, 'database')
        if (self.idNormalizer is not None):
            id = self.idNormalizer.process_string(session, id)
        elif type(id) == unicode:
            id = id.encode('utf-8')
        else:
            id = str(id)
        data = cxn.get(id)

        if (
            data and
            data[:44] == "\0http://www.cheshire3.org/ns/status/DELETED:"
        ):
            data = DeletedObject(self, id, data[41:])

        if data and self.expires:
            # update touched
            expires = self.generate_expires(session)
            self.store_metadata(session, id, 'expires', expires)
        return data

    def delete_data(self, session, id):
        self._openAll(session)
        cxn = self._openDb(session, 'database')

        if (self.idNormalizer is not None):
            id = self.idNormalizer.process_string(session, id)
        elif type(id) == unicode:
            id = id.encode('utf-8')
        else:
            id = str(id)

        # Main database is a storageType now
        for dbt in self.storageTypes:
            cxn = self._openDb(session, dbt)
            if cxn is not None:
                if dbt in self.reverseMetadataTypes:
                    # fetch value here, delete reverse
                    data = cxn.get(id)
                    cxn2 = self._openDb(session, dbt + "Reverse")
                    if cxn2 is not None:
                        cxn2.delete(data)
                cxn.delete(id)
                cxn.sync()

        # Maybe store the fact that this object used to exist.
        if self.get_setting(session, 'storeDeletions', 0):
            cxn = self._openDb(session, 'database')
            now = datetime.datetime.now(dateutil.tz.tzutc())
            now = now.strftime("%Y-%m-%dT%H:%M:%S%Z").replace('UTC', 'Z')
            cxn.put(id,
                    "\0http://www.cheshire3.org/ns/status/DELETED:%s" % now)
            cxn.sync()

    def fetch_metadata(self, session, id, mType):
        if not mType.endswith("Reverse"):
            if (self.idNormalizer is not None):
                id = self.idNormalizer.process_string(session, id)
            elif type(id) == unicode:
                id = id.encode('utf-8')
            elif type(id) != str:
                id = str(id)
#        self.log_debug(session, mType)
        cxn = self._openDb(session, mType)
#        self.log_debug(session, cxn)
        if cxn is not None:
            data = cxn.get(id)
            if data:
                if data.startswith("\0http://www.cheshire3.org/ns/"
                                   "datatype/PICKLED:"):
                    data = data.split(':', 2)[2]
                    data = pickle.loads(data)
                elif mType.endswith(("Count", "Position", "Amount", "Offset")):
                    data = long(data)
                elif mType.endswith("Date"):
                    data = dateparser.parse(data)
            return data
        else:
            return None

    def store_metadata(self, session, id, mType, value):
        if value is None:
            return
        cxn = self._openDb(session, mType)
        if cxn is None:
            self._initDb(session, mType)
            self._verifyDb(session, mType)
            cxn = self._openDb(session, mType)
        if cxn is not None:
            if isinstance(value, (int, long, float, datetime.datetime)):
                value = str(value)
            elif isinstance(value, (dict, list, tuple)):
                value = ("\0http://www.cheshire3.org/ns/datatype/PICKLED:"
                         "{0}".format(pickle.dumps(value)))
            cxn.put(id, value)
            if mType in self.reverseMetadataTypes:
                cxn = self._openDb(session, mType + "Reverse")
                if cxn is not None:
                    cxn.put(value, id)

    def flush(self, session):
        # Call sync to flush all to disk
        for cxn in self.cxns.values():
            if cxn is not None:
                cxn.sync()

    def clear(self, session):
        self._closeAll(session)
        self.cxns = {}
        for t in self.get_storageTypes(session):
            p = self.get_path(session, "%sPath" % t)
            self._remove(session, p)
            self._initDb(session, t)
            self._verifyDb(session, t)
        for t in self.get_reverseMetadataTypes(session):
            p = self.get_path(session, "%sReversePath" % t)
            self._remove(session, p)
            self._initDb(session, "%sReverse" % t)
            self._verifyDb(session, "%sReverse" % t)
        return self

    def clean(self, session):
        self._openAll(session)
        now = time.time()
        cxn = self._open(session, 'expiresReverse')
        c = cxn.cursor()
        deleted = 0
        try:
            (key, data) = c.set_range(str(now))
            # float(time) --> object id
            if (float(key) <= now):
                self.delete_data(session, data)
                deleted = 1
            (key, data) = c.prev()
        except:
            # No database
            deleted = 0
            key = False
        while key:
            self.delete_data(session, data)
            deleted += 1
            try:
                (key, data) = c.prev()
                if not float(key):
                    break
            except:
                # Reached beginning
                break
        self._closeAll(session)
        return self


class FileSystemIter(object):
    store = None
    cursor = None
    cxn = None
    nextData = None

    def __init__(self, store):
        self.store = store
        self.session = Session()
        self.cxn = store._open(self.session, 'byteCount')
        self.cursor = self.cxn.cursor()
        (key, val) = self.cursor.first()
        self.nextData = (key, self.store.fetch_data(self.session, key))

    def __iter__(self):
        return self

    def next(self):
        try:
            d = self.nextData
            if not d:
                raise StopIteration()
            (key, val) = self.cursor.next()
            self.nextData = (key, self.store.fetch_data(self.session, key))
            return d
        except:
            raise StopIteration()

    def jump(self, position):
        # Jump to this position
        self.nextData = self.cursor.set_range(position)
        return self.nextData[0]


class FileSystemStore(BdbStore):
    # Leave the data somewhere on disk
    # Use metadata to map identifier to file, offset, length
    # Object to be stored needs this information, obviously!

    currFileHandle = None
    currFileName = ""

    def __iter__(self):
        return FileSystemIter(self.session, self)

    def get_storageTypes(self, session):
        return ['filename', 'byteCount', 'byteOffset']

    def get_reverseMetadataTypes(self, session):
        return ['digest', 'expires']

    def store_data(self, session, id, data, metadata={}):
        dig = metadata.get('digest', "")
        if dig:
            cxn = self._openDb(session, 'digestReverse')
            if cxn:
                exists = cxn.get(dig)
                if exists:
                    raise ObjectAlreadyExistsException(exists)

        if (self.idNormalizer is not None):
            id = self.idNormalizer.process_string(session, id)
        elif type(id) == unicode:
            id = id.encode('utf-8')
        else:
            id = str(id)

        if not ('filename' in metadata and
                'byteCount' in metadata and
                'byteOffset' in metadata):
            msg = "Need file, byteOffset and byteCount to use FileSystemStore"
            raise ValueError(msg)

        for (m, val) in metadata.iteritems():
            self.store_metadata(session, id, m, val)
        return None

    def fetch_data(self, session, id):
        if (self.idNormalizer is not None):
            id = self.idNormalizer.process_string(session, id)
        elif type(id) == unicode:
            id = id.encode('utf-8')
        else:
            id = str(id)

        filename = self.fetch_metadata(session, id, 'filename')
        start = self.fetch_metadata(session, id, 'byteOffset')
        length = self.fetch_metadata(session, id, 'byteCount')

        if filename is None and start is None and length is None:
            # Data has been deleted
            return None

        if filename != self.currFileName:
            if self.currFileHandle:
                self.currFileHandle.close()
            self.currFileHandle = file(filename)
        try:
            self.currFileHandle.seek(start)
        except:
            # closed, reopen
            self.currFileHandle = file(filename)
            self.currFileHandle.seek(start)
        data = self.currFileHandle.read(length)

        if (
            data and
            data[:44] == "\0http://www.cheshire3.org/ns/status/DELETED:"
        ):
            data = DeletedObject(self, id, data[41:])
        elif self.outWorkflow:
            data = self.outWorkflow.process(session, data)

        if data and self.expires:
            # update touched
            expires = self.generate_expires(session)
            self.store_metadata(session, id, 'expires', expires)
        return data

    def delete_data(self, session, id):
        self._openAll(session)

        if (self.idNormalizer is not None):
            id = self.idNormalizer.process_string(session, id)
        elif type(id) == unicode:
            id = id.encode('utf-8')
        else:
            id = str(id)

        filename = self.fetch_metadata(session, id, 'filename')
        start = self.fetch_metadata(session, id, 'byteOffset')
        length = self.fetch_metadata(session, id, 'byteCount')

        # Main database is a storageType now
        for dbt in self.storageTypes:
            cxn = self._openDb(session, dbt)
            if cxn is not None:
                if dbt in self.reverseMetadataTypes:
                    # Fetch value here, delete reverse
                    data = cxn.get(id)
                    cxn2 = self._openDb(session, dbt + "Reverse")
                    if cxn2 is not None:
                        cxn2.delete(data)
                cxn.delete(id)
                cxn.sync()

        # Maybe store the fact that this object used to exist.
        if self.get_setting(session, 'storeDeletions', 0):
            now = datetime.datetime.now(dateutil.tz.tzutc())
            now = now.strftime("%Y-%m-%dT%H:%M:%S%Z").replace('UTC', 'Z')
            out = "\0http://www.cheshire3.org/ns/status/DELETED:%s" % now

            if len(out) < length:
                f = file(filename, 'w')
                f.seek(start)
                f.write(out)
                f.close()
            else:
                # Can't write deleted status as original doc is shorter than
                # deletion info!
                pass


def directoryStoreIter(store):
    session = Session()
    databasePath = store.get_path(session, 'databasePath')
    for root, dirs, files in os.walk(databasePath):
        for name in files:
            filepath = os.path.join(root, name)
            # Split off identifier
            id_ = filepath[len(databasePath) + 1:]
            # De-normalize id
            if not store.allowStoreSubDirs:
                id_ = unquote(id_)
            if store.outIdNormalizer is not None:
                id_ = store.outIdNormalizer.process_string(session, id_)
            # Read in data
            with open(filepath, 'r') as fh:
                data = fh.read()
            # Check for DeletedObject
            if (
                data and
                data.startswith("\0http://www.cheshire3.org/ns/status/"
                                "DELETED:")
            ):
                data = DeletedObject(self, id, data[41:])

            # Update expires
            if data and store.expires:
                expires = store.generate_expires(session)
                store.store_metadata(session, id, 'expires', expires)

            yield (id_, data)
        # By default don't iterate over VCS directories
        for vcs in ['CVS', '.git', '.hg', '.svn']:
            try:
                dirs.remove(vcs)
            except:
                # This VCS isn't there, so no need to remove
                continue


class DirectoryStore(BdbStore):
    """Store Objects as files in a directory on the filesystem.

    Really simple Store to store Objects as files within a directory (and
    possibly sub-directories). An important thing to remember is that
    files may be added/modified/deleted by an external entity.
    """

    _possibleSettings = {
        'createSubDir': {
            'docs': ('Should a sub-directory/sub-collection be used for this '
                     'store'),
            'type': int,
            'options': "0|1"
        },
        'allowStoreSubDirs': {
            'docs': ('Allow Store to create sub-directories if it encounters '
                     'an operating system path separator in an identifier. If'
                     ' false (0) operating system path separators are escaped.'
                     ),
            'type': int,
            'options': '0|1'
        }
    }

    def __init__(self, session, config, parent):
        BdbStore.__init__(self, session, config, parent)
        if self.switching:
            raise ConfigFileException('Switching not supported by {0}'
                                      ''.format(self.__class__.__name__))
        self.allowStoreSubDirs = self.get_setting(session,
                                                  'allowStoreSubDirs',
                                                  1)
        # TODO: Refresh metadata in case files have changed

    def __iter__(self):
        return directoryStoreIter(self)

    def _initDb(self, session, dbt):
        dbp = dbt + "Path"
        databasePath = self.get_path(session, dbp, "")
        if not databasePath:
            databasePath = self.id
        elif self.get_setting(session, 'createSubDir', 0):
            databasePath = os.path.join(databasePath, self.id)
        if (not os.path.isabs(databasePath)):
            # Prepend defaultPath from parents
            dfp = self.get_path(session, 'defaultPath')
            if not dfp:
                msg = ("Store has relative path, and no visible "
                       "defaultPath.")
                raise ConfigFileException(msg)
            databasePath = os.path.join(dfp, databasePath)
        self.paths[dbp] = databasePath

    def _verifyDb(self, session, dbType):
        dbp = self.get_path(session, dbType + "Path")
        if dbType == 'database':
            # Simply the directory in which to store data
            # Ensure that it exists (including any intermediate dirs)
            if not os.path.exists(dbp):
                os.makedirs(dbp)
        else:
            return BdbStore._verify(self, session, dbp)

    def _openDb(self, session, dbType):
        if dbType == 'database':
            # Simply the directory in which to store data
            # Ensure that it exists
            dbp = self.get_path(session, dbType + 'Path')
            if dbp is None:
                self._initDb(session, dbType)
                self._verifyDb(session, dbType)
        else:
            return BdbStore._openDb(self, session, dbType)

    def _closeDb(self, session, dbType):
        if dbType == 'database':
            # Simply the directory in which to store data - do nothing
            pass
        else:
            return BdbStore._closeDb(self, session, dbType)

    def _normalizeIdentifier(self, session, identifier):
        # Apply any necessary normalization to the identifier
        if self.idNormalizer is not None:
            identifier = self.idNormalizer.process_string(session, identifier)
        elif type(id) == unicode:
            identifier = identifier.encode('utf-8')
        else:
            identifier = str(identifier)
        return identifier

    def _getFilePath(self, session, identifier):
        if os.path.sep in identifier and not self.allowStoreSubDirs:
            # Escape os path separator
            identifier = quote(identifier)
        databasePath = self.get_path(session, 'databasePath')
        return os.path.join(databasePath, identifier)

    def generate_id(self, session):
        """Generate and return a new unique identifier."""
        return self.get_dbSize(session)

    def get_storageTypes(self, session):
        return ['database']

    def get_reverseMetadataTypes(self, session):
        return ['digest']

    def get_dbSize(self, session):
        """Return number of items in storage."""
        databasePath = self.get_path(session, 'databasePath')
        return sum([len(t[2]) for t in os.walk(databasePath)])

    def delete_data(self, session, identifier):
        """Delete data stored against id."""
        self._openAll(session)
        identifier = self._normalizeIdentifier(session, identifier)
        filepath = self._getFilePath(session, identifier)

        # Main database is a storageType now
        for dbt in self.storageTypes:
            if dbt == 'database':
                # Simply the directory in which to store data
                # Delete the file
                os.remove(filepath)
            else:
                cxn = self._openDb(session, dbt)
                if cxn is not None:
                    if dbt in self.reverseMetadataTypes:
                        # Fetch value here, delete reverse
                        data = cxn.get(identifier)
                        cxn2 = self._openDb(session, dbt + "Reverse")
                        if cxn2 is not None:
                            cxn2.delete(data)
                    cxn.delete(identifier)
                    cxn.sync()

        # Maybe store the fact that this object used to exist.
        if self.get_setting(session, 'storeDeletions', 0):
            now = datetime.datetime.now(dateutil.tz.tzutc())
            now = now.strftime("%Y-%m-%dT%H:%M:%S%Z").replace('UTC', 'Z')
            with open(filepath, 'w') as fh:
                fh.write("\0http://www.cheshire3.org/ns/status/DELETED:{0}"
                         "".format(now)
                         )

    def fetch_data(self, session, identifier):
        """Return data stored against identifier."""
        identifier = self._normalizeIdentifier(session, identifier)
        filepath = self._getFilePath(session, identifier)
        try:
            with open(filepath) as fh:
                data = fh.read()
        except IOError:
            # No file
            data = None
        if (
            data and
            data.startswith("\0http://www.cheshire3.org/ns/status/DELETED:")
        ):
            data = DeletedObject(self, identifier, data[41:])
        if data and self.expires:
            expires = self.generate_expires(session)
            self.store_metadata(session, identifier, 'expires', expires)
        return data

    def store_data(self, session, identifier, data, metadata={}):
        """Store data against identifier."""
        dig = metadata.get('digest', "")
        if dig:
            cxn = self._openDb(session, 'digestReverse')
            if cxn:
                exists = cxn.get(dig)
                if exists:
                    raise ObjectAlreadyExistsException(exists)
        # Should always have an id by now, but just in case
        if identifier is None:
            identifier = self.generate_id(session)
        identifier = self._normalizeIdentifier(session, identifier)
        filepath = self._getFilePath(session, identifier)
        # Check for subdirectories
        if os.path.sep in identifier and self.allowStoreSubDirs:
            # Create necessary sub-directories
            directory, filename = os.path.split(filepath)
            os.makedirs(directory)
        # Encode data if necessary
        if type(data) == unicode:
            data = data.encode('utf-8')
        with open(filepath, 'w') as fh:
            fh.write(data)
        for (m, val) in metadata.iteritems():
            self.store_metadata(session, identifier, m, val)
        return None

    def fetch_metadata(self, session, identifier, mType):
        """Return mType metadata stored against identifier."""
        return BdbStore.fetch_metadata(self, session, identifier, mType)

    def store_metadata(self, session, identifier, mType, value):
        """Store value for mType metadata against identifier."""
        return BdbStore.store_metadata(self, session, identifier, mType, value)

    def clean(self, session):
        """Delete expired data objects."""
        return BdbStore.clear(self, session)

    def clear(self, session):
        """Delete all the data out of self."""
        self._closeAll(session)
        self.cxns = {}
        for t in self.get_storageTypes(session):
            p = self.get_path(session, "%sPath" % t)
            if t == 'database':
                # Simply the directory in which to store data
                # Clear the entire directory
                shutil.rmtree(p)
            else:
                self._remove(session, p)
            self._initDb(session, t)
            self._verifyDb(session, t)
        for t in self.get_reverseMetadataTypes(session):
            p = self.get_path(session, "%sReversePath" % t)
            self._remove(session, p)
            self._initDb(session, "%sReverse" % t)
            self._verifyDb(session, "%sReverse" % t)
        return self

    def flush(self, session):
        """Ensure all data is flushed to disk."""
        return BdbStore.flush(self, session)
