resultSetStore

Module resultSetStore |
[hide private]
[frames] | no frames]

## Source Code for Module resultSetStore

      1
      2  from configParser import C3Object
      3  from baseObjects import ResultSetStore
      4  from baseStore import BdbStore
      5  from resultSet import SimpleResultSet, SimpleResultSetItem
      6  import os, types, struct, sys, commands, string, time, dynamic
      7  from random import Random
      8  try:
      9      # Python 2.3 vs 2.2
     10      import bsddb as bdb
     11  except:
     12      import bsddb3 as bdb
     13
     14  try:
     15      from Ft.Lib.Uuid import GenerateUuid, UuidAsString
     16      use4Suite = 1
     17  except:
     18      use4Suite = 0
     19
     20  randomGen = Random(time.time())
     21  asciiChars = string.ascii_letters + string.digits + "@%#!-=."
     22
     23  # name of result set needs to be unique within RSS
     24  # But users may name result sets non uniquely in Z
     25  # This map needs to happen at the user/session end
     26  # Hence only the RSS can name a result set.
     27
     28
     29 -class SimpleResultSetStore(ResultSetStore):
     30      storeHash = {}
     31      storeHashReverse = {}
     32      databaseHash = {}
     33      databaseHashReverse = {}
     34
     35 -    def __init__(self, session, parent, config):
     36          ResultSetStore.__init__(self, session, parent, config)
     37          rsh = self.get_path(session, 'recordStoreHash')
     38          if rsh:
     39              wds = rsh.split()
     40              for w in range(len(wds)):
     41                  self.storeHash[long(w)] = wds[w]
     42                  self.storeHashReverse[wds[w]] = long(w)
     43          dbsh = self.get_path(session, 'databaseHash')
     44          if dbsh:
     45              wds = dbsh.split()
     46              for w in range(len(wds)):
     47                  self.databaseHash[long(w)] = wds[w]
     48                  self.databaseHashReverse[wds[w]] = long(w)
     49
     50 -    def _verifyDefault(self, type, value):
     51          if (type == "resultSetTimeout"):
     52              return int(value)
     53          else:
     54              return value
     55
     56 -    def generate_id(self):
     57          if (use4Suite):
     58              id = UuidAsString(GenerateUuid())
     59          else:
     60              id = commands.getoutput('uuidgen')
     61              if (len(id) <> 36 or id[8] <> '-'):
     62                  c = []
     63                  for x in range(16):
     64                      c.append(asciiChars[randomGen.randrange(len(asciiChars))])
     65                  id = ''.join(c)
     66          return id
     67
     68
     69 -class BdbResultSetStore(SimpleResultSetStore, BdbStore):
     70      cxn = None
     71      txn = None
     72
     73 -    def __init__(self, session, parent, config):
     74          SimpleResultSetStore.__init__(self, session, parent, config)
     75          BdbStore.__init__(self, session, parent, config)
     76
     77 -    def create_resultSet(self, session, rset=None):
     78          id = self.generate_id()
     79          if (not rset):
     80              # Create a place holder with no information
     81              now = time.time()
     82              expires = now + self.get_default(session, 'resultSetTimeout')
     83              self.store_data(session, "_%s__%f" % (id, expires), "")
     84              self.store_data(session, "%f" % expires, id)
     85              self.commit_storing(session)
     86          else:
     87              rset.id = id
     88              self.store_resultSet(session, rset)
     89          return id
     90
     91      # XXX
     92 -    def delete_resultSet(self, session, rsid):
     93          self._openContainer(session)
     94          c = self.cxn.cursor()
     95          (key, data) = c.set_range("_%s__" % (rsid))
     96          if (key[1:len(rsid)+1] <> rsid):
     97              # Not result set
     98              raise ValueError
     99          cexp = key[len(rsid)+3:]
    100          self.cxn.delete(cexp)
    101          self.cxn.delete(key)
    102          self.commit_storing(session)
    103
    104      # XXX
    105 -    def fetch_resultSet(self, session, rsid):
    106          self._openContainer(session)
    107          c = self.cxn.cursor()
    108          try:
    109              (key, data) = c.set_range("_%s__" % (rsid))
    110          except:
    111              raise ObjectDoesNotExistException("%s/%s", (self.id,rsid))
    112          if (key[1:len(rsid)+1] <> rsid):
    113              # Not result set
    114              raise ObjectDoesNotExistException("%s/%s", (self.id,rsid))
    115
    116          # XXX:  Need to know requested timeout??
    117          now = time.time()
    118          expires = now + self.get_default(session, 'resultSetTimeout')
    119
    120          # Update expires
    121          self.delete_resultSet(session, rsid)
    122          self._openContainer(session)
    123          self.cxn.put("_%s__%f" % (rsid, expires), data)
    124          self.cxn.put("%f" % (expires), rsid)
    125
    126          if (data):
    127              unpacked = struct.unpack("L" * (len(data) / 4), data)
    128              items = []
    129              for o in range(len(unpacked))[::4]:
    130                  db = self.databaseHash[unpacked[o+3]]
    131                  items.append(SimpleResultSetItem(session, unpacked[o], self.storeHash[unpacked[o+1]], unpacked[o+2], db))
    132              self.commit_storing(session)
    133              return SimpleResultSet(session, items, rsid)
    134          else:
    135              self.commit_storing(session)
    136              return SimpleResultSet(session, [], rsid)
    137
    138 -    def fetch_resultSetList(self, session):
    139          self._openContainer(session)
    140          c = self.cxn.cursor()
    141          (key, data) = c.set_range("_")
    142          keys = [key[1:37]]
    143          n = c.next()
    144          while(n):
    145              keys.append(n[0][1:37])
    146              n = c.next()
    147          return keys
    148
    149
    150 -    def store_resultSet(self, session, rset):
    151          # Store document links
    152          # (long, long) -> (store, docid)
    153          # load store map ala IndexStore
    154
    155          self._openContainer(session)
    156          idlist = []
    157          for k in range(len(rset)):
    158              storeid = rset[k].recordStore
    159
    160              id = rset[k].docid
    161              if (type(storeid) <> types.IntType):
    162                  # Map
    163                  if (self.storeHashReverse.has_key(storeid)):
    164                      storeid = self.storeHashReverse[storeid]
    165                  else:
    166                      self.storeHashReverse[storeid] = len(self.storeHash.keys())
    167                      self.storeHash[self.storeHashReverse[storeid]] = storeid
    168                      storeid = self.storeHashReverse[storeid]
    169                      # XXX: Need to make this persistent!!!
    170              databaseid = rset[k].database
    171              if (type(databaseid) <> types.IntType):
    172                  # Map
    173                  if (self.databaseHashReverse.has_key(databaseid)):
    174                      databaseid = self.databaseHashReverse[databaseid]
    175                  else:
    176                      self.databaseHashReverse[databaseid] = len(self.databaseHash.keys())
    177                      self.databaseHash[self.databaseHashReverse[databaseid]] = databaseid
    178                      databaseid = self.databaseHashReverse[databaseid]
    179                      # XXX: Need to make this persistent!!!
    180
    181              idlist.extend([id, storeid, rset[k].occurences, databaseid])
    182          params = ['L' * len(idlist)]
    183          params.extend(idlist)
    184          data = apply(struct.pack, params)
    185          now = time.time()
    186          if (rset.expires):
    187              expires = now + rset.expires
    188          else:
    189              expires = now + self.get_default(session, 'resultSetTimeout')
    190          try:
    191              self.delete_resultSet(session, rset.id)
    192          except:
    193              pass
    194          self._openContainer(session)
    195          self.cxn.put("_%s__%f" % (rset.id, expires), data)
    196          self.cxn.put("%f" % (expires), rset.id)
    197          self.commit_storing(session)
    198
    199 -    def clean(self, session):
    200          now = time.time()
    201          self._openContainer(session)
    202          c = self.cxn.cursor()
    203          try:
    204              (key, data) = c.set_range(str(now))
    205          except:
    206              # No database
    207              return 0
    208          deleted = 0
    209          while key:
    210              if (key[0] <> '_'):
    211                  if (float(key) <= now):
    212                      self.delete_resultSet(session, data)
    213                      deleted += 1
    214              try:
    215                  (key, data) = c.prev()
    216              except:
    217                  return deleted
    218
    219
    220 -class BdbResultSetStore2(BdbResultSetStore):
    221      storeHash = {}
    222      storeHashReverse = {}
    223      databaseHash = {}
    224      databaseHashReverse = {}
    225      cxn = None
    226      txn = None
    227
    228 -    def __init__(self, session, parent, config):
    229          ResultSetStore.__init__(self, session, parent, config)
    230          BdbStore.__init__(self, session, parent, config)
    231
    232
    233 -    def fetch_resultSet(self, session, rsid):
    234          self._openContainer(session)
    235          c = self.cxn.cursor()
    236          try:
    237              (key, data) = c.set_range("_%s__" % (rsid))
    238          except:
    239              return None
    240          if (key[1:len(rsid)+1] <> rsid):
    241              # Not result set
    242              return None
    243
    244          # XXX:  Need to know requested timeout??
    245          now = time.time()
    246          expires = now + self.get_default(session, 'resultSetTimeout')
    247
    248          # Update expires
    249          self.delete_resultSet(session, rsid)
    250          self._openContainer(session)
    251          self.cxn.put("_%s__%f" % (rsid, expires), data)
    252          self.cxn.put("%f" % (expires), rsid)
    253
    254          if (data):
    255              (cl, srlz) = data.split('||', 1)
    256              rset = dynamic.buildObject(session, cl, [])
    257              rset.deserialise(session, srlz)
    258              self.commit_storing(session)
    259              return rset
    260          else:
    261              self.commit_storing(session)
    262              return SimpleResultSet(session, [], rsid)
    263
    264
    265 -    def store_resultSet(self, session, rset):
    266          self._openContainer(session)
    267
    268          # Serialise resultSet to data + class
    269          srlz = rset.serialise(session)
    270          cl = str(rset.__class__)
    271          data = cl + "||" + srlz
    272
    273          # Store
    274          now = time.time()
    275          if (rset.expires):
    276              expires = now + rset.expires
    277          else:
    278              expires = now + self.get_default(session, 'resultSetTimeout')
    279          try:
    280              self.delete_resultSet(session, rset.id)
    281          except:
    282              pass
    283          self._openContainer(session)
    284          self.cxn.put("_%s__%f" % (rset.id, expires), data)
    285          self.cxn.put("%f" % (expires), rset.id)
    286          self.commit_storing(session)
    287