starcluster.utils

Package starcluster :: Module utils |
[hide private]
[frames] | no frames]

# Source Code for Module starcluster.utils

      1  #!/usr/bin/env python
      2  """
      3  Utils module for StarCluster
      4  """
      5
      6  import os
      7  import re
      8  import time
      9  import types
     10  import inspect
     11  import calendar
     12  import urlparse
     13  import decorator
     14  from datetime import datetime
     15
     16  from starcluster import iptools
     17  from starcluster import exception
     18  from starcluster.logger import log
     19
     20  try:
     21      import IPython
     22      if IPython.__version__ < '0.11':
     23          from IPython.Shell import IPShellEmbed
     24          ipy_shell = IPShellEmbed(argv=[])
     25      else:
     26          from IPython import embed
     27          ipy_shell = lambda local_ns=None: embed(user_ns=local_ns)
     28  except ImportError:
     29
     30 -    def ipy_shell(local_ns=None):
     31          log.error("Unable to load IPython.")
     32          log.error("Please check that IPython is installed and working.")
     33          log.error("If not, you can install it via: easy_install ipython")
     34
     35  try:
     36      import pudb
     37      set_trace = pudb.set_trace
     38  except ImportError:
     39
     40 -    def set_trace():
     41          log.error("Unable to load PuDB")
     42          log.error("Please check that PuDB is installed and working.")
     43          log.error("If not, you can install it via: easy_install pudb")
     44
     45
     46 -class AttributeDict(dict):
     47      """
     48      Subclass of dict that allows read-only attribute-like access to
     49      dictionary key/values
     50      """
     51 -    def __getattr__(self, name):
     52          try:
     53              return self.__getitem__(name)
     54          except KeyError:
     55              return super(AttributeDict, self).__getattribute__(name)
     56
     57
     58 -def print_timing(msg=None):
     59      """
     60      Decorator for printing execution time (in mins) of a function
     61      Optionally takes a user-friendly msg as argument. This msg will
     62      appear in the sentence "[msg] took XXX mins". If no msg is specified,
     63      msg will default to the decorated function's name. e.g:
     64
     65      @print_timing
     66      def myfunc():
     67          print 'Running myfunc'
     68      >>> myfunc()
     69      Running myfunc
     70      myfunc took 0.000 mins
     71
     72      @print_timing('My function')
     73      def myfunc():
     74          print 'Running myfunc'
     75      >>> myfunc()
     76      Running myfunc
     77      My function took 0.000 mins
     78      """
     79      prefix = msg
     80      if type(msg) == types.FunctionType:
     81          prefix = msg.func_name
     82
     83      def wrap_f(func, *arg, **kargs):
     84          """Raw timing function """
     85          time1 = time.time()
     86          res = func(*arg, **kargs)
     87          time2 = time.time()
     88          log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0))
     89          return res
     90
     91      if type(msg) == types.FunctionType:
     92          return decorator.decorator(wrap_f, msg)
     93      else:
     94          return decorator.decorator(wrap_f)
     95
     96
     97 -def is_valid_device(dev):
     98      """
     99      Checks that dev matches the following regular expression:
    100      /dev/sd[a-z]$
    101      """
    102      regex = re.compile('/dev/sd[a-z]$')
    103      try:
    104          return regex.match(dev) is not None
    105      except TypeError:
    106          return False
    107
    108
    109 -def is_valid_partition(part):
    110      """
    111      Checks that part matches the following regular expression:
    112      /dev/sd[a-z][1-9][0-9]?$
    113      """
    114      regex = re.compile('/dev/sd[a-z][1-9][0-9]?$')
    115      try:
    116          return regex.match(part) is not None
    117      except TypeError:
    118          return False
    119
    120
    121 -def is_valid_bucket_name(bucket_name):
    122      """
    123      Check if bucket_name is a valid S3 bucket name (as defined by the AWS
    124      docs):
    125
    126      1. 3 <= len(bucket_name) <= 255
    127      2. all chars one of: a-z 0-9 .  _ -
    128      3. first char one of: a-z 0-9
    129      4. name must not be a valid ip
    130      """
    131      regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$')
    132      if not regex.match(bucket_name):
    133          return False
    134      if iptools.validate_ip(bucket_name):
    135          return False
    136      return True
    137
    138
    139 -def is_valid_image_name(image_name):
    140      """
    141      Check if image_name is a valid AWS image name (as defined by the AWS docs)
    142
    143      1. 3<= len(image_name) <=128
    144      2. all chars one of: a-z A-Z 0-9 ( ) . - / _
    145      """
    146      regex = re.compile('[\w\(\)\.\-\/_]{3,128}$')
    147      try:
    148          return regex.match(image_name) is not None
    149      except TypeError:
    150          return False
    151
    152
    153 -def make_one_liner(script):
    154      """
    155      Returns command to execute python script as a one-line python program
    156
    157      e.g.
    158
    159          import os
    160          script = '''
    161          import os
    162          print os.path.exists('hi')
    163          '''
    164          os.system(make_one_liner(script))
    165
    166      Will print out:
    167
    168          <module 'os' from ...>
    169          False
    170      """
    171      return 'python -c "%s"' % script.strip().replace('\n', ';')
    172
    173
    174 -def is_url(url):
    175      """
    176      Returns True if the provided string is a valid url
    177      """
    178      try:
    179          parts = urlparse.urlparse(url)
    180          scheme = parts[0]
    181          netloc = parts[1]
    182          if scheme and netloc:
    183              return True
    184          else:
    185              return False
    186      except:
    187          return False
    188
    189
    190 -def is_iso_time(iso):
    191      """
    192      Returns True if provided time can be parsed in iso format
    193      to a datetime tuple
    194      """
    195      try:
    196          iso_to_datetime_tuple(iso)
    197          return True
    198      except ValueError:
    199          return False
    200
    201
    202 -def iso_to_datetime_tuple(iso):
    203      """
    204      Converts an iso time string to a datetime tuple
    205      """
    206      #remove timezone
    207      iso = iso.split('.')[0]
    208      try:
    209          return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S")
    210      except AttributeError:
    211          # python2.4 datetime module doesnt have strptime
    212          return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
    213
    214
    215 -def datetime_tuple_to_iso(tup):
    216      """
    217      Converts a datetime tuple to iso time string
    218      """
    219      iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S")
    220      return iso
    221
    222
    223 -def get_elapsed_time(past_time):
    224      ptime = iso_to_localtime_tuple(past_time)
    225      now = datetime.now()
    226      delta = now - ptime
    227      timestr = time.strftime("%H:%M:%S", time.gmtime(delta.seconds))
    228      if delta.days != -1:
    229          timestr = "%d days, %s" % (delta.days, timestr)
    230      return timestr
    231
    232
    233 -def iso_to_unix_time(iso):
    234      dtup = iso_to_datetime_tuple(iso)
    235      secs = calendar.timegm(dtup.timetuple())
    236      return secs
    237
    238
    239 -def iso_to_javascript_timestamp(iso):
    240      """
    241      Convert dates to Javascript timestamps (number of milliseconds since
    242      January 1st 1970 UTC)
    243      """
    244      secs = iso_to_unix_time(iso)
    245      return secs * 1000
    246
    247
    248 -def iso_to_localtime_tuple(iso):
    249      secs = iso_to_unix_time(iso)
    250      t = time.mktime(time.localtime(secs))
    251      return datetime.fromtimestamp(t)
    252
    253
    254 -def permute(a):
    255      """
    256      Returns generator of all permutations of a
    257
    258      The following code is an in-place permutation of a given list, implemented
    259      as a generator. Since it only returns references to the list, the list
    260      should not be modified outside the generator. The solution is
    261      non-recursive, so uses low memory. Work well also with multiple copies of
    262      elements in the input list.
    263
    264      Retrieved from:
    265          http://stackoverflow.com/questions/104420/ \
    266          how-to-generate-all-permutations-of-a-list-in-python
    267      """
    268      a.sort()
    269      yield list(a)
    270      if len(a) <= 1:
    271          return
    272      first = 0
    273      last = len(a)
    274      while 1:
    275          i = last - 1
    276          while 1:
    277              i = i - 1
    278              if a[i] < a[i + 1]:
    279                  j = last - 1
    280                  while not (a[i] < a[j]):
    281                      j = j - 1
    282                  # swap the values
    283                  a[i], a[j] = a[j], a[i]
    284                  r = a[i + 1:last]
    285                  r.reverse()
    286                  a[i + 1:last] = r
    287                  yield list(a)
    288                  break
    289              if i == first:
    290                  a.reverse()
    291                  return
    292
    293
    294 -def has_required(programs):
    295      """
    296      Same as check_required but returns False if not all commands exist
    297      """
    298      try:
    299          return check_required(programs)
    300      except exception.CommandNotFound:
    301          return False
    302
    303
    304 -def check_required(programs):
    305      """
    306      Checks that all commands in the programs list exist. Returns
    307      True if all commands exist and raises exception.CommandNotFound if not.
    308      """
    309      for prog in programs:
    310          if not which(prog):
    311              raise exception.CommandNotFound(prog)
    312      return True
    313
    314
    315 -def which(program):
    316      """
    317      Returns the path to the program provided it exists and
    318      is on the system's PATH
    319
    320      retrieved from code snippet by Jay:
    321
    322      http://stackoverflow.com/questions/377017/ \
    323      test-if-executable-exists-in-python
    324      """
    325      def is_exe(fpath):
    326          return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
    327      fpath, fname = os.path.split(program)
    328      if fpath:
    329          if is_exe(program):
    330              return program
    331      else:
    332          for path in os.environ["PATH"].split(os.pathsep):
    333              exe_file = os.path.join(path, program)
    334              if is_exe(exe_file):
    335                  return exe_file
    336
    337
    338 -def tailf(filename):
    339      """
    340      Constantly displays the last lines in filename
    341      Similar to 'tail -f' unix command
    342      """
    343      #Set the filename and open the file
    344      file = open(filename, 'r')
    345
    346      #Find the size of the file and move to the end
    347      st_results = os.stat(filename)
    348      st_size = st_results[6]
    349      file.seek(st_size)
    350
    351      while True:
    352          where = file.tell()
    353          line = file.readline()
    354          if not line:
    355              time.sleep(1)
    356              file.seek(where)
    357              continue
    358          print line,  # already has newline
    359
    360
    361 -def v2fhelper(v, suff, version, weight):
    362      parts = v.split(suff)
    363      if 2 != len(parts):
    364          return v
    365      version[4] = weight
    366      version[5] = parts[1]
    367      return parts[0]
    368
    369
    370 -def version_to_float(v):
    371      # This code was written by Krzysztof Kowalczyk (http://blog.kowalczyk.info)
    372      # and is placed in public domain.
    373      """
    374      Convert a Mozilla-style version string into a floating-point number
    375      1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc
    376      """
    377      version = [
    378          0, 0, 0, 0,  # 4-part numerical revision
    379          4,  # Alpha, beta, RC or (default) final
    380          0,  # Alpha, beta, or RC version revision
    381          1   # Pre or (default) final
    382      ]
    383      parts = v.split("pre")
    384      if 2 == len(parts):
    385          version[6] = 0
    386          v = parts[0]
    387
    388      v = v2fhelper(v, "a",  version, 1)
    389      v = v2fhelper(v, "b",  version, 2)
    390      v = v2fhelper(v, "rc", version, 3)
    391
    392      parts = v.split(".")[:4]
    393      for (p, i) in zip(parts, range(len(parts))):
    394          version[i] = p
    395      ver = float(version[0])
    396      ver += float(version[1]) / 100.
    397      ver += float(version[2]) / 10000.
    398      ver += float(version[3]) / 1000000.
    399      ver += float(version[4]) / 100000000.
    400      ver += float(version[5]) / 10000000000.
    401      ver += float(version[6]) / 1000000000000.
    402      return ver
    403
    404
    405 -def program_version_greater(ver1, ver2):
    406      """
    407      Return True if ver1 > ver2 using semantics of comparing version
    408      numbers
    409      """
    410      v1f = version_to_float(ver1)
    411      v2f = version_to_float(ver2)
    412      return v1f > v2f
    413
    414
    415 -def test_version_to_float():
    416      assert program_version_greater("1", "0.9")
    417      assert program_version_greater("0.0.0.2", "0.0.0.1")
    418      assert program_version_greater("1.0", "0.9")
    419      assert program_version_greater("2.0.1", "2.0.0")
    420      assert program_version_greater("2.0.1", "2.0")
    421      assert program_version_greater("2.0.1", "2")
    422      assert program_version_greater("0.9.1", "0.9.0")
    423      assert program_version_greater("0.9.2", "0.9.1")
    424      assert program_version_greater("0.9.11", "0.9.2")
    425      assert program_version_greater("0.9.12", "0.9.11")
    426      assert program_version_greater("0.10", "0.9")
    427      assert program_version_greater("2.0", "2.0b35")
    428      assert program_version_greater("1.10.3", "1.10.3b3")
    429      assert program_version_greater("88", "88a12")
    430      assert program_version_greater("0.0.33", "0.0.33rc23")
    431      assert program_version_greater("0.91.2", "0.91.1")
    432      assert program_version_greater("0.9999", "0.91.1")
    433      assert program_version_greater("0.9999", "0.92")
    434      assert program_version_greater("0.91.10", "0.91.1")
    435      assert program_version_greater("0.92", "0.91.11")
    436      assert program_version_greater("0.92", "0.92b1")
    437      assert program_version_greater("0.9999", "0.92b3")
    438      print("All tests passed")
    439
    440
    441 -def get_arg_spec(func):
    442      """
    443      Convenience wrapper around inspect.getargspec
    444
    445      Returns a tuple whose first element is a list containing the names of all
    446      required arguments and whose second element is a list containing the names
    447      of all keyword (optional) arguments.
    448      """
    449      allargs, varargs, keywords, defaults = inspect.getargspec(func)
    450      if 'self' in allargs:
    451          allargs.remove('self')  # ignore self
    452      nargs = len(allargs)
    453      ndefaults = 0
    454      if defaults:
    455          ndefaults = len(defaults)
    456      nrequired = nargs - ndefaults
    457      args = allargs[:nrequired]
    458      kwargs = allargs[nrequired:]
    459      log.debug('nargs = %s' % nargs)
    460      log.debug('ndefaults = %s' % ndefaults)
    461      log.debug('nrequired = %s' % nrequired)
    462      log.debug('args = %s' % args)
    463      log.debug('kwargs = %s' % kwargs)
    464      log.debug('defaults = %s' % str(defaults))
    465      return args, kwargs
    466
    467
    468 -def chunk_list(ls, items=8):
    469      """
    470      iterate through 'chunks' of a list. final chunk consists of remaining
    471      elements if items does not divide len(ls) evenly.
    472
    473      items - size of 'chunks'
    474      """
    475      itms = []
    476      for i, v in enumerate(ls):
    477          if i >= items and i % items == 0:
    478              yield itms
    479              itms = [v]
    480          else:
    481              itms.append(v)
    482      if itms:
    483          yield itms
    484