starcluster.utils

Package starcluster :: Module utils |
[hide private]
[frames] | no frames]

# Source Code for Module starcluster.utils

      1  #!/usr/bin/env python
      2  """
      3  Utils module for StarCluster
      4  """
      5
      6  import os
      7  import re
      8  import time
      9  import types
     10  import calendar
     11  import urlparse
     12  from datetime import datetime
     13
     14  from starcluster import iptools
     15  from starcluster import exception
     16  from starcluster.logger import log
     17
     18
     19 -class AttributeDict(dict):
     20      """
     21      Subclass of dict that allows read-only attribute-like access to
     22      dictionary key/values
     23      """
     24 -    def __getattr__(self, name):
     25          try:
     26              return self.__getitem__(name)
     27          except KeyError:
     28              return super(AttributeDict, self).__getattribute__(name)
     29
     30
     31 -def print_timing(msg=None):
     32      """
     33      Decorator for printing execution time (in mins) of a function
     34      Optionally takes a user-friendly msg as argument. This msg will
     35      appear in the sentence "[msg] took XXX mins". If no msg is specified,
     36      msg will default to the decorated function's name. e.g:
     37
     38      @print_timing
     39      def myfunc():
     40          print 'hi'
     41      >>> myfunc()
     42      hi
     43      myfunc took 0.000 mins
     44
     45      @print_timing('My function')
     46      def myfunc():
     47          print 'hi'
     48      >>> myfunc()
     49      hi
     50      My function took 0.000 mins
     51      """
     52      if type(msg) == types.FunctionType:
     53          func = msg
     54
     55          def wrap_f(*arg, **kargs):
     56              """Raw timing function """
     57              time1 = time.time()
     58              res = func(*arg, **kargs)
     59              time2 = time.time()
     60              prefix = func.func_name
     61              log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0))
     62              return res
     63          return wrap_f
     64
     65      def wrap(func):
     66          def wrap_f(*arg, **kargs):
     67              """Raw timing function """
     68              time1 = time.time()
     69              res = func(*arg, **kargs)
     70              time2 = time.time()
     71              prefix = msg
     72              log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0))
     73              return res
     74          return wrap_f
     75      return wrap
     76
     77
     78 -def is_valid_device(dev):
     79      """
     80      Checks that dev matches the following regular expression:
     81      /dev/sd[a-z]$
     82      """
     83      regex = re.compile('/dev/sd[a-z]$')
     84      try:
     85          return regex.match(dev) is not None
     86      except TypeError:
     87          return False
     88
     89
     90 -def is_valid_partition(part):
     91      """
     92      Checks that part matches the following regular expression:
     93      /dev/sd[a-z][1-9][0-9]?$
     94      """
     95      regex = re.compile('/dev/sd[a-z][1-9][0-9]?$')
     96      try:
     97          return regex.match(part) is not None
     98      except TypeError:
     99          return False
    100
    101
    102 -def is_valid_bucket_name(bucket_name):
    103      """
    104      Check if bucket_name is a valid S3 bucket name (as defined by the AWS
    105      docs):
    106
    107      1. 3 <= len(bucket_name) <= 255
    108      2. all chars one of: a-z 0-9 .  _ -
    109      3. first char one of: a-z 0-9
    110      4. name must not be a valid ip
    111      """
    112      regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$')
    113      if not regex.match(bucket_name):
    114          return False
    115      if iptools.validate_ip(bucket_name):
    116          return False
    117      return True
    118
    119
    120 -def is_valid_image_name(image_name):
    121      """
    122      Check if image_name is a valid AWS image name (as defined by the AWS docs)
    123
    124      1. 3<= len(image_name) <=128
    125      2. all chars one of: a-z A-Z 0-9 ( ) . - / _
    126      """
    127      regex = re.compile('[\w\(\)\.\-\/_]{3,128}$')
    128      try:
    129          return regex.match(image_name) is not None
    130      except TypeError:
    131          return False
    132
    133
    134 -def make_one_liner(script):
    135      """
    136      Returns command to execute python script as a one-line python program
    137
    138      e.g.
    139
    140          import os
    141          script = '''
    142          import os
    143          print os.path.exists('hi')
    144          '''
    145          os.system(make_one_liner(script))
    146
    147      Will print out:
    148
    149          <module 'os' from ...>
    150          False
    151      """
    152      return 'python -c "%s"' % script.strip().replace('\n', ';')
    153
    154
    155 -def is_url(url):
    156      """
    157      Returns True if the provided string is a valid url
    158      """
    159      try:
    160          parts = urlparse.urlparse(url)
    161          scheme = parts[0]
    162          netloc = parts[1]
    163          if scheme and netloc:
    164              return True
    165          else:
    166              return False
    167      except:
    168          return False
    169
    170
    171 -def is_iso_time(iso):
    172      """
    173      Returns True if provided time can be parsed in iso format
    174      to a datetime tuple
    175      """
    176      try:
    177          iso_to_datetime_tuple(iso)
    178          return True
    179      except ValueError:
    180          return False
    181
    182
    183 -def iso_to_datetime_tuple(iso):
    184      """
    185      Converts an iso time string to a datetime tuple
    186      """
    187      #remove timezone
    188      iso = iso.split('.')[0]
    189      try:
    190          return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S")
    191      except AttributeError:
    192          # python2.4 datetime module doesnt have strptime
    193          return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
    194
    195
    196 -def datetime_tuple_to_iso(tup):
    197      """
    198      Converts a datetime tuple to iso time string
    199      """
    200      iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S")
    201      return iso
    202
    203
    204 -def get_elapsed_time(past_time):
    205      ptime = iso_to_localtime_tuple(past_time)
    206      now = datetime.now()
    207      delta = now - ptime
    208      return time.strftime("%H:%M:%S", time.gmtime(delta.seconds))
    209
    210
    211 -def iso_to_localtime_tuple(iso):
    212      dtup = iso_to_datetime_tuple(iso)
    213      secs = calendar.timegm(dtup.timetuple())
    214      t = time.mktime(time.localtime(secs))
    215      return datetime.fromtimestamp(t)
    216
    217  try:
    218      import IPython.Shell
    219      ipy_shell = IPython.Shell.IPShellEmbed(argv=[])
    220  except ImportError:
    221
    222 -    def ipy_shell(local_ns=None):
    223          log.error("Unable to load IPython.")
    224          log.error("Please check that IPython is installed and working.")
    225          log.error("If not, you can install it via: easy_install ipython")
    226
    227
    228 -def permute(a):
    229      """
    230      Returns generator of all permutations of a
    231
    232      The following code is an in-place permutation of a given list, implemented
    233      as a generator. Since it only returns references to the list, the list
    234      should not be modified outside the generator. The solution is
    235      non-recursive, so uses low memory. Work well also with multiple copies of
    236      elements in the input list.
    237
    238      Retrieved from:
    239          http://stackoverflow.com/questions/104420/ \
    240          how-to-generate-all-permutations-of-a-list-in-python
    241      """
    242      a.sort()
    243      yield list(a)
    244      if len(a) <= 1:
    245          return
    246      first = 0
    247      last = len(a)
    248      while 1:
    249          i = last - 1
    250          while 1:
    251              i = i - 1
    252              if a[i] < a[i + 1]:
    253                  j = last - 1
    254                  while not (a[i] < a[j]):
    255                      j = j - 1
    256                  # swap the values
    257                  a[i], a[j] = a[j], a[i]
    258                  r = a[i + 1:last]
    259                  r.reverse()
    260                  a[i + 1:last] = r
    261                  yield list(a)
    262                  break
    263              if i == first:
    264                  a.reverse()
    265                  return
    266
    267
    268 -def has_required(programs):
    269      """
    270      Same as check_required but returns False if not all commands exist
    271      """
    272      try:
    273          return check_required(programs)
    274      except exception.CommandNotFound:
    275          return False
    276
    277
    278 -def check_required(programs):
    279      """
    280      Checks that all commands in the programs list exist. Returns
    281      True if all commands exist and raises exception.CommandNotFound if not.
    282      """
    283      for prog in programs:
    284          if not which(prog):
    285              raise exception.CommandNotFound(prog)
    286      return True
    287
    288
    289 -def which(program):
    290      """
    291      Returns the path to the program provided it exists and
    292      is on the system's PATH
    293
    294      retrieved from code snippet by Jay:
    295
    296      http://stackoverflow.com/questions/377017/ \
    297      test-if-executable-exists-in-python
    298      """
    299      def is_exe(fpath):
    300          return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
    301      fpath, fname = os.path.split(program)
    302      if fpath:
    303          if is_exe(program):
    304              return program
    305      else:
    306          for path in os.environ["PATH"].split(os.pathsep):
    307              exe_file = os.path.join(path, program)
    308              if is_exe(exe_file):
    309                  return exe_file
    310
    311
    312 -def tailf(filename):
    313      """
    314      Constantly displays the last lines in filename
    315      Similar to 'tail -f' unix command
    316      """
    317      #Set the filename and open the file
    318      file = open(filename, 'r')
    319
    320      #Find the size of the file and move to the end
    321      st_results = os.stat(filename)
    322      st_size = st_results[6]
    323      file.seek(st_size)
    324
    325      while True:
    326          where = file.tell()
    327          line = file.readline()
    328          if not line:
    329              time.sleep(1)
    330              file.seek(where)
    331              continue
    332          print line,  # already has newline
    333
    334
    335 -def v2fhelper(v, suff, version, weight):
    336      parts = v.split(suff)
    337      if 2 != len(parts):
    338          return v
    339      version[4] = weight
    340      version[5] = parts[1]
    341      return parts[0]
    342
    343
    344 -def version_to_float(v):
    345      # This code was written by Krzysztof Kowalczyk (http://blog.kowalczyk.info)
    346      # and is placed in public domain.
    347      """
    348      Convert a Mozilla-style version string into a floating-point number
    349      1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc
    350      """
    351      version = [
    352          0, 0, 0, 0,  # 4-part numerical revision
    353          4,  # Alpha, beta, RC or (default) final
    354          0,  # Alpha, beta, or RC version revision
    355          1   # Pre or (default) final
    356      ]
    357      parts = v.split("pre")
    358      if 2 == len(parts):
    359          version[6] = 0
    360          v = parts[0]
    361
    362      v = v2fhelper(v, "a",  version, 1)
    363      v = v2fhelper(v, "b",  version, 2)
    364      v = v2fhelper(v, "rc", version, 3)
    365
    366      parts = v.split(".")[:4]
    367      for (p, i) in zip(parts, range(len(parts))):
    368          version[i] = p
    369      ver = float(version[0])
    370      ver += float(version[1]) / 100.
    371      ver += float(version[2]) / 10000.
    372      ver += float(version[3]) / 1000000.
    373      ver += float(version[4]) / 100000000.
    374      ver += float(version[5]) / 10000000000.
    375      ver += float(version[6]) / 1000000000000.
    376      return ver
    377
    378
    379 -def program_version_greater(ver1, ver2):
    380      """
    381      Return True if ver1 > ver2 using semantics of comparing version
    382      numbers
    383      """
    384      v1f = version_to_float(ver1)
    385      v2f = version_to_float(ver2)
    386      return v1f > v2f
    387
    388
    389 -def test_version_to_float():
    390      assert program_version_greater("1", "0.9")
    391      assert program_version_greater("0.0.0.2", "0.0.0.1")
    392      assert program_version_greater("1.0", "0.9")
    393      assert program_version_greater("2.0.1", "2.0.0")
    394      assert program_version_greater("2.0.1", "2.0")
    395      assert program_version_greater("2.0.1", "2")
    396      assert program_version_greater("0.9.1", "0.9.0")
    397      assert program_version_greater("0.9.2", "0.9.1")
    398      assert program_version_greater("0.9.11", "0.9.2")
    399      assert program_version_greater("0.9.12", "0.9.11")
    400      assert program_version_greater("0.10", "0.9")
    401      assert program_version_greater("2.0", "2.0b35")
    402      assert program_version_greater("1.10.3", "1.10.3b3")
    403      assert program_version_greater("88", "88a12")
    404      assert program_version_greater("0.0.33", "0.0.33rc23")
    405      assert program_version_greater("0.91.2", "0.91.1")
    406      assert program_version_greater("0.9999", "0.91.1")
    407      assert program_version_greater("0.9999", "0.92")
    408      assert program_version_greater("0.91.10", "0.91.1")
    409      assert program_version_greater("0.92", "0.91.11")
    410      assert program_version_greater("0.92", "0.92b1")
    411      assert program_version_greater("0.9999", "0.92b3")
    412      print("All tests passed")
    413