starcluster.config

Package starcluster :: Module config |
[hide private]
[frames] | no frames]

# Source Code for Module starcluster.config

      1  #!/usr/bin/env python
      2  import os
      3  import urllib
      4  import StringIO
      5  import ConfigParser
      6
      7  from starcluster import utils
      8  from starcluster import static
      9  from starcluster import cluster
     10  from starcluster import awsutils
     11  from starcluster import exception
     12  from starcluster.cluster import Cluster
     13  from starcluster.utils import AttributeDict
     14
     15  from starcluster.logger import log
     16
     17  DEBUG_CONFIG = False
     18
     19
     20 -def get_easy_s3(config_file=None, cache=False):
     21      """
     22      Factory for EasyS3 class that attempts to load AWS credentials from
     23      the StarCluster config file. Returns an EasyS3 object if
     24      successful.
     25      """
     26      cfg = get_config(config_file, cache)
     27      return cfg.get_easy_s3()
     28
     29
     30 -def get_easy_ec2(config_file=None, cache=False):
     31      """
     32      Factory for EasyEC2 class that attempts to load AWS credentials from
     33      the StarCluster config file. Returns an EasyEC2 object if
     34      successful.
     35      """
     36      cfg = get_config(config_file, cache)
     37      return cfg.get_easy_ec2()
     38
     39
     40 -def get_cluster_manager(config_file=None, cache=False):
     41      """
     42      Factory for ClusterManager class that attempts to load AWS credentials from
     43      the StarCluster config file. Returns a ClusterManager object if successful
     44      """
     45      cfg = get_config(config_file, cache)
     46      return cfg.get_cluster_manager()
     47
     48
     49 -def get_config(config_file=None, cache=False):
     50      """Factory for StarClusterConfig object"""
     51      return StarClusterConfig(config_file, cache)
     52
     53
     54 -class StarClusterConfig(object):
     55      """
     56      Loads StarCluster configuration settings defined in config_file
     57      which defaults to ~/.starclustercfg
     58
     59      Settings are available as follows:
     60
     61      cfg = StarClusterConfig()
     62      or
     63      cfg = StarClusterConfig('/path/to/my/config.cfg')
     64      cfg.load()
     65      aws_info = cfg.aws
     66      cluster_cfg = cfg.clusters['mycluster']
     67      key_cfg = cfg.keys['gsg-keypair']
     68      print cluster_cfg
     69      """
     70
     71      global_settings = static.GLOBAL_SETTINGS
     72      aws_settings = static.AWS_SETTINGS
     73      key_settings = static.KEY_SETTINGS
     74      volume_settings = static.EBS_VOLUME_SETTINGS
     75      plugin_settings = static.PLUGIN_SETTINGS
     76      cluster_settings = static.CLUSTER_SETTINGS
     77      permission_settings = static.PERMISSION_SETTINGS
     78
     79      # until i can find a way to query AWS for instance types...
     80      instance_types = static.INSTANCE_TYPES
     81
     82 -    def __init__(self, config_file=None, cache=False):
     83          self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE
     84          self.cfg_file = os.path.expanduser(self.cfg_file)
     85          self.cfg_file = os.path.expandvars(self.cfg_file)
     86          self.type_validators = {
     87              int: self._get_int,
     88              float: self._get_float,
     89              str: self._get_string,
     90              bool: self._get_bool,
     91              list: self._get_list,
     92          }
     93          self._config = None
     94          self.globals = AttributeDict()
     95          self.aws = AttributeDict()
     96          self.clusters = AttributeDict()
     97          self.keys = AttributeDict()
     98          self.vols = AttributeDict()
     99          self.plugins = AttributeDict()
    100          self.permissions = AttributeDict()
    101          self.cache = cache
    102
    103 -    def __repr__(self):
    104          return "<StarClusterConfig: %s>" % self.cfg_file
    105
    106 -    def _get_urlfp(self, url):
    107          log.debug("Loading url: %s" % url)
    108          try:
    109              fp = urllib.urlopen(url)
    110              if fp.getcode() == 404:
    111                  raise exception.ConfigError("url %s does not exist" % url)
    112              fp.name = url
    113              return fp
    114          except IOError, e:
    115              raise exception.ConfigError(
    116                  "error loading config from url %s\n%s" % (url, e))
    117
    118 -    def _get_fp(self, cfg_file):
    119          log.debug("Loading file: %s" % cfg_file)
    120          if os.path.exists(cfg_file):
    121              if not os.path.isfile(cfg_file):
    122                  raise exception.ConfigError(
    123                      'config %s exists but is not a regular file' % cfg_file)
    124          else:
    125              raise exception.ConfigNotFound("config file %s does not exist\n" %
    126                                             cfg_file, cfg_file)
    127          return open(cfg_file)
    128
    129 -    def _get_cfg_fp(self, cfg_file=None):
    130          cfg = cfg_file or self.cfg_file
    131          if utils.is_url(cfg):
    132              return self._get_urlfp(cfg)
    133          else:
    134              return self._get_fp(cfg)
    135
    136 -    def _get_bool(self, config, section, option):
    137          try:
    138              opt = config.getboolean(section, option)
    139              return opt
    140          except ConfigParser.NoSectionError:
    141              pass
    142          except ConfigParser.NoOptionError:
    143              pass
    144          except ValueError:
    145              raise exception.ConfigError(
    146                  "Expected True/False value for setting %s in section [%s]" %
    147                  (option, section))
    148
    149 -    def _get_int(self, config, section, option):
    150          try:
    151              opt = config.getint(section, option)
    152              return opt
    153          except ConfigParser.NoSectionError:
    154              pass
    155          except ConfigParser.NoOptionError:
    156              pass
    157          except ValueError:
    158              raise exception.ConfigError(
    159                  "Expected integer value for setting %s in section [%s]" %
    160                  (option, section))
    161
    162 -    def _get_float(self, config, section, option):
    163          try:
    164              opt = config.getfloat(section, option)
    165              return opt
    166          except ConfigParser.NoSectionError:
    167              pass
    168          except ConfigParser.NoOptionError:
    169              pass
    170          except ValueError:
    171              raise exception.ConfigError(
    172                  "Expected float value for setting %s in section [%s]" %
    173                  (option, section))
    174
    175 -    def _get_string(self, config, section, option):
    176          try:
    177              opt = config.get(section, option)
    178              return opt
    179          except ConfigParser.NoSectionError:
    180              pass
    181          except ConfigParser.NoOptionError:
    182              pass
    183
    184 -    def _get_list(self, config, section, option):
    185          val = self._get_string(config, section, option)
    186          if val:
    187              val = [v.strip() for v in val.split(',')]
    188          return val
    189
    190 -    def __load_config(self):
    191          """
    192          Populates self._config with a new ConfigParser instance
    193          """
    194          cfg = self._get_cfg_fp()
    195          try:
    196              cp = ConfigParser.ConfigParser()
    197              cp.readfp(cfg)
    198              self._config = cp
    199              try:
    200                  self.globals = self._load_section('global',
    201                                                    self.global_settings)
    202                  includes = self.globals.get('include')
    203                  if not includes:
    204                      return cp
    205                  mashup = StringIO.StringIO()
    206                  cfg = self._get_cfg_fp()
    207                  mashup.write(cfg.read() + '\n')
    208                  for include in includes:
    209                      include = os.path.expanduser(include)
    210                      include = os.path.expandvars(include)
    211                      try:
    212                          contents = self._get_cfg_fp(include).read()
    213                          mashup.write(contents + '\n')
    214                      except exception.ConfigNotFound:
    215                          raise exception.ConfigError("include %s not found" %
    216                                                      include)
    217                  mashup.seek(0)
    218                  cp = ConfigParser.ConfigParser()
    219                  cp.readfp(mashup)
    220                  self._config = cp
    221              except exception.ConfigSectionMissing:
    222                  pass
    223              return cp
    224          except ConfigParser.MissingSectionHeaderError:
    225              raise exception.ConfigHasNoSections(cfg.name)
    226          except ConfigParser.ParsingError, e:
    227              raise exception.ConfigError(e)
    228
    229 -    def reload(self):
    230          """
    231          Reloads the configuration file
    232          """
    233          self.__load_config()
    234          return self.load()
    235
    236      @property
    237 -    def config(self):
    238          if self._config is None:
    239              self._config = self.__load_config()
    240          return self._config
    241
    242 -    def _load_settings(self, section_name, settings, store,
    243                         filter_settings=True):
    244          """
    245          Load section settings into a dictionary
    246          """
    247          section = self.config._sections.get(section_name)
    248          if not section:
    249              raise exception.ConfigSectionMissing(
    250                  'Missing section %s in config' % section_name)
    251          store.update(section)
    252          section_conf = store
    253          for setting in settings:
    254              requirements = settings[setting]
    255              func, required, default, options, callback = requirements
    256              func = self.type_validators.get(func)
    257              value = func(self.config, section_name, setting)
    258              if value is not None:
    259                  if options and not value in options:
    260                      raise exception.ConfigError(
    261                          '"%s" setting in section "%s" must be one of: %s' %
    262                          (setting, section_name,
    263                           ', '.join([str(o) for o in options])))
    264                  if callback:
    265                      value = callback(value)
    266                  section_conf[setting] = value
    267          if filter_settings:
    268              for key in store.keys():
    269                  if key not in settings and key != '__name__':
    270                      store.pop(key)
    271
    272 -    def _check_required(self, section_name, settings, store):
    273          """
    274          Check that all required settings were specified in the config.
    275          Raises ConfigError otherwise.
    276
    277          Note that if a setting specified has required=True and
    278          default is not None then this method will not raise an error
    279          because a default was given. In short, if a setting is required
    280          you must provide None as the 'default' value.
    281          """
    282          section_conf = store
    283          for setting in settings:
    284              requirements = settings[setting]
    285              required = requirements[1]
    286              value = section_conf.get(setting)
    287              if value is None and required:
    288                  raise exception.ConfigError(
    289                      'missing required option %s in section "%s"' %
    290                      (setting, section_name))
    291
    292 -    def _load_defaults(self, settings, store):
    293          """
    294          Sets the default for each setting in settings regardless of whether
    295          the setting was specified in the config or not.
    296          """
    297          section_conf = store
    298          for setting in settings:
    299              default = settings[setting][2]
    300              if section_conf.get(setting) is None:
    301                  if DEBUG_CONFIG:
    302                      log.debug('%s setting not specified. Defaulting to %s' % \
    303                                (setting, default))
    304                  section_conf[setting] = default
    305
    306 -    def _load_extends_settings(self, section_name, store):
    307          """
    308          Loads all settings from other template(s) specified by a section's
    309          'extends' setting.
    310
    311          This method walks a dependency tree of sections from bottom up. Each
    312          step is a group of settings for a section in the form of a dictionary.
    313          A 'master' dictionary is updated with the settings at each step. This
    314          causes the next group of settings to override the previous, and so on.
    315          The 'section_name' settings are at the top of the dep tree.
    316          """
    317          section = store[section_name]
    318          extends = section.get('extends')
    319          if extends is None:
    320              return
    321          if DEBUG_CONFIG:
    322              log.debug('%s extends %s' % (section_name, extends))
    323          extensions = [section]
    324          while True:
    325              extends = section.get('extends', None)
    326              if not extends:
    327                  break
    328              try:
    329                  section = store[extends]
    330                  if section in extensions:
    331                      exts = ', '.join([self._get_section_name(x['__name__'])
    332                                        for x in extensions])
    333                      raise exception.ConfigError(
    334                          ("Cyclical dependency between sections %s. " % exts) \
    335                          + "Check your extends settings.")
    336                  extensions.insert(0, section)
    337              except KeyError:
    338                  raise exception.ConfigError(
    339                      "%s can't extend non-existent section %s" % \
    340                      (section_name, extends))
    341          transform = AttributeDict()
    342          for extension in extensions:
    343              transform.update(extension)
    344          store[section_name] = transform
    345
    346 -    def _load_keypairs(self, store):
    347          cluster_section = store
    348          keyname = cluster_section.get('keyname')
    349          if not keyname:
    350              return
    351          keypair = self.keys.get(keyname)
    352          if keypair is None:
    353              raise exception.ConfigError(
    354                  "keypair '%s' not defined in config" % keyname)
    355          cluster_section['keyname'] = keyname
    356          cluster_section['key_location'] = keypair.get('key_location')
    357
    358 -    def _load_volumes(self, store):
    359          cluster_section = store
    360          volumes = cluster_section.get('volumes')
    361          if not volumes or isinstance(volumes, AttributeDict):
    362              return
    363          vols = AttributeDict()
    364          cluster_section['volumes'] = vols
    365          for volume in volumes:
    366              if not volume in self.vols:
    367                  raise exception.ConfigError(
    368                      "volume '%s' not defined in config" % volume)
    369              vol = self.vols.get(volume)
    370              vols[volume] = vol
    371
    372 -    def _load_plugins(self, store):
    373          cluster_section = store
    374          plugins = cluster_section.get('plugins')
    375          if not plugins or isinstance(plugins[0], AttributeDict):
    376              return
    377          plugs = []
    378          cluster_section['plugins'] = plugs
    379          for plugin in plugins:
    380              if plugin in self.plugins:
    381                  p = self.plugins.get(plugin)
    382                  p['__name__'] = p['__name__'].split()[-1]
    383                  plugs.append(p)
    384              else:
    385                  raise exception.ConfigError(
    386                      "plugin '%s' not defined in config" % plugin)
    387
    388 -    def _load_permissions(self, store):
    389          cluster_section = store
    390          permissions = cluster_section.get('permissions')
    391          if not permissions or isinstance(permissions, AttributeDict):
    392              return
    393          perms = AttributeDict()
    394          cluster_section['permissions'] = perms
    395          for perm in permissions:
    396              if perm in self.permissions:
    397                  p = self.permissions.get(perm)
    398                  p['__name__'] = p['__name__'].split()[-1]
    399                  perms[perm] = p
    400              else:
    401                  raise exception.ConfigError(
    402                      "permission '%s' not defined in config" % perm)
    403
    404 -    def _load_instance_types(self, store):
    405          cluster_section = store
    406          instance_types = cluster_section.get('node_instance_type')
    407          if isinstance(instance_types, basestring):
    408              return
    409          itypes = []
    410          cluster_section['node_instance_types'] = itypes
    411          total_num_nodes = 0
    412          choices_string = ', '.join(static.INSTANCE_TYPES.keys())
    413          try:
    414              default_instance_type = instance_types[-1]
    415              if not default_instance_type in static.INSTANCE_TYPES:
    416                  raise exception.ConfigError(
    417                      ("invalid node_instance_type specified: '%s'\n" +
    418                       "must be one of: %s") %
    419                      (default_instance_type, choices_string))
    420          except IndexError:
    421              default_instance_type = None
    422          cluster_section['node_instance_type'] = default_instance_type
    423          for type_spec in instance_types[:-1]:
    424              type_spec = type_spec.split(':')
    425              if len(type_spec) > 3:
    426                  raise exception.ConfigError(
    427                      "invalid node_instance_type item specified: %s" % \
    428                      type_spec)
    429              itype = type_spec[0]
    430              itype_image = None
    431              itype_num = 1
    432              if not itype in static.INSTANCE_TYPES:
    433                  raise exception.ConfigError(
    434                      ("invalid type specified (%s) in node_instance_type " + \
    435                       "item: '%s'\nmust be one of: %s") %
    436                      (itype, type_spec, choices_string))
    437              if len(type_spec) == 2:
    438                  itype, next_var = type_spec
    439                  try:
    440                      itype_num = int(next_var)
    441                  except (TypeError, ValueError):
    442                      itype_image = next_var
    443              elif len(type_spec) == 3:
    444                  itype, itype_image, itype_num = type_spec
    445              try:
    446                  itype_num = int(itype_num)
    447                  if itype_num < 1:
    448                      raise TypeError
    449                  total_num_nodes += itype_num
    450              except (ValueError, TypeError):
    451                  raise exception.ConfigError(
    452                      ("number of instances (%s) of type '%s' must " + \
    453                       "be an integer > 1") % (itype_num, itype))
    454              itype_dic = AttributeDict(size=itype_num, image=itype_image,
    455                                        type=itype)
    456              itypes.append(itype_dic)
    457
    458 -    def _load_section(self, section_name, section_settings,
    459                        filter_settings=True):
    460          """
    461          Returns a dictionary containing all section_settings for a given
    462          section_name by first loading the settings in the config, loading
    463          the defaults for all settings not specified, and then checking
    464          that all required options have been specified
    465          """
    466          store = AttributeDict()
    467          self._load_settings(section_name, section_settings, store,
    468                              filter_settings)
    469          self._load_defaults(section_settings, store)
    470          self._check_required(section_name, section_settings, store)
    471          return store
    472
    473 -    def _get_section_name(self, section):
    474          """
    475          Returns section name minus prefix
    476          e.g.
    477          $ print self._get_section('cluster smallcluster')
    478          $ smallcluster
    479          """
    480          return section.split()[1]
    481
    482 -    def _get_sections(self, section_prefix):
    483          """
    484          Returns all sections starting with section_prefix
    485          e.g.
    486          $ print self._get_sections('cluster')
    487          $ ['cluster smallcluster', 'cluster mediumcluster', ..]
    488          """
    489          return [s for s in self.config.sections() if
    490                  s.startswith(section_prefix)]
    491
    492 -    def _load_sections(self, section_prefix, section_settings,
    493                         filter_settings=True):
    494          """
    495          Loads all sections starting with section_prefix and returns a
    496          dictionary containing the name and dictionary of settings for each
    497          section.
    498          keys --> section name (as returned by self._get_section_name)
    499          values --> dictionary of settings for a given section
    500
    501          e.g.
    502          $ print self._load_sections('volumes', self.plugin_settings)
    503
    504          {'myvol': {'__name__': 'volume myvol',
    505                      'device': None,
    506                      'mount_path': '/home',
    507                      'partition': 1,
    508                      'volume_id': 'vol-999999'},
    509           'myvol2': {'__name__': 'volume myvol2',
    510                         'device': None,
    511                         'mount_path': '/myvol2',
    512                         'partition': 1,
    513                         'volume_id': 'vol-999999'},
    514          """
    515          sections = self._get_sections(section_prefix)
    516          sections_store = AttributeDict()
    517          for sec in sections:
    518              name = self._get_section_name(sec)
    519              sections_store[name] = self._load_section(sec, section_settings,
    520                                                        filter_settings)
    521          return sections_store
    522
    523 -    def _load_cluster_sections(self, cluster_sections):
    524          """
    525          Loads all cluster sections. Similar to _load_sections but also handles
    526          populating specified keypair,volume,plugins,permissions,etc settings
    527          """
    528          clusters = cluster_sections
    529          cluster_store = AttributeDict()
    530          for cl in clusters:
    531              name = self._get_section_name(cl)
    532              cluster_store[name] = AttributeDict()
    533              self._load_settings(cl, self.cluster_settings, cluster_store[name])
    534          for cl in clusters:
    535              name = self._get_section_name(cl)
    536              self._load_extends_settings(name, cluster_store)
    537              self._load_defaults(self.cluster_settings, cluster_store[name])
    538              self._load_keypairs(cluster_store[name])
    539              self._load_volumes(cluster_store[name])
    540              self._load_plugins(cluster_store[name])
    541              self._load_permissions(cluster_store[name])
    542              self._load_instance_types(cluster_store[name])
    543              self._check_required(cl, self.cluster_settings,
    544                                   cluster_store[name])
    545          return cluster_store
    546
    547 -    def load(self):
    548          """
    549          Populate this config object from the StarCluster config
    550          """
    551          log.debug('Loading config')
    552          try:
    553              self.globals = self._load_section('global', self.global_settings)
    554          except exception.ConfigSectionMissing:
    555              pass
    556          try:
    557              self.aws = self._load_section('aws info', self.aws_settings)
    558          except exception.ConfigSectionMissing:
    559              log.warn("no [aws info] section found in config")
    560              log.warn("attempting to load credentials from environment...")
    561              self.aws.update(self.get_aws_from_environ())
    562          self.keys = self._load_sections('key', self.key_settings)
    563          self.vols = self._load_sections('volume', self.volume_settings)
    564          self.plugins = self._load_sections('plugin', self.plugin_settings,
    565                                             filter_settings=False)
    566          self.permissions = self._load_sections('permission',
    567                                                 self.permission_settings)
    568          sections = self._get_sections('cluster')
    569          self.clusters = self._load_cluster_sections(sections)
    570          return self
    571
    572 -    def get_aws_from_environ(self):
    573          """
    574          Returns AWS credentials defined in the user's shell
    575          environment.
    576          """
    577          awscreds = {}
    578          for key in static.AWS_SETTINGS:
    579              if key.upper() in os.environ:
    580                  awscreds[key] = os.environ.get(key.upper())
    581              elif key in os.environ:
    582                  awscreds[key] = os.environ.get(key)
    583          return awscreds
    584
    585 -    def get_aws_credentials(self):
    586          """
    587          Returns AWS credentials defined in the configuration
    588          file. Defining any of the AWS settings in the environment
    589          overrides the configuration file.
    590          """
    591          # first override with environment settings if they exist
    592          self.aws.update(self.get_aws_from_environ())
    593          return self.aws
    594
    595 -    def get_cluster_names(self):
    596          return self.clusters
    597
    598 -    def get_cluster_template(self, template_name, tag_name=None,
    599                               ec2_conn=None):
    600          """
    601          Returns Cluster instance configured with the settings in the
    602          config file.
    603
    604          template_name is the name of a cluster section defined in the config
    605
    606          tag_name, if specified, will be passed to Cluster instance
    607          as cluster_tag
    608          """
    609          try:
    610              kwargs = {}
    611              if tag_name:
    612                  kwargs.update(dict(cluster_tag=tag_name))
    613              kwargs.update(self.clusters[template_name])
    614              if not ec2_conn:
    615                  ec2_conn = self.get_easy_ec2()
    616              clust = Cluster(ec2_conn, **kwargs)
    617              return clust
    618          except KeyError:
    619              raise exception.ClusterTemplateDoesNotExist(template_name)
    620
    621 -    def get_default_cluster_template(self):
    622          """
    623          Returns the default_template specified in the [global] section
    624          of the config. Raises NoDefaultTemplateFound if no default cluster
    625          template has been specified in the config.
    626          """
    627          default = self.globals.get('default_template')
    628          if not default:
    629              raise exception.NoDefaultTemplateFound(
    630                  options=self.clusters.keys())
    631          if not default in self.clusters:
    632              raise exception.ClusterTemplateDoesNotExist(default)
    633          return default
    634
    635 -    def get_clusters(self):
    636          clusters = []
    637          for cl in self.clusters:
    638              cl.append(self.get_cluster_template(cluster))
    639          return clusters
    640
    641 -    def get_plugin(self, plugin):
    642          try:
    643              return self.plugins[plugin]
    644          except KeyError:
    645              raise exception.PluginNotFound(plugin)
    646
    647 -    def get_key(self, keyname):
    648          try:
    649              return self.keys[keyname]
    650          except KeyError:
    651              raise exception.KeyNotFound(keyname)
    652
    653 -    def get_easy_s3(self):
    654          """
    655          Factory for EasyEC2 class that attempts to load AWS credentials from
    656          the StarCluster config file. Returns an EasyS3 object if
    657          successful.
    658          """
    659          aws = self.get_aws_credentials()
    660          try:
    661              s3 = awsutils.EasyS3(**aws)
    662              return s3
    663          except TypeError:
    664              raise exception.ConfigError("no aws credentials found")
    665
    666 -    def get_easy_ec2(self):
    667          """
    668          Factory for EasyEC2 class that attempts to load AWS credentials from
    669          the StarCluster config file. Returns an EasyEC2 object if
    670          successful.
    671          """
    672          aws = self.get_aws_credentials()
    673          try:
    674              ec2 = awsutils.EasyEC2(**aws)
    675              return ec2
    676          except TypeError:
    677              raise exception.ConfigError("no aws credentials found")
    678
    679 -    def get_cluster_manager(self):
    680          ec2 = self.get_easy_ec2()
    681          return cluster.ClusterManager(self, ec2)
    682
    683
    684  if __name__ == "__main__":
    685      from pprint import pprint
    686      cfg = StarClusterConfig().load()
    687      pprint(cfg.aws)
    688      pprint(cfg.clusters)
    689      pprint(cfg.keys)
    690      pprint(cfg.vols)
    691