starcluster.config

Package starcluster :: Module config |
[hide private]
[frames] | no frames]

# Source Code for Module starcluster.config

      1  #!/usr/bin/env python
      2  import os
      3  import urllib
      4  import ConfigParser
      5
      6  from starcluster import utils
      7  from starcluster import static
      8  from starcluster import awsutils
      9  from starcluster import exception
     10  from starcluster.cluster import Cluster
     11  from starcluster.utils import AttributeDict
     12
     13  from starcluster.logger import log
     14
     15  DEBUG_CONFIG = False
     16
     17
     18 -def get_easy_s3(config_file=None, cache=False):
     19      """
     20      Factory for EasyS3 class that attempts to load AWS credentials from
     21      the StarCluster config file. Returns an EasyS3 object if
     22      successful.
     23      """
     24      cfg = get_config(config_file, cache)
     25      return cfg.get_easy_s3()
     26
     27
     28 -def get_easy_ec2(config_file=None, cache=False):
     29      """
     30      Factory for EasyEC2 class that attempts to load AWS credentials from
     31      the StarCluster config file. Returns an EasyEC2 object if
     32      successful.
     33      """
     34      cfg = get_config(config_file, cache)
     35      return cfg.get_easy_ec2()
     36
     37
     38 -def get_config(config_file=None, cache=False):
     39      """Factory for StarClusterConfig object"""
     40      return StarClusterConfig(config_file, cache)
     41
     42
     43 -class StarClusterConfig(object):
     44      """
     45      Loads StarCluster configuration settings defined in config_file
     46      which defaults to ~/.starclustercfg
     47
     48      Settings are available as follows:
     49
     50      cfg = StarClusterConfig()
     51      or
     52      cfg = StarClusterConfig('/path/to/my/config.cfg')
     53      cfg.load()
     54      aws_info = cfg.aws
     55      cluster_cfg = cfg.clusters['mycluster']
     56      key_cfg = cfg.keys['gsg-keypair']
     57      print cluster_cfg
     58      """
     59
     60      global_settings = static.GLOBAL_SETTINGS
     61      aws_settings = static.AWS_SETTINGS
     62      key_settings = static.KEY_SETTINGS
     63      volume_settings = static.EBS_VOLUME_SETTINGS
     64      plugin_settings = static.PLUGIN_SETTINGS
     65      cluster_settings = static.CLUSTER_SETTINGS
     66      permission_settings = static.PERMISSION_SETTINGS
     67
     68      # until i can find a way to query AWS for instance types...
     69      instance_types = static.INSTANCE_TYPES
     70
     71 -    def __init__(self, config_file=None, cache=False):
     72          self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE
     73          self.type_validators = {
     74              int: self._get_int,
     75              float: self._get_float,
     76              str: self._get_string,
     77              bool: self._get_bool,
     78              list: self._get_list,
     79          }
     80          self._config = None
     81          self.globals = AttributeDict()
     82          self.aws = AttributeDict()
     83          self.clusters = AttributeDict()
     84          self.keys = AttributeDict()
     85          self.vols = AttributeDict()
     86          self.plugins = AttributeDict()
     87          self.permissions = AttributeDict()
     88          self.cache = cache
     89
     90 -    def __repr__(self):
     91          return "<StarClusterConfig: %s>" % self.cfg_file
     92
     93 -    def _get_urlfp(self, url):
     94          log.debug("Loading url: %s" % url)
     95          try:
     96              fp = urllib.urlopen(url)
     97              if fp.getcode() == 404:
     98                  raise exception.ConfigError("url %s does not exist" % url)
     99              fp.name = url
    100              return fp
    101          except IOError, e:
    102              raise exception.ConfigError(
    103                  "error loading config from url %s\n%s" % (url, e))
    104
    105 -    def _get_fp(self, cfg_file):
    106          if not os.path.isdir(static.STARCLUSTER_CFG_DIR):
    107              os.makedirs(static.STARCLUSTER_CFG_DIR)
    108          log.debug("Loading file: %s" % cfg_file)
    109          if os.path.exists(cfg_file):
    110              if not os.path.isfile(cfg_file):
    111                  raise exception.ConfigError(
    112                      'config %s exists but is not a regular file' % cfg_file)
    113          else:
    114              raise exception.ConfigNotFound(
    115                  ("config file %s does not exist\n") %
    116                  cfg_file, cfg_file,
    117              )
    118          return open(cfg_file)
    119
    120 -    def _get_bool(self, config, section, option):
    121          try:
    122              opt = config.getboolean(section, option)
    123              return opt
    124          except ConfigParser.NoSectionError:
    125              pass
    126          except ConfigParser.NoOptionError:
    127              pass
    128          except ValueError:
    129              raise exception.ConfigError(
    130                  "Expected True/False value for setting %s in section [%s]" %
    131                  (option, section))
    132
    133 -    def _get_int(self, config, section, option):
    134          try:
    135              opt = config.getint(section, option)
    136              return opt
    137          except ConfigParser.NoSectionError:
    138              pass
    139          except ConfigParser.NoOptionError:
    140              pass
    141          except ValueError:
    142              raise exception.ConfigError(
    143                  "Expected integer value for setting %s in section [%s]" %
    144                  (option, section))
    145
    146 -    def _get_float(self, config, section, option):
    147          try:
    148              opt = config.getfloat(section, option)
    149              return opt
    150          except ConfigParser.NoSectionError:
    151              pass
    152          except ConfigParser.NoOptionError:
    153              pass
    154          except ValueError:
    155              raise exception.ConfigError(
    156                  "Expected float value for setting %s in section [%s]" %
    157                  (option, section))
    158
    159 -    def _get_string(self, config, section, option):
    160          try:
    161              opt = config.get(section, option)
    162              return opt
    163          except ConfigParser.NoSectionError:
    164              pass
    165          except ConfigParser.NoOptionError:
    166              pass
    167
    168 -    def _get_list(self, config, section, option):
    169          val = self._get_string(config, section, option)
    170          if val:
    171              val = [v.strip() for v in val.split(',')]
    172          return val
    173
    174 -    def __load_config(self):
    175          """
    176          Populates self._config with a new ConfigParser instance
    177          """
    178          cfg = self.cfg_file
    179          if utils.is_url(cfg):
    180              cfg = self._get_urlfp(cfg)
    181          else:
    182              cfg = self._get_fp(cfg)
    183          try:
    184              cp = ConfigParser.ConfigParser()
    185              cp.readfp(cfg)
    186              return cp
    187          except ConfigParser.MissingSectionHeaderError:
    188              raise exception.ConfigHasNoSections(cfg.name)
    189          except ConfigParser.ParsingError, e:
    190              raise exception.ConfigError(e)
    191
    192 -    def reload(self):
    193          """
    194          Reloads the configuration file
    195          """
    196          self._config = self.__load_config()
    197          return self.load()
    198
    199      @property
    200 -    def config(self):
    201          if self._config is None:
    202              self._config = self.__load_config()
    203          return self._config
    204
    205 -    def _load_settings(self, section_name, settings, store,
    206                         filter_settings=True):
    207          """
    208          Load section settings into a dictionary
    209          """
    210          section = self.config._sections.get(section_name)
    211          if not section:
    212              raise exception.ConfigSectionMissing(
    213                  'Missing section %s in config' % section_name)
    214          store.update(section)
    215          section_conf = store
    216          for setting in settings:
    217              requirements = settings[setting]
    218              func, required, default, options, callback = requirements
    219              func = self.type_validators.get(func)
    220              value = func(self.config, section_name, setting)
    221              if value is not None:
    222                  if options and not value in options:
    223                      raise exception.ConfigError(
    224                          '"%s" setting in section "%s" must be one of: %s' %
    225                          (setting, section_name,
    226                           ', '.join([str(o) for o in options])))
    227                  if callback:
    228                      value = callback(value)
    229                  section_conf[setting] = value
    230          if filter_settings:
    231              for key in store.keys():
    232                  if key not in settings and key != '__name__':
    233                      store.pop(key)
    234
    235 -    def _check_required(self, section_name, settings, store):
    236          """
    237          Check that all required settings were specified in the config.
    238          Raises ConfigError otherwise.
    239
    240          Note that if a setting specified has required=True and
    241          default is not None then this method will not raise an error
    242          because a default was given. In short, if a setting is required
    243          you must provide None as the 'default' value.
    244          """
    245          section_conf = store
    246          for setting in settings:
    247              requirements = settings[setting]
    248              required = requirements[1]
    249              value = section_conf.get(setting)
    250              if value is None and required:
    251                  raise exception.ConfigError(
    252                      'missing required option %s in section "%s"' %
    253                      (setting, section_name))
    254
    255 -    def _load_defaults(self, settings, store):
    256          """
    257          Sets the default for each setting in settings regardless of whether
    258          the setting was specified in the config or not.
    259          """
    260          section_conf = store
    261          for setting in settings:
    262              default = settings[setting][2]
    263              if section_conf.get(setting) is None:
    264                  if DEBUG_CONFIG:
    265                      log.debug('%s setting not specified. Defaulting to %s' % \
    266                                (setting, default))
    267                  section_conf[setting] = default
    268
    269 -    def _load_extends_settings(self, section_name, store):
    270          """
    271          Loads all settings from other template(s) specified by a section's
    272          'extends' setting.
    273
    274          This method walks a dependency tree of sections from bottom up. Each
    275          step is a group of settings for a section in the form of a dictionary.
    276          A 'master' dictionary is updated with the settings at each step. This
    277          causes the next group of settings to override the previous, and so on.
    278          The 'section_name' settings are at the top of the dep tree.
    279          """
    280          section = store[section_name]
    281          extends = section.get('extends')
    282          if extends is None:
    283              return
    284          if DEBUG_CONFIG:
    285              log.debug('%s extends %s' % (section_name, extends))
    286          extensions = [section]
    287          while True:
    288              extends = section.get('extends', None)
    289              if not extends:
    290                  break
    291              try:
    292                  section = store[extends]
    293                  if section in extensions:
    294                      exts = ', '.join([self._get_section_name(x['__name__'])
    295                                        for x in extensions])
    296                      raise exception.ConfigError(
    297                          ("Cyclical dependency between sections %s. " % exts) \
    298                          + "Check your extends settings.")
    299                  extensions.insert(0, section)
    300              except KeyError:
    301                  raise exception.ConfigError(
    302                      "%s can't extend non-existent section %s" % \
    303                      (section_name, extends))
    304          transform = AttributeDict()
    305          for extension in extensions:
    306              transform.update(extension)
    307          store[section_name] = transform
    308
    309 -    def _load_keypairs(self, store):
    310          cluster_section = store
    311          keyname = cluster_section.get('keyname')
    312          if not keyname:
    313              return
    314          keypair = self.keys.get(keyname)
    315          if keypair is None:
    316              raise exception.ConfigError(
    317                  "keypair '%s' not defined in config" % keyname)
    318          cluster_section['keyname'] = keyname
    319          cluster_section['key_location'] = keypair.get('key_location')
    320
    321 -    def _load_volumes(self, store):
    322          cluster_section = store
    323          volumes = cluster_section.get('volumes')
    324          if not volumes or isinstance(volumes, AttributeDict):
    325              return
    326          vols = AttributeDict()
    327          cluster_section['volumes'] = vols
    328          for volume in volumes:
    329              if not volume in self.vols:
    330                  raise exception.ConfigError(
    331                      "volume '%s' not defined in config" % volume)
    332              vol = self.vols.get(volume)
    333              vols[volume] = vol
    334
    335 -    def _load_plugins(self, store):
    336          cluster_section = store
    337          plugins = cluster_section.get('plugins')
    338          if not plugins or isinstance(plugins[0], AttributeDict):
    339              return
    340          plugs = []
    341          cluster_section['plugins'] = plugs
    342          for plugin in plugins:
    343              if plugin in self.plugins:
    344                  p = self.plugins.get(plugin)
    345                  p['__name__'] = p['__name__'].split()[-1]
    346                  plugs.append(p)
    347              else:
    348                  raise exception.ConfigError(
    349                      "plugin '%s' not defined in config" % plugin)
    350
    351 -    def _load_permissions(self, store):
    352          cluster_section = store
    353          permissions = cluster_section.get('permissions')
    354          if not permissions or isinstance(permissions, AttributeDict):
    355              return
    356          perms = AttributeDict()
    357          cluster_section['permissions'] = perms
    358          for perm in permissions:
    359              if perm in self.permissions:
    360                  p = self.permissions.get(perm)
    361                  p['__name__'] = p['__name__'].split()[-1]
    362                  perms[perm] = p
    363              else:
    364                  raise exception.ConfigError(
    365                      "permission '%s' not defined in config" % perm)
    366
    367 -    def _load_instance_types(self, store):
    368          cluster_section = store
    369          instance_types = cluster_section.get('node_instance_type')
    370          if isinstance(instance_types, basestring):
    371              return
    372          itypes = []
    373          cluster_section['node_instance_types'] = itypes
    374          total_num_nodes = 0
    375          choices_string = ', '.join(static.INSTANCE_TYPES.keys())
    376          try:
    377              default_instance_type = instance_types[-1]
    378              if not default_instance_type in static.INSTANCE_TYPES:
    379                  raise exception.ConfigError(
    380                      ("invalid node_instance_type specified: '%s'\n" +
    381                       "must be one of: %s") %
    382                      (default_instance_type, choices_string))
    383          except IndexError:
    384              default_instance_type = None
    385          cluster_section['node_instance_type'] = default_instance_type
    386          for type_spec in instance_types[:-1]:
    387              type_spec = type_spec.split(':')
    388              if len(type_spec) > 3:
    389                  raise exception.ConfigError(
    390                      "invalid node_instance_type item specified: %s" % \
    391                      type_spec)
    392              itype = type_spec[0]
    393              itype_image = None
    394              itype_num = 1
    395              if not itype in static.INSTANCE_TYPES:
    396                  raise exception.ConfigError(
    397                      ("invalid type specified (%s) in node_instance_type " + \
    398                       "item: '%s'\nmust be one of: %s") %
    399                      (itype, type_spec, choices_string))
    400              if len(type_spec) == 2:
    401                  itype, next_var = type_spec
    402                  try:
    403                      itype_num = int(next_var)
    404                  except (TypeError, ValueError):
    405                      itype_image = next_var
    406              elif len(type_spec) == 3:
    407                  itype, itype_image, itype_num = type_spec
    408              try:
    409                  itype_num = int(itype_num)
    410                  if itype_num < 1:
    411                      raise TypeError
    412                  total_num_nodes += itype_num
    413              except (ValueError, TypeError):
    414                  raise exception.ConfigError(
    415                      ("number of instances (%s) of type '%s' must " + \
    416                       "be an integer > 1") % (itype_num, itype))
    417              itype_dic = AttributeDict(size=itype_num, image=itype_image,
    418                                        type=itype)
    419              itypes.append(itype_dic)
    420
    421 -    def _load_section(self, section_name, section_settings,
    422                        filter_settings=True):
    423          """
    424          Returns a dictionary containing all section_settings for a given
    425          section_name by first loading the settings in the config, loading
    426          the defaults for all settings not specified, and then checking
    427          that all required options have been specified
    428          """
    429          store = AttributeDict()
    430          self._load_settings(section_name, section_settings, store,
    431                              filter_settings)
    432          self._load_defaults(section_settings, store)
    433          self._check_required(section_name, section_settings, store)
    434          return store
    435
    436 -    def _get_section_name(self, section):
    437          """
    438          Returns section name minus prefix
    439          e.g.
    440          $ print self._get_section('cluster smallcluster')
    441          $ smallcluster
    442          """
    443          return section.split()[1]
    444
    445 -    def _get_sections(self, section_prefix):
    446          """
    447          Returns all sections starting with section_prefix
    448          e.g.
    449          $ print self._get_sections('cluster')
    450          $ ['cluster smallcluster', 'cluster mediumcluster', ..]
    451          """
    452          return [s for s in self.config.sections() if
    453                  s.startswith(section_prefix)]
    454
    455 -    def _load_sections(self, section_prefix, section_settings,
    456                         filter_settings=True):
    457          """
    458          Loads all sections starting with section_prefix and returns a
    459          dictionary containing the name and dictionary of settings for each
    460          section.
    461          keys --> section name (as returned by self._get_section_name)
    462          values --> dictionary of settings for a given section
    463
    464          e.g.
    465          $ print self._load_sections('volumes', self.plugin_settings)
    466
    467          {'myvol': {'__name__': 'volume myvol',
    468                      'device': None,
    469                      'mount_path': '/home',
    470                      'partition': 1,
    471                      'volume_id': 'vol-999999'},
    472           'myvol2': {'__name__': 'volume myvol2',
    473                         'device': None,
    474                         'mount_path': '/myvol2',
    475                         'partition': 1,
    476                         'volume_id': 'vol-999999'},
    477          """
    478          sections = self._get_sections(section_prefix)
    479          sections_store = AttributeDict()
    480          for sec in sections:
    481              name = self._get_section_name(sec)
    482              sections_store[name] = self._load_section(sec, section_settings,
    483                                                        filter_settings)
    484          return sections_store
    485
    486 -    def _load_cluster_sections(self, cluster_sections):
    487          """
    488          Loads all cluster sections. Similar to _load_sections but also handles
    489          populating specified keypair,volume,plugins,permissions,etc settings
    490          """
    491          clusters = cluster_sections
    492          cluster_store = AttributeDict()
    493          for cluster in clusters:
    494              name = self._get_section_name(cluster)
    495              cluster_store[name] = AttributeDict()
    496              self._load_settings(cluster, self.cluster_settings,
    497                                 cluster_store[name])
    498          for cluster in clusters:
    499              name = self._get_section_name(cluster)
    500              self._load_extends_settings(name, cluster_store)
    501              self._load_defaults(self.cluster_settings, cluster_store[name])
    502              self._load_keypairs(cluster_store[name])
    503              self._load_volumes(cluster_store[name])
    504              self._load_plugins(cluster_store[name])
    505              self._load_permissions(cluster_store[name])
    506              self._load_instance_types(cluster_store[name])
    507              self._check_required(cluster, self.cluster_settings,
    508                                 cluster_store[name])
    509          return cluster_store
    510
    511 -    def load(self):
    512          """
    513          Populate this config object from the StarCluster config
    514          """
    515          log.debug('Loading config')
    516          try:
    517              self.globals = self._load_section('global', self.global_settings)
    518          except exception.ConfigSectionMissing:
    519              pass
    520          try:
    521              self.aws = self._load_section('aws info', self.aws_settings)
    522          except exception.ConfigSectionMissing:
    523              log.warn("no [aws info] section found in config")
    524              log.warn("attempting to load credentials from environment...")
    525              self.aws.update(self.get_aws_from_environ())
    526          self.keys = self._load_sections('key', self.key_settings)
    527          self.vols = self._load_sections('volume', self.volume_settings)
    528          self.plugins = self._load_sections('plugin', self.plugin_settings,
    529                                             filter_settings=False)
    530          self.permissions = self._load_sections('permission',
    531                                                 self.permission_settings)
    532          sections = self._get_sections('cluster')
    533          self.clusters = self._load_cluster_sections(sections)
    534          return self
    535
    536 -    def get_aws_from_environ(self):
    537          """
    538          Returns AWS credentials defined in the user's shell
    539          environment.
    540          """
    541          awscreds = {}
    542          for key in static.AWS_SETTINGS:
    543              if key.upper() in os.environ:
    544                  awscreds[key] = os.environ.get(key.upper())
    545              elif key in os.environ:
    546                  awscreds[key] = os.environ.get(key)
    547          return awscreds
    548
    549 -    def get_aws_credentials(self):
    550          """
    551          Returns AWS credentials defined in the configuration
    552          file. Defining any of the AWS settings in the environment
    553          overrides the configuration file.
    554          """
    555          # first override with environment settings if they exist
    556          self.aws.update(self.get_aws_from_environ())
    557          return self.aws
    558
    559 -    def get_cluster_names(self):
    560          return self.clusters
    561
    562 -    def get_cluster_template(self, template_name, tag_name=None,
    563                               ec2_conn=None):
    564          """
    565          Returns Cluster instance configured with the settings in the
    566          config file.
    567
    568          template_name is the name of a cluster section defined in the config
    569
    570          tag_name, if specified, will be passed to Cluster instance
    571          as cluster_tag
    572          """
    573          try:
    574              kwargs = {}
    575              if tag_name:
    576                  kwargs.update(dict(cluster_tag=tag_name))
    577              kwargs.update(self.clusters[template_name])
    578              if not ec2_conn:
    579                  ec2_conn = self.get_easy_ec2()
    580              clust = Cluster(ec2_conn, **kwargs)
    581              return clust
    582          except KeyError:
    583              raise exception.ClusterTemplateDoesNotExist(template_name)
    584
    585 -    def get_default_cluster_template(self):
    586          """
    587          Returns the default_template specified in the [global] section
    588          of the config. Raises NoDefaultTemplateFound if no default cluster
    589          template has been specified in the config.
    590          """
    591          default = self.globals.get('default_template')
    592          if not default:
    593              raise exception.NoDefaultTemplateFound(
    594                  options=self.clusters.keys())
    595          if not default in self.clusters:
    596              raise exception.ClusterTemplateDoesNotExist(default)
    597          return default
    598
    599 -    def get_clusters(self):
    600          clusters = []
    601          for cluster in self.clusters:
    602              clusters.append(self.get_cluster_template(cluster))
    603          return clusters
    604
    605 -    def get_plugin(self, plugin):
    606          try:
    607              return self.plugins[plugin]
    608          except KeyError:
    609              raise exception.PluginNotFound(plugin)
    610
    611 -    def get_key(self, keyname):
    612          try:
    613              return self.keys[keyname]
    614          except KeyError:
    615              raise exception.KeyNotFound(keyname)
    616
    617 -    def get_easy_s3(self):
    618          """
    619          Factory for EasyEC2 class that attempts to load AWS credentials from
    620          the StarCluster config file. Returns an EasyS3 object if
    621          successful.
    622          """
    623          aws = self.get_aws_credentials()
    624          try:
    625              s3 = awsutils.EasyS3(**aws)
    626              return s3
    627          except TypeError:
    628              raise exception.ConfigError("no aws credentials found")
    629
    630 -    def get_easy_ec2(self):
    631          """
    632          Factory for EasyEC2 class that attempts to load AWS credentials from
    633          the StarCluster config file. Returns an EasyEC2 object if
    634          successful.
    635          """
    636          aws = self.get_aws_credentials()
    637          try:
    638              ec2 = awsutils.EasyEC2(**aws)
    639              return ec2
    640          except TypeError:
    641              raise exception.ConfigError("no aws credentials found")
    642
    643  if __name__ == "__main__":
    644      from pprint import pprint
    645      cfg = StarClusterConfig().load()
    646      pprint(cfg.aws)
    647      pprint(cfg.clusters)
    648      pprint(cfg.keys)
    649      pprint(cfg.vols)
    650