starcluster.node

Package starcluster :: Module node |
[hide private]
[frames] | no frames]

# Source Code for Module starcluster.node

      1  #!/usr/bin/env python
      2  import os
      3  import pwd
      4  import grp
      5  import time
      6  import stat
      7  import base64
      8  import posixpath
      9
     10  from starcluster import ssh
     11  from starcluster import utils
     12  from starcluster import static
     13  from starcluster import awsutils
     14  from starcluster import managers
     15  from starcluster import exception
     16  from starcluster.logger import log
     17
     18
     19 -class NodeManager(managers.Manager):
     20      """
     21      Manager class for Node objects
     22      """
     23 -    def ssh_to_node(self, node_id, user='root', command=None):
     24          node = self.get_node(node_id, user=user)
     25          if command:
     26              current_user = node.ssh.get_current_user()
     27              node.ssh.switch_user(user)
     28              node.ssh.execute(command, silent=False)
     29              node.ssh.switch_user(current_user)
     30              return node.ssh.get_last_status()
     31          else:
     32              node.shell(user=user)
     33
     34 -    def get_node(self, node_id, user='root'):
     35          """Factory for Node class"""
     36          instances = self.ec2.get_all_instances()
     37          node = None
     38          for instance in instances:
     39              if instance.dns_name == node_id:
     40                  node = instance
     41                  break
     42              elif instance.id == node_id:
     43                  node = instance
     44                  break
     45          if not node:
     46              raise exception.InstanceDoesNotExist(node_id)
     47          key = self.cfg.get_key(node.key_name)
     48          node = Node(node, key.key_location, user=user)
     49          return node
     50
     51
     52 -class Node(object):
     53      """
     54      This class represents a single compute node in a StarCluster.
     55
     56      It contains all useful metadata for the node such as the internal/external
     57      hostnames, ips, etc as well as a paramiko ssh object for executing
     58      commands, creating/modifying files on the node.
     59
     60      'instance' arg must be an instance of boto.ec2.instance.Instance
     61
     62      'key_location' arg is a string that contains the full path to the
     63      private key corresponding to the keypair used to launch this node
     64
     65      'alias' keyword arg optionally names the node. If no alias is provided,
     66      the alias is retrieved from the node's user_data based on the node's
     67      launch index
     68
     69      'user' keyword optionally specifies user to ssh as (defaults to root)
     70      """
     71 -    def __init__(self, instance, key_location, alias=None, user='root'):
     72          self.instance = instance
     73          self.ec2 = awsutils.EasyEC2(None, None)
     74          self.ec2._conn = instance.connection
     75          self.key_location = key_location
     76          self.user = user
     77          self._alias = alias
     78          self._groups = None
     79          self._ssh = None
     80          self._num_procs = None
     81          self._memory = None
     82
     83 -    def __repr__(self):
     84          return '<Node: %s (%s)>' % (self.alias, self.id)
     85
     86 -    def _get_user_data(self, tries=5):
     87          tries = range(tries)
     88          last_try = tries[-1]
     89          for i in tries:
     90              try:
     91                  user_data = self.ec2.get_instance_user_data(self.id)
     92                  return user_data
     93              except exception.InstanceDoesNotExist:
     94                  if i == last_try:
     95                      log.debug("failed fetching user data")
     96                      raise
     97                  log.debug("InvalidInstanceID.NotFound: "
     98                            "retrying fetching user data (tries: %s)" % (i + 1))
     99                  time.sleep(5)
    100
    101      @property
    102 -    def alias(self):
    103          """
    104          Fetches the node's alias stored in a tag from either the instance
    105          or the instance's parent spot request. If no alias tag is found an
    106          exception is raised.
    107          """
    108          if not self._alias:
    109              alias = self.tags.get('alias')
    110              if not alias:
    111                  user_data = self._get_user_data(tries=5)
    112                  aliases = user_data.split('|')
    113                  index = self.ami_launch_index
    114                  try:
    115                      alias = aliases[index]
    116                  except IndexError:
    117                      log.debug(
    118                          "invalid user_data: %s (index: %d)" % (aliases, index))
    119                      alias = None
    120                  if not alias:
    121                      raise exception.BaseException(
    122                          "instance %s has no alias" % self.id)
    123                  self.add_tag('alias', alias)
    124              name = self.tags.get('Name')
    125              if not name:
    126                  self.add_tag('Name', alias)
    127              self._alias = alias
    128          return self._alias
    129
    130 -    def _remove_all_tags(self):
    131          tags = self.tags.keys()[:]
    132          for t in tags:
    133              self.remove_tag(t)
    134
    135      @property
    136 -    def tags(self):
    137          return self.instance.tags
    138
    139 -    def add_tag(self, key, value=None):
    140          return self.instance.add_tag(key, value)
    141
    142 -    def remove_tag(self, key, value=None):
    143          return self.instance.remove_tag(key, value)
    144
    145      @property
    146 -    def groups(self):
    147          if not self._groups:
    148              groups = map(lambda x: x.id, self.instance.groups)
    149              self._groups = self.ec2.get_all_security_groups(groupnames=groups)
    150          return self._groups
    151
    152      @property
    153 -    def cluster_groups(self):
    154          sg_prefix = static.SECURITY_GROUP_PREFIX
    155          return filter(lambda x: x.name.startswith(sg_prefix), self.groups)
    156
    157      @property
    158 -    def num_processors(self):
    159          if not self._num_procs:
    160              self._num_procs = int(
    161                  self.ssh.execute(
    162                      'cat /proc/cpuinfo | grep processor | wc -l')[0])
    163          return self._num_procs
    164
    165      @property
    166 -    def memory(self):
    167          if not self._memory:
    168              self._memory = float(
    169                  self.ssh.execute(
    170                      "free -m | grep -i mem | awk '{print $2}'")[0])
    171          return self._memory
    172
    173      @property
    174 -    def ip_address(self):
    175          return self.instance.ip_address
    176
    177      @property
    178 -    def public_dns_name(self):
    179          return self.instance.public_dns_name
    180
    181      @property
    182 -    def private_ip_address(self):
    183          return self.instance.private_ip_address
    184
    185      @property
    186 -    def private_dns_name(self):
    187          return self.instance.private_dns_name
    188
    189      @property
    190 -    def private_dns_name_short(self):
    191          return self.instance.private_dns_name.split('.')[0]
    192
    193      @property
    194 -    def id(self):
    195          return self.instance.id
    196
    197      @property
    198 -    def block_device_mapping(self):
    199          return self.instance.block_device_mapping
    200
    201      @property
    202 -    def dns_name(self):
    203          return self.instance.dns_name
    204
    205      @property
    206 -    def state(self):
    207          return self.instance.state
    208
    209      @property
    210 -    def launch_time(self):
    211          return self.instance.launch_time
    212
    213      @property
    214 -    def local_launch_time(self):
    215          ltime = utils.iso_to_localtime_tuple(self.launch_time)
    216          return time.strftime("%Y-%m-%d %H:%M:%S", ltime.timetuple())
    217
    218      @property
    219 -    def uptime(self):
    220          return utils.get_elapsed_time(self.launch_time)
    221
    222      @property
    223 -    def ami_launch_index(self):
    224          try:
    225              return int(self.instance.ami_launch_index)
    226          except TypeError:
    227              log.error("instance %s (state: %s) has no ami_launch_index" % \
    228                        (self.id, self.state))
    229              log.error("returning 0 as ami_launch_index...")
    230              return 0
    231
    232      @property
    233 -    def key_name(self):
    234          return self.instance.key_name
    235
    236      @property
    237 -    def arch(self):
    238          return self.instance.architecture
    239
    240      @property
    241 -    def kernel(self):
    242          return self.instance.kernel
    243
    244      @property
    245 -    def ramdisk(self):
    246          return self.instance.ramdisk
    247
    248      @property
    249 -    def instance_type(self):
    250          return self.instance.instance_type
    251
    252      @property
    253 -    def image_id(self):
    254          return self.instance.image_id
    255
    256      @property
    257 -    def placement(self):
    258          return self.instance.placement
    259
    260      @property
    261 -    def root_device_name(self):
    262          return self.instance.root_device_name
    263
    264      @property
    265 -    def root_device_type(self):
    266          return self.instance.root_device_type
    267
    268 -    def add_user_to_group(self, user, group):
    269          """
    270          Add user (if exists) to group (if exists)
    271          """
    272          if not user in self.get_user_map():
    273              raise exception.BaseException("user %s does not exist" % user)
    274          if group in self.get_group_map():
    275              self.ssh.execute('gpasswd -a %s %s' % (user, 'utmp'))
    276          else:
    277              raise exception.BaseException("group %s does not exist" % group)
    278
    279 -    def get_group_map(self, key_by_gid=False):
    280          """
    281          Returns dictionary where keys are remote group names and values are
    282          grp.struct_grp objects from the standard grp module
    283
    284          key_by_gid=True will use the integer gid as the returned dictionary's
    285          keys instead of the group's name
    286          """
    287          grp_file = self.ssh.remote_file('/etc/group', 'r')
    288          groups = [l.strip().split(':') for l in grp_file.readlines()]
    289          grp_file.close()
    290          grp_map = {}
    291          for group in groups:
    292              name, passwd, gid, mems = group
    293              gid = int(gid)
    294              mems = mems.split(',')
    295              key = name
    296              if key_by_gid:
    297                  key = gid
    298              grp_map[key] = grp.struct_group([name, passwd, gid, mems])
    299          return grp_map
    300
    301 -    def get_user_map(self, key_by_uid=False):
    302          """
    303          Returns dictionary where keys are remote usernames and values are
    304          pwd.struct_passwd objects from the standard pwd module
    305
    306          key_by_uid=True will use the integer uid as the returned dictionary's
    307          keys instead of the user's login name
    308          """
    309          etc_passwd = self.ssh.remote_file('/etc/passwd', 'r')
    310          users = [l.strip().split(':') for l in etc_passwd.readlines()]
    311          etc_passwd.close()
    312          user_map = {}
    313          for user in users:
    314              name, passwd, uid, gid, gecos, home, shell = user
    315              uid = int(uid)
    316              gid = int(gid)
    317              key = name
    318              if key_by_uid:
    319                  key = uid
    320              user_map[key] = pwd.struct_passwd([name, passwd, uid, gid,
    321                                                 gecos, home, shell])
    322          return user_map
    323
    324 -    def getgrgid(self, gid):
    325          """
    326          Remote version of the getgrgid method in the standard grp module
    327
    328          returns a grp.struct_group
    329          """
    330          gmap = self.get_group_map(key_by_gid=True)
    331          return gmap.get(gid)
    332
    333 -    def getgrnam(self, groupname):
    334          """
    335          Remote version of the getgrnam method in the standard grp module
    336
    337          returns a grp.struct_group
    338          """
    339          gmap = self.get_group_map()
    340          return gmap.get(groupname)
    341
    342 -    def getpwuid(self, uid):
    343          """
    344          Remote version of the getpwuid method in the standard pwd module
    345
    346          returns a pwd.struct_passwd
    347          """
    348          umap = self.get_user_map(key_by_uid=True)
    349          return umap.get(uid)
    350
    351 -    def getpwnam(self, username):
    352          """
    353          Remote version of the getpwnam method in the standard pwd module
    354
    355          returns a pwd.struct_passwd
    356          """
    357          umap = self.get_user_map()
    358          return umap.get(username)
    359
    360 -    def add_user(self, name, uid=None, gid=None, shell="bash"):
    361          """
    362          Add a user to the remote system.
    363
    364          name - the username of the user being added
    365          uid - optional user id to use when creating new user
    366          gid - optional group id to use when creating new user
    367          shell - optional shell assign to new user (default: bash)
    368          """
    369          if gid:
    370              self.ssh.execute('groupadd -o -g %s %s' % (gid, name))
    371          user_add_cmd = 'useradd -o '
    372          if uid:
    373              user_add_cmd += '-u %s ' % uid
    374          if gid:
    375              user_add_cmd += '-g %s ' % gid
    376          if shell:
    377              user_add_cmd += '-s `which %s` ' % shell
    378          user_add_cmd += "-m %s" % name
    379          self.ssh.execute(user_add_cmd)
    380
    381 -    def generate_key_for_user(self, username, ignore_existing=False,
    382                                auth_new_key=False, auth_conn_key=False):
    383          """
    384          Generates an id_rsa/id_rsa.pub keypair combo for a user on the remote
    385          machine.
    386
    387          ignore_existing - if False, any existing key combos will be used rather
    388          than generating a new RSA key
    389
    390          auth_new_key - if True, add the newly generated public key to the
    391          remote user's authorized_keys file
    392
    393          auth_conn_key - if True, add the public key used to establish this ssh
    394          connection to the remote user's authorized_keys
    395          """
    396          user = self.getpwnam(username)
    397          home_folder = user.pw_dir
    398          ssh_folder = posixpath.join(home_folder, '.ssh')
    399          if not self.ssh.isdir(ssh_folder):
    400              self.ssh.mkdir(ssh_folder)
    401          private_key = posixpath.join(ssh_folder, 'id_rsa')
    402          public_key = private_key + '.pub'
    403          authorized_keys = posixpath.join(ssh_folder, 'authorized_keys')
    404          key_exists = self.ssh.isfile(private_key)
    405          if key_exists and not ignore_existing:
    406              log.info("Using existing key: %s" % private_key)
    407              key = self.ssh.load_remote_rsa_key(private_key)
    408          else:
    409              key = self.ssh.generate_rsa_key()
    410          pubkey_contents = self.ssh.get_public_key(key)
    411          if not key_exists or ignore_existing:
    412              # copy public key to remote machine
    413              pub_key = self.ssh.remote_file(public_key, 'w')
    414              pub_key.write(pubkey_contents)
    415              pub_key.chown(user.pw_uid, user.pw_gid)
    416              pub_key.chmod(0400)
    417              pub_key.close()
    418              # copy private key to remote machine
    419              priv_key = self.ssh.remote_file(private_key, 'w')
    420              key.write_private_key(priv_key)
    421              priv_key.chown(user.pw_uid, user.pw_gid)
    422              priv_key.chmod(0400)
    423              priv_key.close()
    424          if not auth_new_key or not auth_conn_key:
    425              return key
    426          auth_keys_contents = ''
    427          if self.ssh.isfile(authorized_keys):
    428              auth_keys = self.ssh.remote_file(authorized_keys, 'r')
    429              auth_keys_contents = auth_keys.read()
    430              auth_keys.close()
    431          auth_keys = self.ssh.remote_file(authorized_keys, 'a')
    432          if auth_new_key:
    433              # add newly generated public key to user's authorized_keys
    434              if pubkey_contents not in auth_keys_contents:
    435                  log.debug("adding auth_key_contents")
    436                  auth_keys.write('%s\n' % pubkey_contents)
    437          if auth_conn_key and self.ssh._pkey:
    438              # add public key used to create the connection to user's
    439              # authorized_keys
    440              conn_key = self.ssh._pkey
    441              conn_pubkey_contents = self.ssh.get_public_key(conn_key)
    442              if conn_pubkey_contents not in auth_keys_contents:
    443                  log.debug("adding conn_pubkey_contents")
    444                  auth_keys.write('%s\n' % conn_pubkey_contents)
    445          auth_keys.chown(user.pw_uid, user.pw_gid)
    446          auth_keys.chmod(0600)
    447          auth_keys.close()
    448          return key
    449
    450 -    def add_to_known_hosts(self, username, nodes, add_self=True):
    451          """
    452          Populate user's known_hosts file with pub keys from hosts in nodes list
    453
    454          username - name of the user to add to known hosts for
    455          nodes - the nodes to add to the user's known hosts file
    456          add_self - add this Node to known_hosts in addition to nodes
    457
    458          NOTE: this node's hostname will also be added to the known_hosts
    459          file
    460          """
    461          user = self.getpwnam(username)
    462          known_hosts_file = posixpath.join(user.pw_dir, '.ssh', 'known_hosts')
    463          self.remove_from_known_hosts(username, nodes)
    464          khosts = []
    465          for node in nodes:
    466              server_pkey = node.ssh.get_server_public_key()
    467              khosts.append(' '.join([node.alias, server_pkey.get_name(),
    468                                      base64.b64encode(str(server_pkey))]))
    469          if add_self and self not in nodes:
    470              server_pkey = self.ssh.get_server_public_key()
    471              khosts.append(' '.join([self.alias, server_pkey.get_name(),
    472                                      base64.b64encode(str(server_pkey))]))
    473          khostsf = self.ssh.remote_file(known_hosts_file, 'a')
    474          khostsf.write('\n'.join(khosts) + '\n')
    475          khostsf.chown(user.pw_uid, user.pw_gid)
    476          khostsf.close()
    477
    478 -    def remove_from_known_hosts(self, username, nodes):
    479          """
    480          Remove all network names for nodes from username's known_hosts file
    481          on this Node
    482          """
    483          user = self.getpwnam(username)
    484          known_hosts_file = posixpath.join(user.pw_dir, '.ssh', 'known_hosts')
    485          hostnames = map(lambda n: n.alias, nodes)
    486          if self.ssh.isfile(known_hosts_file):
    487              regex = '|'.join(hostnames)
    488              self.ssh.remove_lines_from_file(known_hosts_file, regex)
    489
    490 -    def enable_passwordless_ssh(self, username, nodes):
    491          """
    492          Configure passwordless ssh for user between this Node and nodes
    493          """
    494          user = self.getpwnam(username)
    495          ssh_folder = posixpath.join(user.pw_dir, '.ssh')
    496          priv_key_file = posixpath.join(ssh_folder, 'id_rsa')
    497          pub_key_file = priv_key_file + '.pub'
    498          known_hosts_file = posixpath.join(ssh_folder, 'known_hosts')
    499          auth_key_file = posixpath.join(ssh_folder, 'authorized_keys')
    500          self.add_to_known_hosts(username, nodes)
    501          # exclude this node from copying
    502          nodes = filter(lambda n: n.id != self.id, nodes)
    503          # copy private key and public key to node
    504          self.copy_remote_file_to_nodes(priv_key_file, nodes)
    505          self.copy_remote_file_to_nodes(pub_key_file, nodes)
    506          # copy authorized_keys and known_hosts to node
    507          self.copy_remote_file_to_nodes(auth_key_file, nodes)
    508          self.copy_remote_file_to_nodes(known_hosts_file, nodes)
    509
    510 -    def copy_remote_file_to_node(self, remote_file, node, dest=None):
    511          return self.copy_remote_file_to_nodes(remote_file, [node], dest=dest)
    512
    513 -    def copy_remote_file_to_nodes(self, remote_file, nodes, dest=None):
    514          """
    515          Copies a remote file from this Node instance to another Node instance
    516          without passwordless ssh between the two.
    517
    518          dest - path to store the data in on the node (defaults to remote_file)
    519          """
    520          if not dest:
    521              dest = remote_file
    522          rf = self.ssh.remote_file(remote_file, 'r')
    523          contents = rf.read()
    524          sts = rf.stat()
    525          mode = stat.S_IMODE(sts.st_mode)
    526          uid = sts.st_uid
    527          gid = sts.st_gid
    528          rf.close()
    529          for node in nodes:
    530              if self.id == node.id and remote_file == dest:
    531                  log.warn("src and destination are the same: %s, skipping" %
    532                           remote_file)
    533                  continue
    534              nrf = node.ssh.remote_file(dest, 'w')
    535              nrf.write(contents)
    536              nrf.chown(uid, gid)
    537              nrf.chmod(mode)
    538              nrf.close()
    539
    540 -    def remove_user(self, name):
    541          """
    542          Remove a user from the remote system
    543          """
    544          self.ssh.execute('userdel %s' % name)
    545          self.ssh.execute('groupdel %s' % name)
    546
    547 -    def export_fs_to_nodes(self, nodes, export_paths):
    548          """
    549          Export each path in export_paths to each node in nodes via NFS
    550
    551          nodes - list of nodes to export each path to
    552          export_paths - list of paths on this remote host to export to each node
    553
    554          Example:
    555          # export /home and /opt/sge6 to each node in nodes
    556          $ node.start_nfs_server()
    557          $ node.export_fs_to_nodes(\
    558                  nodes=[node1,node2], export_paths=['/home', '/opt/sge6']
    559          """
    560          # setup /etc/exports
    561          nfs_export_settings = "(async,no_root_squash,no_subtree_check,rw)"
    562          etc_exports = self.ssh.remote_file('/etc/exports')
    563          for node in nodes:
    564              for path in export_paths:
    565                  etc_exports.write(' '.join([path, node.alias + \
    566                                              nfs_export_settings + '\n']))
    567          etc_exports.close()
    568          self.ssh.execute('exportfs -a')
    569
    570 -    def stop_exporting_fs_to_nodes(self, nodes):
    571          """
    572          Removes nodes from this node's /etc/exportfs
    573
    574          nodes - list of nodes to stop
    575
    576          Example:
    577          $ node.remove_export_fs_to_nodes(nodes=[node1,node2])
    578          """
    579          regex = '|'.join(map(lambda x: x.alias, nodes))
    580          self.ssh.remove_lines_from_file('/etc/exports', regex)
    581          self.ssh.execute('exportfs -a')
    582
    583 -    def start_nfs_server(self):
    584          self.ssh.execute('/etc/init.d/portmap start')
    585          self.ssh.execute('mount -t rpc_pipefs sunrpc /var/lib/nfs/rpc_pipefs/',
    586                           ignore_exit_status=True)
    587          self.ssh.execute('/etc/init.d/nfs start')
    588          self.ssh.execute('/usr/sbin/exportfs -r')
    589
    590 -    def mount_nfs_shares(self, server_node, remote_paths):
    591          """
    592          Mount each path in remote_paths from the remote server_node
    593
    594          server_node - remote server node that is sharing the remote_paths
    595          remote_paths - list of remote paths to mount from server_node
    596          """
    597          self.ssh.execute('/etc/init.d/portmap start')
    598          # TODO: move this fix for xterm somewhere else
    599          self.ssh.execute('mount -t devpts none /dev/pts',
    600                           ignore_exit_status=True)
    601          remote_paths_regex = '|'.join(map(lambda x: x.center(len(x) + 2),
    602                                            remote_paths))
    603          self.ssh.remove_lines_from_file('/etc/fstab', remote_paths_regex)
    604          fstab = self.ssh.remote_file('/etc/fstab', 'a')
    605          for path in remote_paths:
    606              fstab.write('%s:%s %s nfs user,rw,exec,noauto 0 0\n' %
    607                          (server_node.alias, path, path))
    608          fstab.close()
    609          for path in remote_paths:
    610              if not self.ssh.path_exists(path):
    611                  self.ssh.makedirs(path)
    612              self.ssh.execute('mount %s' % path)
    613
    614 -    def get_mount_map(self):
    615          mount_map = {}
    616          mount_lines = self.ssh.execute('mount')
    617          for line in mount_lines:
    618              dev, on_label, path, type_label, fstype, options = line.split()
    619              mount_map[dev] = [path, fstype, options]
    620          return mount_map
    621
    622 -    def mount_device(self, device, path):
    623          """
    624          Mount device to path
    625          """
    626          self.ssh.remove_lines_from_file('/etc/fstab',
    627                                          path.center(len(path) + 2))
    628          master_fstab = self.ssh.remote_file('/etc/fstab', mode='a')
    629          master_fstab.write("%s %s auto noauto,defaults 0 0\n" % \
    630                             (device, path))
    631          master_fstab.close()
    632          if not self.ssh.path_exists(path):
    633              self.ssh.makedirs(path)
    634          self.ssh.execute('mount %s' % path)
    635
    636 -    def add_to_etc_hosts(self, nodes):
    637          """
    638          Adds all names for node in nodes arg to this node's /etc/hosts file
    639          """
    640          self.remove_from_etc_hosts(nodes)
    641          host_file = self.ssh.remote_file('/etc/hosts', 'a')
    642          for node in nodes:
    643              print >> host_file, node.get_hosts_entry()
    644          host_file.close()
    645
    646 -    def remove_from_etc_hosts(self, nodes):
    647          """
    648          Remove all network names for node in nodes arg from this node's
    649          /etc/hosts file
    650          """
    651          aliases = map(lambda x: x.alias, nodes)
    652          self.ssh.remove_lines_from_file('/etc/hosts', '|'.join(aliases))
    653
    654 -    def set_hostname(self, hostname=None):
    655          """
    656          Set this node's hostname to self.alias
    657
    658          hostname - optional hostname to set (defaults to self.alias)
    659          """
    660          hostname = hostname or self.alias
    661          hostname_file = self.ssh.remote_file("/etc/hostname", "w")
    662          hostname_file.write(hostname)
    663          hostname_file.close()
    664          self.ssh.execute('hostname -F /etc/hostname')
    665
    666      @property
    667 -    def network_names(self):
    668          """ Returns all network names for this node in a dictionary"""
    669          names = {}
    670          names['INTERNAL_IP'] = self.private_ip_address
    671          names['INTERNAL_NAME'] = self.private_dns_name
    672          names['INTERNAL_NAME_SHORT'] = self.private_dns_name_short
    673          names['INTERNAL_ALIAS'] = self.alias
    674          return names
    675
    676      @property
    677 -    def attached_vols(self):
    678          """
    679          Returns a dictionary of all attached volumes minus the root device in
    680          the case of EBS backed instances
    681          """
    682          attached_vols = {}
    683          attached_vols.update(self.block_device_mapping)
    684          if self.is_ebs_backed():
    685              # exclude the root device from the list
    686              if self.root_device_name in attached_vols:
    687                  attached_vols.pop(self.root_device_name)
    688          return attached_vols
    689
    690 -    def detach_external_volumes(self):
    691          """
    692          Detaches all volumes returned by self.attached_vols
    693          """
    694          block_devs = self.attached_vols
    695          for dev in block_devs:
    696              vol_id = block_devs[dev].volume_id
    697              vol = self.ec2.get_volume(vol_id)
    698              log.info("Detaching volume %s from %s" % (vol.id, self.alias))
    699              if vol.status not in ['available', 'detaching']:
    700                  vol.detach()
    701
    702 -    def delete_root_volume(self):
    703          """
    704          Detach and destroy EBS root volume (EBS-backed node only)
    705          """
    706          if not self.is_ebs_backed():
    707              return
    708          root_vol = self.block_device_mapping[self.root_device_name]
    709          vol_id = root_vol.volume_id
    710          vol = self.ec2.get_volume(vol_id)
    711          vol.detach()
    712          while vol.update() != 'availabile':
    713              time.sleep(5)
    714          log.info("Deleting node %s's root volume" % self.alias)
    715          root_vol.delete()
    716
    717      @property
    718 -    def spot_id(self):
    719          if self.instance.spot_instance_request_id:
    720              return self.instance.spot_instance_request_id
    721
    722 -    def get_spot_request(self):
    723          spot = self.ec2.get_all_spot_requests(
    724              filters={'spot-instance-request-id': self.spot_id})
    725          if spot:
    726              return spot[0]
    727
    728 -    def is_master(self):
    729          return self.alias == "master"
    730
    731 -    def is_instance_store(self):
    732          return self.instance.root_device_type == "instance-store"
    733
    734 -    def is_ebs_backed(self):
    735          return self.instance.root_device_type == "ebs"
    736
    737 -    def is_cluster_compute(self):
    738          return self.instance.instance_type in static.CLUSTER_COMPUTE_TYPES
    739
    740 -    def is_gpu_compute(self):
    741          return self.instance.instance_type in static.CLUSTER_GPU_TYPES
    742
    743 -    def is_cluster_type(self):
    744          return self.instance.instance_type in static.CLUSTER_TYPES
    745
    746 -    def is_spot(self):
    747          return self.spot_id is not None
    748
    749 -    def is_stoppable(self):
    750          return self.is_ebs_backed() and not self.is_spot()
    751
    752 -    def is_stopped(self):
    753          return self.state == "stopped"
    754
    755 -    def start(self):
    756          """
    757          Starts EBS-backed instance and puts it in the 'running' state.
    758          Only works if this node is EBS-backed, raises
    759          exception.InvalidOperation otherwise.
    760          """
    761          if not self.is_ebs_backed():
    762              raise exception.InvalidOperation(
    763                  "Only EBS-backed instances can be started")
    764          return self.instance.start()
    765
    766 -    def stop(self):
    767          """
    768          Shutdown EBS-backed instance and put it in the 'stopped' state.
    769          Only works if this node is EBS-backed, raises
    770          exception.InvalidOperation otherwise.
    771
    772          NOTE: The EBS root device will *not* be deleted and the instance can
    773          be 'started' later on.
    774          """
    775          if self.is_spot():
    776              raise exception.InvalidOperation(
    777                  "spot instances can not be stopped")
    778          elif not self.is_ebs_backed():
    779              raise exception.InvalidOperation(
    780                  "Only EBS-backed instances can be stopped")
    781          if not self.is_stopped():
    782              log.info("Stopping node: %s (%s)" % (self.alias, self.id))
    783              return self.instance.stop()
    784          else:
    785              log.info("Node '%s' is already stopped" % self.alias)
    786
    787 -    def terminate(self):
    788          """
    789          Shutdown and destroy this instance. For EBS-backed nodes, this
    790          will also destroy the node's EBS root device. Puts this node
    791          into a 'terminated' state.
    792          """
    793          log.info("Terminating node: %s (%s)" % (self.alias, self.id))
    794          return self.instance.terminate()
    795
    796 -    def shutdown(self):
    797          """
    798          Shutdown this instance. This method will terminate traditional
    799          instance-store instances and stop EBS-backed instances
    800          (ie not destroy EBS root dev)
    801          """
    802          if self.is_stoppable():
    803              self.stop()
    804          else:
    805              self.terminate()
    806
    807 -    def reboot(self):
    808          """
    809          Reboot this instance.
    810          """
    811          self.instance.reboot()
    812
    813 -    def is_ssh_up(self):
    814          try:
    815              return self.ssh.transport is not None
    816          except exception.SSHError:
    817              return False
    818
    819 -    def is_up(self):
    820          if self.update() != 'running':
    821              return False
    822          if not self.is_ssh_up():
    823              return False
    824          if self.private_ip_address is None:
    825              log.debug("instance %s has no private_ip_address" % self.id)
    826              log.debug(("attempting to determine private_ip_address for" + \
    827                         "instance %s") % self.id)
    828              try:
    829                  private_ip = self.ssh.execute((
    830                      'python -c ' + \
    831                      '"import socket; print socket.gethostbyname(\'%s\')"') % \
    832                      self.private_dns_name)[0].strip()
    833                  log.debug("determined instance %s's private ip to be %s" % \
    834                            (self.id, private_ip))
    835                  self.instance.private_ip_address = private_ip
    836              except Exception, e:
    837                  print e
    838                  return False
    839          return True
    840
    841 -    def update(self):
    842          res = self.ec2.get_all_instances(filters={'instance-id': self.id})
    843          self.instance = res[0]
    844          return self.state
    845
    846      @property
    847 -    def ssh(self):
    848          if not self._ssh:
    849              self._ssh = ssh.SSHClient(self.instance.dns_name,
    850                                        username=self.user,
    851                                        private_key=self.key_location)
    852          return self._ssh
    853
    854 -    def shell(self, user=None):
    855          """
    856          Attempts to launch an interactive shell by first trying the system's
    857          ssh client. If the system does not have the ssh command it falls back
    858          to a pure-python ssh shell.
    859          """
    860          if self.update() != 'running':
    861              try:
    862                  alias = self.alias
    863              except exception.BaseException:
    864                  alias = None
    865              label = 'instance'
    866              if alias == "master":
    867                  label = "master"
    868              elif alias:
    869                  label = "node"
    870              instance_id = alias or self.id
    871              raise exception.InstanceNotRunning(instance_id, self.state,
    872                                                 label=label)
    873          user = user or self.user
    874          if utils.has_required(['ssh']):
    875              log.debug("using system's ssh client")
    876              ssh_cmd = static.SSH_TEMPLATE % (self.key_location, user,
    877                                               self.dns_name)
    878              log.debug("ssh_cmd: %s" % ssh_cmd)
    879              os.system(ssh_cmd)
    880          else:
    881              log.debug("using pure-python ssh client")
    882              self.ssh.interactive_shell(user=user)
    883
    884 -    def get_hosts_entry(self):
    885          """ Returns /etc/hosts entry for this node """
    886          etc_hosts_line = "%(INTERNAL_IP)s %(INTERNAL_ALIAS)s"
    887          etc_hosts_line = etc_hosts_line % self.network_names
    888          return etc_hosts_line
    889
    890 -    def __del__(self):
    891          if self._ssh:
    892              self._ssh.close()
    893