starcluster.awsutils

Package starcluster :: Module awsutils |
[hide private]
[frames] | no frames]

# Source Code for Module starcluster.awsutils

       1  #!/usr/bin/env python
       2  """
       3  EC2/S3 Utility Classes
       4  """
       5
       6  import os
       7  import re
       8  import time
       9  import base64
      10  import string
      11  import tempfile
      12
      13  import boto
      14  import boto.ec2
      15  import boto.s3.connection
      16
      17  from starcluster import image
      18  from starcluster import utils
      19  from starcluster import static
      20  from starcluster import exception
      21  from starcluster import progressbar
      22  from starcluster.utils import print_timing
      23  from starcluster.logger import log
      24
      25
      26 -class EasyAWS(object):
      27 -    def __init__(self, aws_access_key_id, aws_secret_access_key,
      28                   connection_authenticator, **kwargs):
      29          """
      30          Create an EasyAWS object.
      31
      32          Requires aws_access_key_id/aws_secret_access_key from an Amazon Web
      33          Services (AWS) account and a connection_authenticator function that
      34          returns an authenticated AWS connection object
      35
      36          Providing only the keys will default to using Amazon EC2
      37
      38          kwargs are passed to the connection_authenticator's constructor
      39          """
      40          self.aws_access_key_id = aws_access_key_id
      41          self.aws_secret_access_key = aws_secret_access_key
      42          self.connection_authenticator = connection_authenticator
      43          self._conn = None
      44          self._kwargs = kwargs
      45
      46 -    def reload(self):
      47          self._conn = None
      48          return self.conn
      49
      50      @property
      51 -    def conn(self):
      52          if self._conn is None:
      53              log.debug('creating self._conn w/ connection_authenticator ' +
      54                        'kwargs = %s' % self._kwargs)
      55              self._conn = self.connection_authenticator(
      56                  self.aws_access_key_id, self.aws_secret_access_key,
      57                  **self._kwargs)
      58          return self._conn
      59
      60
      61 -class EasyEC2(EasyAWS):
      62 -    def __init__(self, aws_access_key_id, aws_secret_access_key,
      63                   aws_ec2_path='/', aws_s3_host=None, aws_s3_path='/',
      64                   aws_port=None, aws_region_name=None, aws_is_secure=True,
      65                   aws_region_host=None, cache=False, **kwargs):
      66          aws_region = None
      67          if aws_region_name and aws_region_host:
      68              aws_region = boto.ec2.regioninfo.RegionInfo(
      69                  name=aws_region_name, endpoint=aws_region_host)
      70          kwargs = dict(is_secure=aws_is_secure, region=aws_region,
      71                        port=aws_port, path=aws_ec2_path)
      72          super(EasyEC2, self).__init__(aws_access_key_id, aws_secret_access_key,
      73                                        boto.connect_ec2, **kwargs)
      74          kwargs = dict(aws_s3_host=aws_s3_host,
      75                        aws_s3_path=aws_s3_path,
      76                        aws_port=aws_port,
      77                        aws_is_secure=aws_is_secure,
      78                        cache=cache)
      79          self.s3 = EasyS3(aws_access_key_id, aws_secret_access_key, **kwargs)
      80          self.cache = cache
      81          self._instance_response = None
      82          self._keypair_response = None
      83          self._images = None
      84          self._executable_images = None
      85          self._security_group_response = None
      86          self._regions = None
      87
      88 -    def __repr__(self):
      89          return '<EasyEC2: %s (%s)>' % (self.region.name, self.region.endpoint)
      90
      91 -    def __check_for_auth_failure(self, e):
      92          if e.error_code in ["AuthFailure", "SignatureDoesNotMatch"]:
      93              raise e
      94
      95 -    def connect_to_region(self, region_name):
      96          """
      97          Connects to a given region if it exists, raises RegionDoesNotExist
      98          otherwise. Once connected, this object will return only data from the
      99          given region.
     100          """
     101          region = self.get_region(region_name)
     102          self._kwargs['region'] = region
     103          self.reload()
     104          return self
     105
     106      @property
     107 -    def region(self):
     108          """
     109          Returns the current EC2 region used by this EasyEC2 object
     110          """
     111          return self.conn.region
     112
     113      @property
     114 -    def regions(self):
     115          """
     116          This property returns all AWS Regions, caching the results the first
     117          time a request is made to Amazon
     118          """
     119          if not self._regions:
     120              self._regions = {}
     121              regions = self.conn.get_all_regions()
     122              for region in regions:
     123                  self._regions[region.name] = region
     124          return self._regions
     125
     126 -    def get_region(self, region_name):
     127          """
     128          Returns boto Region object if it exists, raises RegionDoesNotExist
     129          otherwise.
     130          """
     131          if not region_name in self.regions:
     132              raise exception.RegionDoesNotExist(region_name)
     133          return self.regions.get(region_name)
     134
     135 -    def list_regions(self):
     136          """
     137          Print name/endpoint for all AWS regions
     138          """
     139          for r in self.regions:
     140              region = self.regions.get(r)
     141              print 'name: ', region.name
     142              print 'endpoint: ', region.endpoint
     143              print
     144
     145      @property
     146 -    def registered_images(self):
     147          if not self.cache or self._images is None:
     148              self._images = self.conn.get_all_images(owners=["self"])
     149          return self._images
     150
     151      @property
     152 -    def executable_images(self):
     153          if not self.cache or self._images is None:
     154              self._executable_images = self.conn.get_all_images(
     155                  executable_by=["self"])
     156          return self._executable_images
     157
     158 -    def get_registered_image(self, image_id):
     159          if not image_id.startswith('ami') or len(image_id) != 12:
     160              raise TypeError("invalid AMI name/id requested: %s" % image_id)
     161          for img in self.registered_images:
     162              if img.id == image_id:
     163                  return img
     164
     165 -    def create_group(self, name, description, auth_ssh=False,
     166                       auth_group_traffic=False):
     167          """
     168          Create security group with name/description. auth_ssh=True
     169          will open port 22 to world (0.0.0.0/0). auth_group_traffic
     170          will allow all traffic between instances in the same security
     171          group
     172          """
     173          if not name:
     174              return None
     175          log.info("Creating security group %s..." % name)
     176          sg = self.conn.create_security_group(name, description)
     177          if auth_ssh:
     178              sg.authorize('tcp', 22, 22, '0.0.0.0/0')
     179          if auth_group_traffic:
     180              sg.authorize('icmp', -1, -1,
     181                           src_group=self.get_group_or_none(name))
     182              sg.authorize('tcp', 1, 65535,
     183                           src_group=self.get_group_or_none(name))
     184              sg.authorize('udp', 1, 65535,
     185                           src_group=self.get_group_or_none(name))
     186          return sg
     187
     188 -    def get_all_security_groups(self, groupnames=[]):
     189          """
     190          Returns group with name if it exists otherwise returns None
     191          """
     192          try:
     193              return self.conn.get_all_security_groups(groupnames=groupnames)
     194          except boto.exception.EC2ResponseError, e:
     195              self.__check_for_auth_failure(e)
     196          except IndexError, e:
     197              pass
     198
     199 -    def get_group_or_none(self, name):
     200          """
     201          Returns group with name if it exists otherwise returns None
     202          """
     203          sgs = self.get_all_security_groups(groupnames=[name])
     204          if sgs:
     205              return sgs[0]
     206
     207 -    def get_or_create_group(self, name, description, auth_ssh=True,
     208                              auth_group_traffic=False):
     209          """
     210          Try to return a security group by name.
     211          If the group is not found, attempt to create it.
     212          Description only applies to creation.
     213
     214          Authorizes all traffic between members of the group
     215          """
     216          sg = self.get_group_or_none(name)
     217          if not sg:
     218              sg = self.create_group(name, description, auth_ssh,
     219                                       auth_group_traffic)
     220          return sg
     221
     222 -    def has_permission(self, group, ip_protocol, from_port, to_port, cidr_ip):
     223          """
     224          Checks whether group has the specified port range permission
     225          (ip_protocol, from_port, to_port, cidr_ip) defined
     226          """
     227          for rule in group.rules:
     228              if rule.ip_protocol != ip_protocol:
     229                  continue
     230              if int(rule.from_port) != from_port:
     231                  continue
     232              if int(rule.to_port) != to_port:
     233                  continue
     234              cidr_grants = [g for g in rule.grants if g.cidr_ip == cidr_ip]
     235              if not cidr_grants:
     236                  continue
     237              return True
     238          return False
     239
     240 -    def get_placement_group_or_none(self, name):
     241          """
     242          Returns placement group with name if it exists otherwise returns None
     243          """
     244          try:
     245              pg = self.conn.get_all_placement_groups(groupnames=[name])[0]
     246              return pg
     247          except boto.exception.EC2ResponseError, e:
     248              self.__check_for_auth_failure(e)
     249          except IndexError:
     250              pass
     251
     252 -    def create_placement_group(self, name):
     253          """
     254          Create a new placement group for your account.
     255          This will create the placement group within the region you
     256          are currently connected to.
     257          """
     258          if not name:
     259              return
     260          log.info("Creating placement group %s..." % name)
     261          success = self.conn.create_placement_group(name)
     262          if success:
     263              return self.get_placement_group_or_none(name)
     264
     265 -    def get_or_create_placement_group(self, name):
     266          """
     267          Try to return a placement group by name.
     268          If the group is not found, attempt to create it.
     269          """
     270          pg = self.get_placement_group_or_none(name)
     271          if not pg:
     272              pg = self.create_placement_group(name)
     273          return pg
     274
     275 -    def request_instances(self, image_id, price=None, instance_type='m1.small',
     276                            min_count=1, max_count=1, count=1, key_name=None,
     277                            security_groups=None, launch_group=None,
     278                            availability_zone_group=None, placement=None,
     279                            user_data=None, placement_group=None):
     280          """
     281          Convenience method for running spot or flat-rate instances
     282          """
     283          if price:
     284              return self.request_spot_instances(
     285                  price, image_id, instance_type=instance_type,
     286                  count=count, launch_group=launch_group, key_name=key_name,
     287                  security_groups=security_groups,
     288                  availability_zone_group=availability_zone_group,
     289                  placement=placement, user_data=user_data)
     290          else:
     291              return self.run_instances(
     292                  image_id, instance_type=instance_type,
     293                  min_count=min_count, max_count=max_count,
     294                  key_name=key_name, security_groups=security_groups,
     295                  placement=placement, user_data=user_data,
     296                  placement_group=placement_group)
     297
     298 -    def request_spot_instances(self, price, image_id, instance_type='m1.small',
     299                                 count=1, launch_group=None, key_name=None,
     300                                 availability_zone_group=None,
     301                                 security_groups=None,
     302                                 placement=None, user_data=None):
     303          return self.conn.request_spot_instances(
     304              price, image_id, instance_type=instance_type, count=count,
     305              launch_group=launch_group, key_name=key_name,
     306              security_groups=security_groups,
     307              availability_zone_group=availability_zone_group,
     308              placement=placement, user_data=user_data)
     309
     310 -    def run_instances(self, image_id, instance_type='m1.small', min_count=1,
     311                        max_count=1, key_name=None, security_groups=None,
     312                        placement=None, user_data=None, placement_group=None):
     313          return self.conn.run_instances(image_id, instance_type=instance_type,
     314                                         min_count=min_count,
     315                                         max_count=max_count,
     316                                         key_name=key_name,
     317                                         security_groups=security_groups,
     318                                         placement=placement,
     319                                         user_data=user_data,
     320                                         placement_group=placement_group)
     321
     322 -    def create_image(self, instance_id, name, description=None,
     323                       no_reboot=False):
     324          return self.conn.create_image(instance_id, name,
     325                                        description=description,
     326                                        no_reboot=no_reboot)
     327
     328 -    def register_image(self, name, description=None, image_location=None,
     329                         architecture=None, kernel_id=None, ramdisk_id=None,
     330                         root_device_name=None, block_device_map=None):
     331          return self.conn.register_image(name=name, description=description,
     332                                          image_location=image_location,
     333                                          architecture=architecture,
     334                                          kernel_id=kernel_id,
     335                                          ramdisk_id=ramdisk_id,
     336                                          root_device_name=root_device_name,
     337                                          block_device_map=block_device_map)
     338
     339 -    def delete_keypair(self, name):
     340          return self.conn.delete_key_pair(name)
     341
     342 -    def create_keypair(self, name, output_file=None):
     343          """
     344          Create a new EC2 keypair and optionally save to output_file
     345
     346          Returns boto.ec2.keypair.KeyPair
     347          """
     348          if output_file:
     349              output_dir = os.path.dirname(output_file)
     350              if output_dir and not os.path.exists(output_dir):
     351                  raise exception.BaseException(
     352                      "output directory does not exist")
     353              if os.path.exists(output_file):
     354                  raise exception.BaseException(
     355                      "cannot save keypair %s: file already exists" % \
     356                      output_file)
     357          kp = self.conn.create_key_pair(name)
     358          if output_file:
     359              try:
     360                  kfile = open(output_file, 'wb')
     361                  kfile.write(kp.material)
     362                  kfile.close()
     363                  os.chmod(output_file, 0400)
     364              except IOError, e:
     365                  raise exception.BaseException(str(e))
     366          return kp
     367
     368 -    def get_keypair(self, keypair):
     369          try:
     370              return self.conn.get_all_key_pairs(keynames=[keypair])[0]
     371          except boto.exception.EC2ResponseError, e:
     372              self.__check_for_auth_failure(e)
     373              raise exception.KeyPairDoesNotExist(keypair)
     374          except IndexError:
     375              raise exception.KeyPairDoesNotExist(keypair)
     376
     377 -    def get_keypair_or_none(self, keypair):
     378          try:
     379              return self.get_keypair(keypair)
     380          except:
     381              pass
     382
     383 -    def __print_header(self, msg):
     384          print msg
     385          print "-" * len(msg)
     386
     387 -    def get_image_name(self, img):
     388          image_name = re.sub('\.manifest\.xml$', '',
     389                              img.location.split('/')[-1])
     390          return image_name
     391
     392 -    def get_instance_user_data(self, instance_id):
     393          i = self.get_instance(instance_id)
     394          attributes = self.conn.get_instance_attribute(i.id, 'userData')
     395          user_data = attributes['userData'] or ''
     396          return base64.b64decode(user_data)
     397
     398 -    def get_instance(self, instance_id):
     399          try:
     400              res = self.conn.get_all_instances(
     401                  filters={'instance-id': instance_id})
     402              i = res[0].instances[0]
     403              # set group info
     404              i.groups = res[0].groups
     405              return i
     406          except boto.exception.EC2ResponseError, e:
     407              self.__check_for_auth_failure(e)
     408              raise exception.InstanceDoesNotExist(instance_id)
     409          except IndexError:
     410              # for eucalyptus, invalid instance_id returns []
     411              raise exception.InstanceDoesNotExist(instance_id)
     412
     413 -    def is_valid_conn(self):
     414          try:
     415              self.get_all_instances()
     416              return True
     417          except boto.exception.EC2ResponseError, e:
     418              if e.status in [401, 403]:
     419                  return False
     420              raise
     421
     422 -    def get_all_spot_requests(self, spot_ids=[], filters=None):
     423          spots = self.conn.get_all_spot_instance_requests(spot_ids,
     424                                                           filters=filters)
     425          return spots
     426
     427 -    def get_all_instances(self, instance_ids=[], filters=None):
     428          reservations = self.conn.get_all_instances(instance_ids,
     429                                                     filters=filters)
     430          instances = []
     431          for res in reservations:
     432              insts = res.instances
     433              for i in insts:
     434                  # set group info
     435                  i.groups = res.groups
     436              instances.extend(insts)
     437          return instances
     438
     439 -    def list_all_spot_instances(self, show_closed=False):
     440          s = self.conn.get_all_spot_instance_requests()
     441          if not s:
     442              log.info("No spot instance requests found...")
     443              return
     444          spots = []
     445          for spot in s:
     446              if spot.state in ['closed', 'cancelled'] and not show_closed:
     447                  continue
     448              state = spot.state or 'N/A'
     449              spot_id = spot.id or 'N/A'
     450              spots.append(spot_id)
     451              type = spot.type
     452              instance_id = spot.instance_id or 'N/A'
     453              create_time = spot.create_time or 'N/A'
     454              launch_group = spot.launch_group or 'N/A'
     455              zone_group = spot.availability_zone_group or 'N/A'
     456              price = spot.price or 'N/A'
     457              lspec = spot.launch_specification
     458              instance_type = lspec.instance_type
     459              image_id = lspec.image_id
     460              zone = lspec.placement
     461              groups = ', '.join([g.id for g in lspec.groups])
     462              print "id: %s" % spot_id
     463              print "price: $%0.2f" % price
     464              print "spot_request_type: %s" % type
     465              print "state: %s" % state
     466              print "instance_id: %s" % instance_id
     467              print "instance_type: %s" % instance_type
     468              print "image_id: %s" % image_id
     469              print "zone: %s" % zone
     470              print "create_time: %s" % create_time
     471              print "launch_group: %s" % launch_group
     472              print "zone_group: %s" % zone_group
     473              print "security_groups: %s" % groups
     474              print
     475          if not spots:
     476              log.info("No spot instance requests found...")
     477
     478 -    def list_all_instances(self, show_terminated=False):
     479          reservations = self.conn.get_all_instances()
     480          if not reservations:
     481              log.info("No instances found")
     482              return
     483          instances = []
     484          for res in reservations:
     485              groups = ', '.join([g.id for g in res.groups]) or 'N/A'
     486              for instance in res.instances:
     487                  if instance.state in ['shutting-down', 'terminated'] \
     488                     and not show_terminated:
     489                      continue
     490                  id = instance.id or 'N/A'
     491                  instances.append(id)
     492                  dns_name = instance.dns_name or 'N/A'
     493                  private_dns_name = instance.private_dns_name or 'N/A'
     494                  state = instance.state or 'N/A'
     495                  private_ip = instance.private_ip_address or 'N/A'
     496                  public_ip = instance.ip_address or 'N/A'
     497                  zone = instance.placement or 'N/A'
     498                  ami = instance.image_id or 'N/A'
     499                  instance_type = instance.instance_type or 'N/A'
     500                  keypair = instance.key_name or 'N/A'
     501                  uptime = utils.get_elapsed_time(instance.launch_time) or 'N/A'
     502                  print "id: %s" % id
     503                  print "dns_name: %s" % dns_name
     504                  print "private_dns_name: %s" % private_dns_name
     505                  print "state: %s" % state
     506                  print "public_ip: %s" % public_ip
     507                  print "private_ip: %s" % private_ip
     508                  print "zone: %s" % zone
     509                  print "ami: %s" % ami
     510                  print "type: %s" % instance_type
     511                  print "groups: %s" % groups
     512                  print "keypair: %s" % keypair
     513                  print "uptime: %s" % uptime
     514                  print
     515          if not instances:
     516              log.info("No instances found")
     517
     518 -    def list_images(self, images, sort_key=None, reverse=False):
     519          def get_key(obj):
     520              return ' '.join([obj.region.name, obj.location])
     521          if not sort_key:
     522              sort_key = get_key
     523          imgs_i386 = [img for img in images if img.architecture == "i386"]
     524          imgs_i386.sort(key=sort_key, reverse=reverse)
     525          imgs_x86_64 = [img for img in images if img.architecture == "x86_64"]
     526          imgs_x86_64.sort(key=sort_key, reverse=reverse)
     527          print
     528          self.__list_images("32bit Images:", imgs_i386)
     529          self.__list_images("\n64bit Images:", imgs_x86_64)
     530          print "\ntotal images: %d" % len(images)
     531          print
     532
     533 -    def list_registered_images(self):
     534          images = self.registered_images
     535          log.info("Your registered images:")
     536          self.list_images(images)
     537
     538 -    def list_executable_images(self):
     539          images = self.executable_images
     540          log.info("Private images owned by other users that you can execute:")
     541          self.list_images(images)
     542
     543 -    def __list_images(self, msg, imgs):
     544          counter = 0
     545          self.__print_header(msg)
     546          for img in imgs:
     547              name = self.get_image_name(img)
     548              template = "[%d] %s %s %s"
     549              if img.virtualization_type == 'hvm':
     550                  template += ' (HVM-EBS)'
     551              elif img.root_device_type == 'ebs':
     552                  template += ' (EBS)'
     553              print template % (counter, img.id, img.region.name, name)
     554              counter += 1
     555
     556 -    def remove_image_files(self, image_name, pretend=True):
     557          files = self.get_image_files(image_name)
     558          for file in files:
     559              if pretend:
     560                  print file
     561              else:
     562                  print 'removing file %s' % file
     563                  file.delete()
     564
     565          # recursive double check
     566          files = self.get_image_files(image_name)
     567          if len(files) != 0:
     568              if pretend:
     569                  log.info('Not all files deleted, would recurse...exiting')
     570                  return
     571              else:
     572                  log.info('Not all files deleted, recursing...')
     573                  self.remove_image_files(image_name, pretend)
     574
     575      @print_timing("Removing image")
     576 -    def remove_image(self, image_name, pretend=True):
     577          img = self.get_image(image_name)
     578          if pretend:
     579              log.info("Pretending to remove AMI: %s" % image_name)
     580          else:
     581              log.info("Removing AMI: %s" % image_name)
     582
     583          # first remove image files
     584          log.info('Removing image files...')
     585          self.remove_image_files(image_name, pretend=pretend)
     586
     587          # then deregister ami
     588          if pretend:
     589              log.info('Would run deregister_image for ami: %s)' % img.id)
     590          else:
     591              log.info('Deregistering ami: %s' % img.id)
     592              img.deregister()
     593
     594 -    def list_starcluster_public_images(self):
     595          images = self.conn.get_all_images(owners=[static.STARCLUSTER_OWNER_ID])
     596          log.info("Listing all public StarCluster images...")
     597          imgs = [img for img in images if img.is_public]
     598
     599          def sc_public_sort(obj):
     600              split = obj.name.split('-')
     601              osname, osversion, arch = split[2:5]
     602              osversion = float(osversion)
     603              rc = 0
     604              if split[-1].startswith('rc'):
     605                  rc = int(split[-1].replace('rc', ''))
     606              return (osversion, rc)
     607          self.list_images(imgs, sort_key=sc_public_sort, reverse=True)
     608
     609 -    def create_volume(self, size, zone, snapshot_id=None):
     610          return self.conn.create_volume(size, zone, snapshot_id)
     611
     612 -    def remove_volume(self, volume_id):
     613          vol = self.get_volume(volume_id)
     614          vol.delete()
     615
     616 -    def list_keypairs(self):
     617          max_length = max([len(key.name) for key in self.keypairs])
     618          templ = "%" + str(max_length) + "s  %s"
     619          for key in self.keypairs:
     620              print templ % (key.name, key.fingerprint)
     621
     622 -    def list_zones(self, region=None):
     623          conn = self.conn
     624          if region:
     625              regs = self.conn.get_all_regions()
     626              regions = [r.name for r in regs]
     627              if not region in regions:
     628                  raise exception.RegionDoesNotExist(region)
     629              for reg in regs:
     630                  if reg.name == region:
     631                      region = reg
     632                      break
     633              kwargs = {}
     634              kwargs.update(self._kwargs)
     635              kwargs.update(dict(region=region))
     636              conn = self.connection_authenticator(
     637                  self.aws_access_key_id, self.aws_secret_access_key, **kwargs)
     638          for zone in conn.get_all_zones():
     639              print 'name: ', zone.name
     640              print 'region: ', zone.region.name
     641              print 'status: ', zone.state
     642              print
     643
     644 -    def get_zone(self, zone):
     645          """
     646          Return zone object respresenting an EC2 availability zone
     647          Raises exception.ZoneDoesNotExist if not successful
     648          """
     649          try:
     650              return self.conn.get_all_zones(zones=[zone])[0]
     651          except boto.exception.EC2ResponseError, e:
     652              self.__check_for_auth_failure(e)
     653              raise exception.ZoneDoesNotExist(zone, self.region.name)
     654          except IndexError:
     655              raise exception.ZoneDoesNotExist(zone, self.region.name)
     656
     657 -    def get_zone_or_none(self, zone):
     658          """
     659          Return zone object respresenting an EC2 availability zone
     660          Returns None if unsuccessful
     661          """
     662          try:
     663              return self.get_zone(zone)
     664          except exception.ZoneDoesNotExist:
     665              pass
     666
     667 -    def create_s3_image(self, instance_id, key_location, aws_user_id,
     668                          ec2_cert, ec2_private_key, bucket, image_name="image",
     669                          description=None, kernel_id=None, ramdisk_id=None,
     670                          remove_image_files=False, **kwargs):
     671          """
     672          Create instance-store (S3) image from running instance
     673          """
     674          icreator = image.S3ImageCreator(self, instance_id, key_location,
     675                                          aws_user_id, ec2_cert,
     676                                          ec2_private_key, bucket,
     677                                          image_name=image_name,
     678                                          description=description,
     679                                          kernel_id=kernel_id,
     680                                          ramdisk_id=ramdisk_id,
     681                                          remove_image_files=remove_image_files)
     682          return icreator.create_image()
     683
     684 -    def create_ebs_image(self, instance_id, key_location, name,
     685                           description=None, snapshot_description=None,
     686                           kernel_id=None, ramdisk_id=None, root_vol_size=15,
     687                           **kwargs):
     688          """
     689          Create EBS-backed image from running instance
     690          """
     691          sdescription = snapshot_description
     692          icreator = image.EBSImageCreator(self, instance_id, key_location,
     693                                           name, description=description,
     694                                           snapshot_description=sdescription,
     695                                           kernel_id=kernel_id,
     696                                           ramdisk_id=ramdisk_id,
     697                                           **kwargs)
     698          return icreator.create_image(size=root_vol_size)
     699
     700 -    def get_image(self, image_id):
     701          """
     702          Return image object representing an AMI.
     703          Raises exception.AMIDoesNotExist if unsuccessful
     704          """
     705          try:
     706              return self.conn.get_all_images(image_ids=[image_id])[0]
     707          except boto.exception.EC2ResponseError, e:
     708              self.__check_for_auth_failure(e)
     709              raise exception.AMIDoesNotExist(image_id)
     710          except IndexError:
     711              raise exception.AMIDoesNotExist(image_id)
     712
     713 -    def get_image_or_none(self, image_id):
     714          """
     715          Return image object representing an AMI.
     716          Returns None if unsuccessful
     717          """
     718          try:
     719              return self.get_image(image_id)
     720          except:
     721              pass
     722
     723 -    def _get_image_files(self, image, bucket):
     724          """
     725          """
     726          bname = re.escape(bucket.name)
     727          prefix = re.sub('^%s\/' % bname, '', image.location)
     728          prefix = re.sub('\.manifest\.xml$', '', prefix)
     729          files = bucket.list(prefix=prefix)
     730          manifest_regex = re.compile(r'%s\.manifest\.xml' % prefix)
     731          part_regex = re.compile(r'%s\.part\.(\d*)' % prefix)
     732          # boto with eucalyptus returns boto.s3.prefix.Prefix class at the
     733          # end of the list, we ignore these by checking for delete attr
     734          files = [f for f in files if hasattr(f, 'delete') and
     735                   part_regex.match(f.name) or manifest_regex.match(f.name)]
     736          return files
     737
     738 -    def get_image_files(self, image_id):
     739          """
     740          Return list of files on S3 for image_id
     741          The list includes the image's manifest and part files
     742          """
     743          image = self.get_image(image_id)
     744          if image.root_device_type == 'ebs':
     745              raise exception.AWSError(
     746                  "Image %s is an EBS image. No image files on S3." % image_id)
     747          bucket = self.get_image_bucket(image)
     748          return self._get_image_files(image, bucket)
     749
     750 -    def get_image_bucket(self, image):
     751          bucket_name = image.location.split('/')[0]
     752          return self.s3.get_bucket(bucket_name)
     753
     754 -    def get_image_manifest(self, image):
     755          return image.location.split('/')[-1]
     756
     757      @print_timing("Migrating image")
     758 -    def migrate_image(self, image_id, destbucket, migrate_manifest=False,
     759                        kernel_id=None, ramdisk_id=None, region=None, cert=None,
     760                        private_key=None):
     761          """
     762          Migrate image_id files to destbucket
     763          """
     764          if migrate_manifest:
     765              utils.check_required(['ec2-migrate-manifest'])
     766              if not cert:
     767                  raise exception.BaseException("no cert specified")
     768              if not private_key:
     769                  raise exception.BaseException("no private_key specified")
     770              if not kernel_id:
     771                  raise exception.BaseException("no kernel_id specified")
     772              if not ramdisk_id:
     773                  raise exception.BaseException("no ramdisk_id specified")
     774          image = self.get_image(image_id)
     775          if image.root_device_type == "ebs":
     776              raise exception.AWSError(
     777                  "The image you wish to migrate is EBS-based. " +
     778                  "This method only works for instance-store images")
     779          ibucket = self.get_image_bucket(image)
     780          files = self._get_image_files(image, ibucket)
     781          if not files:
     782              log.info("No files found for image: %s" % image_id)
     783              return
     784          log.info("Migrating image: %s" % image_id)
     785          widgets = [files[0].name, progressbar.Percentage(), ' ',
     786                     progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
     787                     progressbar.ETA(), ' ', ' ']
     788          counter = 0
     789          num_files = len(files)
     790          pbar = progressbar.ProgressBar(widgets=widgets,
     791                                         maxval=num_files).start()
     792          for f in files:
     793              widgets[0] = "%s: (%s/%s)" % (f.name, counter + 1, num_files)
     794              # copy file to destination bucket with the same name
     795              f.copy(destbucket, f.name)
     796              pbar.update(counter)
     797              counter += 1
     798          pbar.finish()
     799          if migrate_manifest:
     800              dbucket = self.s3.get_bucket(destbucket)
     801              manifest_key = dbucket.get_key(self.get_image_manifest(image))
     802              f = tempfile.NamedTemporaryFile()
     803              manifest_key.get_contents_to_file(f.file)
     804              f.file.close()
     805              cmd = ('ec2-migrate-manifest -c %s -k %s -m %s --kernel %s ' +
     806                     '--ramdisk %s --no-mapping ') % (cert, private_key,
     807                                                      f.name, kernel_id,
     808                                                      ramdisk_id)
     809              register_cmd = "ec2-register %s/%s" % (destbucket,
     810                                                     manifest_key.name)
     811              if region:
     812                  cmd += '--region %s' % region
     813                  register_cmd += " --region %s" % region
     814              log.info("Migrating manifest file...")
     815              retval = os.system(cmd)
     816              if retval != 0:
     817                  raise exception.BaseException(
     818                      "ec2-migrate-manifest failed with status %s" % retval)
     819              f.file = open(f.name, 'r')
     820              manifest_key.set_contents_from_file(f.file)
     821              # needed so that EC2 has permission to READ the manifest file
     822              manifest_key.add_email_grant('READ', 'za-team@amazon.com')
     823              f.close()
     824              os.unlink(f.name + '.bak')
     825              log.info("Manifest migrated successfully. You can now run:\n" +
     826                       register_cmd + "\nto register your migrated image.")
     827
     828 -    def create_root_block_device_map(self, snapshot_id,
     829                                       root_device_name='/dev/sda1',
     830                                       add_ephemeral_drives=False,
     831                                       ephemeral_drive_0='/dev/sdb1',
     832                                       ephemeral_drive_1='/dev/sdc1',
     833                                       ephemeral_drive_2='/dev/sdd1',
     834                                       ephemeral_drive_3='/dev/sde1'):
     835          """
     836          Utility method for building a new block_device_map for a given snapshot
     837          id. This is useful when creating a new image from a volume snapshot.
     838          The returned block device map can be used with self.register_image
     839          """
     840          bmap = boto.ec2.blockdevicemapping.BlockDeviceMapping()
     841          sda1 = boto.ec2.blockdevicemapping.BlockDeviceType()
     842          sda1.snapshot_id = snapshot_id
     843          sda1.delete_on_termination = True
     844          bmap[root_device_name] = sda1
     845          if add_ephemeral_drives:
     846              sdb1 = boto.ec2.blockdevicemapping.BlockDeviceType()
     847              sdb1.ephemeral_name = 'ephemeral0'
     848              bmap[ephemeral_drive_0] = sdb1
     849              sdc1 = boto.ec2.blockdevicemapping.BlockDeviceType()
     850              sdc1.ephemeral_name = 'ephemeral1'
     851              bmap[ephemeral_drive_1] = sdc1
     852              sdd1 = boto.ec2.blockdevicemapping.BlockDeviceType()
     853              sdd1.ephemeral_name = 'ephemeral2'
     854              bmap[ephemeral_drive_2] = sdd1
     855              sde1 = boto.ec2.blockdevicemapping.BlockDeviceType()
     856              sde1.ephemeral_name = 'ephemeral3'
     857              bmap[ephemeral_drive_3] = sde1
     858          return bmap
     859
     860      @print_timing("Downloading image")
     861 -    def download_image_files(self, image_id, destdir):
     862          """
     863          Downloads the manifest.xml and all AMI parts for image_id to destdir
     864          """
     865          if not os.path.isdir(destdir):
     866              raise exception.BaseException(
     867                  "destination directory '%s' does not exist" % destdir)
     868          widgets = ['file: ', progressbar.Percentage(), ' ',
     869                     progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
     870                     progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
     871          files = self.get_image_files(image_id)
     872
     873          def _dl_progress_cb(trans, total):
     874              pbar.update(trans)
     875          log.info("Downloading image: %s" % image_id)
     876          for file in files:
     877              widgets[0] = "%s:" % file.name
     878              pbar = progressbar.ProgressBar(widgets=widgets,
     879                                             maxval=file.size).start()
     880              file.get_contents_to_filename(os.path.join(destdir, file.name),
     881                                            cb=_dl_progress_cb)
     882              pbar.finish()
     883
     884 -    def list_image_files(self, image_id):
     885          """
     886          Print a list of files for image_id to the screen
     887          """
     888          files = self.get_image_files(image_id)
     889          for file in files:
     890              print file.name
     891
     892      @property
     893 -    def instances(self):
     894          if not self.cache or self._instance_response is None:
     895              log.debug('instance_response = %s, cache = %s' %
     896              (self._instance_response, self.cache))
     897              self._instance_response = self.conn.get_all_instances()
     898          return self._instance_response
     899
     900      @property
     901 -    def keypairs(self):
     902          if not self.cache or self._keypair_response is None:
     903              log.debug('keypair_response = %s, cache = %s' %
     904              (self._keypair_response, self.cache))
     905              self._keypair_response = self.conn.get_all_key_pairs()
     906          return self._keypair_response
     907
     908 -    def terminate_instances(self, instances=None):
     909          if instances:
     910              self.conn.terminate_instances(instances)
     911
     912 -    def get_volumes(self, filters=None):
     913          """
     914          Returns a list of all EBS volumes
     915          """
     916          try:
     917              return self.conn.get_all_volumes(filters=filters)
     918          except boto.exception.EC2ResponseError, e:
     919              self.__check_for_auth_failure(e)
     920
     921 -    def get_volume(self, volume_id):
     922          """
     923          Returns EBS volume object representing volume_id.
     924          Raises exception.VolumeDoesNotExist if unsuccessful
     925          """
     926          try:
     927              return self.conn.get_all_volumes(
     928                  filters={'volume-id': volume_id})[0]
     929          except boto.exception.EC2ResponseError, e:
     930              self.__check_for_auth_failure(e)
     931              raise exception.VolumeDoesNotExist(volume_id)
     932          except IndexError:
     933              raise exception.VolumeDoesNotExist(volume_id)
     934
     935 -    def wait_for_snapshot(self, snapshot, refresh_interval=30):
     936          snap = snapshot
     937          log.info("Waiting for snapshot to complete: %s" % snap.id)
     938          widgets = ['%s: ' % snap.id, '',
     939                     progressbar.Bar(marker=progressbar.RotatingMarker()),
     940                     '', progressbar.Percentage(), ' ', progressbar.ETA()]
     941          pbar = progressbar.ProgressBar(widgets=widgets, maxval=100).start()
     942          while snap.status != 'completed':
     943              try:
     944                  progress = int(snap.update().replace('%', ''))
     945                  pbar.update(progress)
     946              except ValueError:
     947                  time.sleep(5)
     948                  continue
     949              time.sleep(refresh_interval)
     950          pbar.finish()
     951
     952 -    def create_snapshot(self, vol, description=None, wait_for_snapshot=False,
     953                          refresh_interval=30):
     954          log.info("Creating snapshot of volume: %s" % vol.id)
     955          snap = vol.create_snapshot(description)
     956          if wait_for_snapshot:
     957              self.wait_for_snapshot(snap, refresh_interval)
     958          return snap
     959
     960 -    def get_snapshots(self, volume_ids=[]):
     961          """
     962          Returns a list of all EBS volume snapshots for this account
     963          """
     964          filters = {}
     965          if volume_ids:
     966              filters['volume-id'] = volume_ids
     967          try:
     968              return self.conn.get_all_snapshots(owner='self', filters=filters)
     969          except boto.exception.EC2ResponseError, e:
     970              self.__check_for_auth_failure(e)
     971
     972 -    def get_snapshot(self, snapshot_id):
     973          """
     974          Returns EBS snapshot object representing snapshot_id.
     975          Raises exception.SnapshotDoesNotExist if unsuccessful
     976          """
     977          try:
     978              return self.conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
     979          except boto.exception.EC2ResponseError, e:
     980              self.__check_for_auth_failure(e)
     981              raise exception.SnapshotDoesNotExist(snapshot_id)
     982          except IndexError:
     983              raise exception.SnapshotDoesNotExist(snapshot_id)
     984
     985 -    def get_volume_or_none(self, volume_id):
     986          """
     987          Returns EBS volume object representing volume_id.
     988          Returns none if unsuccessful
     989          """
     990          try:
     991              return self.get_volume(volume_id)
     992          except:
     993              pass
     994
     995 -    def list_volumes(self, volume_id=None, status=None,
     996                       attach_status=None, size=None, zone=None,
     997                       snapshot_id=None, show_deleted=False):
     998          """
     999          Print a list of volumes to the screen
    1000          """
    1001          filters = {}
    1002          if status:
    1003              filters['status'] = status
    1004          else:
    1005              filters['status'] = ['creating', 'available', 'in-use', 'error']
    1006              if show_deleted:
    1007                  filters['status'] += ['deleting', 'deleted']
    1008          if attach_status:
    1009              filters['attachment.status'] = attach_status
    1010          if volume_id:
    1011              filters['volume-id'] = volume_id
    1012          if size:
    1013              filters['size'] = size
    1014          if zone:
    1015              filters['availability-zone'] = zone
    1016          if snapshot_id:
    1017              filters['snapshot-id'] = snapshot_id
    1018          vols = self.get_volumes(filters=filters)
    1019          vols.sort(key=lambda x: x.create_time)
    1020          if vols:
    1021              for vol in vols:
    1022                  print "volume_id: %s" % vol.id
    1023                  print "size: %sGB" % vol.size
    1024                  print "status: %s" % vol.status
    1025                  if vol.attachment_state():
    1026                      print "attachment_status: %s" % vol.attachment_state()
    1027                  print "availability_zone: %s" % vol.zone
    1028                  if vol.snapshot_id:
    1029                      print "snapshot_id: %s" % vol.snapshot_id
    1030                  snapshots = self.get_snapshots(volume_ids=[vol.id])
    1031                  if snapshots:
    1032                      snap_list = ' '.join([snap.id for snap in snapshots])
    1033                      print 'snapshots: %s' % snap_list
    1034                  if vol.create_time:
    1035                      lt = utils.iso_to_localtime_tuple(vol.create_time)
    1036                  print "create_time: %s" % lt
    1037                  print
    1038          print 'Total: %s' % len(vols)
    1039
    1040 -    def get_security_group(self, groupname):
    1041          try:
    1042              return self.conn.get_all_security_groups(filters={'group-name':
    1043                                                                groupname})[0]
    1044          except boto.exception.EC2ResponseError, e:
    1045              self.__check_for_auth_failure(e)
    1046              raise exception.SecurityGroupDoesNotExist(groupname)
    1047          except IndexError:
    1048              raise exception.SecurityGroupDoesNotExist(groupname)
    1049
    1050 -    def get_security_groups(self, filters=None):
    1051          return self.conn.get_all_security_groups(filters=filters)
    1052
    1053 -    def get_spot_history(self, instance_type,
    1054                           start=None, end=None, plot=False):
    1055          if not utils.is_iso_time(start):
    1056              raise exception.InvalidIsoDate(start)
    1057          if not utils.is_iso_time(end):
    1058              raise exception.InvalidIsoDate(end)
    1059          hist = self.conn.get_spot_price_history(start_time=start,
    1060                                          end_time=end,
    1061                                          instance_type=instance_type,
    1062                                          product_description="Linux/UNIX")
    1063          if not hist:
    1064              raise exception.SpotHistoryError(start, end)
    1065          dates = [utils.iso_to_datetime_tuple(i.timestamp) for i in hist]
    1066          prices = [i.price for i in hist]
    1067          maximum = max(prices)
    1068          avg = sum(prices) / len(prices)
    1069          log.info("Current price: $%.2f" % hist[-1].price)
    1070          log.info("Max price: $%.2f" % maximum)
    1071          log.info("Average price: $%.2f" % avg)
    1072          if plot:
    1073              try:
    1074                  import pylab
    1075                  pylab.plot_date(pylab.date2num(dates), prices, linestyle='-')
    1076                  pylab.xlabel('Date')
    1077                  pylab.ylabel('Price (US Dollars)')
    1078                  pylab.title('%s Price vs Date (%s - %s)' % (instance_type,
    1079                                                              start, end))
    1080                  xmin, xmax = pylab.xlim()
    1081                  ymin, ymax = pylab.ylim()
    1082                  pylab.xlim([xmin - 1, xmax + 1])
    1083                  pylab.ylim([0, ymax * (1.02)])
    1084                  pylab.grid(True)
    1085                  pylab.show()
    1086              except ImportError, e:
    1087                  log.error("Error importing pylab:")
    1088                  log.error(str(e))
    1089                  log.error("please ensure matplotlib is installed and that:")
    1090                  log.error("   $ python -c 'import pylab'")
    1091                  log.error("completes without error")
    1092          return zip(dates, prices)
    1093
    1094 -    def show_console_output(self, instance_id):
    1095          instance = self.get_instance(instance_id)
    1096          console_output = instance.get_console_output().output
    1097          print ''.join([c for c in console_output if c in string.printable])
    1098
    1099
    1100 -class EasyS3(EasyAWS):
    1101      DefaultHost = 's3.amazonaws.com'
    1102      _calling_format = boto.s3.connection.OrdinaryCallingFormat()
    1103
    1104 -    def __init__(self, aws_access_key_id, aws_secret_access_key,
    1105                   aws_s3_path='/', aws_port=None, aws_is_secure=True,
    1106                   aws_s3_host=DefaultHost, cache=False, **kwargs):
    1107          kwargs = dict(is_secure=aws_is_secure,
    1108                        host=aws_s3_host or self.DefaultHost,
    1109                        port=aws_port,
    1110                        path=aws_s3_path)
    1111          if aws_s3_host:
    1112              kwargs.update(dict(calling_format=self._calling_format))
    1113          super(EasyS3, self).__init__(aws_access_key_id, aws_secret_access_key,
    1114                                       boto.connect_s3, **kwargs)
    1115          self.cache = cache
    1116
    1117 -    def __repr__(self):
    1118          return '<EasyS3: %s>' % self.conn.server_name()
    1119
    1120 -    def __check_for_auth_failure(self, e):
    1121          if e.error_code == "InvalidAccessKeyId":
    1122              raise e
    1123
    1124 -    def create_bucket(self, bucket_name):
    1125          """
    1126          Create a new bucket on S3. bucket_name must be unique, the bucket
    1127          namespace is shared by all AWS users
    1128          """
    1129          bucket_name = bucket_name.split('/')[0]
    1130          try:
    1131              return self.conn.create_bucket(bucket_name)
    1132          except boto.exception.S3CreateError, e:
    1133              self.__check_for_auth_failure(e)
    1134              if e.error_code == "BucketAlreadyExists":
    1135                  raise exception.BucketAlreadyExists(bucket_name)
    1136              raise
    1137
    1138 -    def bucket_exists(self, bucket_name):
    1139          """
    1140          Check if bucket_name exists on S3
    1141          """
    1142          try:
    1143              self.conn.get_bucket(bucket_name)
    1144              return True
    1145          except boto.exception.S3ResponseError, e:
    1146              self.__check_for_auth_failure(e)
    1147              return False
    1148
    1149 -    def get_bucket_or_none(self, bucket_name):
    1150          """
    1151          Returns bucket object representing S3 bucket
    1152          Returns None if unsuccessful
    1153          """
    1154          try:
    1155              return self.get_bucket(bucket_name)
    1156          except exception.BucketDoesNotExist:
    1157              pass
    1158
    1159 -    def get_bucket(self, bucketname):
    1160          """
    1161          Returns bucket object representing S3 bucket
    1162          """
    1163          try:
    1164              return self.conn.get_bucket(bucketname)
    1165          except boto.exception.S3ResponseError, e:
    1166              self.__check_for_auth_failure(e)
    1167              raise exception.BucketDoesNotExist(bucketname)
    1168
    1169 -    def list_bucket(self, bucketname):
    1170          bucket = self.get_bucket(bucketname)
    1171          for file in bucket.list():
    1172              if file.name:
    1173                  print file.name
    1174
    1175 -    def get_buckets(self):
    1176          try:
    1177              buckets = self.conn.get_all_buckets()
    1178          except TypeError:
    1179              #hack until boto fixes get_all_buckets
    1180              raise exception.AWSError("AWS credentials are not valid")
    1181          return buckets
    1182
    1183 -    def list_buckets(self):
    1184          for bucket in self.get_buckets():
    1185              print bucket.name
    1186
    1187 -    def get_bucket_files(self, bucketname):
    1188          files = []
    1189          try:
    1190              bucket = self.get_bucket(bucketname)
    1191              files = [file for file in bucket.list()]
    1192          except:
    1193              pass
    1194          return files
    1195
    1196  if __name__ == "__main__":
    1197      from starcluster.config import get_easy_ec2
    1198      ec2 = get_easy_ec2()
    1199      ec2.list_all_instances()
    1200      ec2.list_registered_images()
    1201