starcluster.image

Package starcluster :: Module image |
[hide private]
[frames] | no frames]

# Source Code for Module starcluster.image

      1  #!/usr/bin/env python
      2  import os
      3  import time
      4  import string
      5
      6  from starcluster import ssh
      7  from starcluster import utils
      8  from starcluster import exception
      9  from starcluster.spinner import Spinner
     10  from starcluster.utils import print_timing
     11  from starcluster.logger import log
     12
     13
     14 -class ImageCreator(object):
     15      """
     16      Base class for S3/EBS Image Creators. Handles fetching the host and setting
     17      up a connection object as well as setting common attributes (description,
     18      kernel_id, ramdisk_id)
     19
     20      easy_ec2 must be an awsutils.EasyEC2 object
     21
     22      instance_id is the id of the instance to be imaged
     23
     24      key_location must point to the private key file corresponding to the
     25      keypair used to launch instance_id
     26      """
     27 -    def __init__(self, easy_ec2, instance_id, key_location, description=None,
     28                   kernel_id=None, ramdisk_id=None):
     29          self.ec2 = easy_ec2
     30          self.host = self.ec2.get_instance(instance_id)
     31          if self.host.state != 'running':
     32              raise exception.InstanceNotRunning(self.host.id, self.host.state,
     33                                                 self.host.dns_name)
     34          self.host_ssh = ssh.SSHClient(self.host.dns_name, username='root',
     35                                        private_key=key_location)
     36          self.description = description
     37          self.kernel_id = kernel_id or self.host.kernel
     38          self.ramdisk_id = ramdisk_id or self.host.ramdisk
     39
     40 -    def clean_private_data(self):
     41          log.info('Removing private data...')
     42          conn = self.host_ssh
     43          conn.execute('find /home -maxdepth 1 -type d -exec rm -rf {}/.ssh \;')
     44          conn.execute('rm -f /var/log/secure')
     45          conn.execute('rm -f /var/log/lastlog')
     46          conn.execute('rm -rf /root/*')
     47          conn.execute('rm -f ~/.bash_history')
     48          conn.execute('rm -rf /tmp/*')
     49          conn.execute('rm -rf /root/*.hist*')
     50          conn.execute('rm -rf /var/log/*.gz')
     51
     52
     53 -class S3ImageCreator(ImageCreator):
     54      """
     55      Class for creating a new instance-store AMI from a running instance
     56      """
     57 -    def __init__(self, easy_ec2, instance_id, key_location, aws_user_id,
     58                   ec2_cert, ec2_private_key, bucket, image_name='image',
     59                   description=None, kernel_id=None, ramdisk_id=None,
     60                   remove_image_files=False, **kwargs):
     61          super(S3ImageCreator, self).__init__(easy_ec2, instance_id,
     62                                               key_location, description,
     63                                               kernel_id, ramdisk_id)
     64          self.userid = aws_user_id
     65          self.cert = ec2_cert
     66          self.private_key = ec2_private_key
     67          self.bucket = bucket
     68          self.prefix = image_name
     69          self.description = description
     70          self.remove_image_files = remove_image_files
     71          for name in self.bucket.split("/"):
     72              if not utils.is_valid_bucket_name(name):
     73                  raise exception.InvalidBucketName(self.bucket)
     74          if not utils.is_valid_image_name(self.prefix):
     75              raise exception.InvalidImageName(self.prefix)
     76          if not self.cert:
     77              try:
     78                  self.cert = os.environ['EC2_CERT']
     79              except KeyError:
     80                  raise exception.EC2CertRequired()
     81          if not self.private_key:
     82              try:
     83                  self.private_key = os.environ['EC2_PRIVATE_KEY']
     84              except KeyError:
     85                  raise exception.EC2PrivateKeyRequired()
     86          if not self.userid:
     87              raise exception.AWSUserIdRequired()
     88          if not os.path.exists(self.cert):
     89              raise exception.EC2CertDoesNotExist(self.cert)
     90          if not os.path.exists(self.private_key):
     91              raise exception.EC2PrivateKeyDoesNotExist(self.private_key)
     92          self.config_dict = {
     93              'access_key': self.ec2.aws_access_key_id,
     94              'secret_key': self.ec2.aws_secret_access_key,
     95              'private_key': os.path.split(self.private_key)[-1],
     96              'userid': self.userid,
     97              'cert': os.path.split(self.cert)[-1],
     98              'bucket': self.bucket,
     99              'prefix': self.prefix,
    100              'arch': self.host.architecture,
    101          }
    102
    103 -    def __repr__(self):
    104          return "<S3ImageCreator: %s>" % self.host.id
    105
    106      @print_timing
    107 -    def create_image(self):
    108          log.info("Creating bucket %s" % self.bucket)
    109          self.ec2.s3.create_bucket(self.bucket)
    110          log.info("Checking for EC2 API tools...")
    111          self.host_ssh.check_required(['ec2-upload-bundle', 'ec2-bundle-vol'])
    112          self._remove_image_files()
    113          self._bundle_image()
    114          self._upload_image()
    115          ami_id = self._register_image()
    116          if self.remove_image_files:
    117              self._remove_image_files()
    118          return ami_id
    119
    120 -    def _remove_image_files(self):
    121          conn = self.host_ssh
    122          conn.execute('umount /mnt/img-mnt', ignore_exit_status=True)
    123          conn.execute('rm -rf /mnt/img-mnt')
    124          conn.execute('rm -rf /mnt/%(prefix)s*' % self.config_dict)
    125
    126 -    def _transfer_pem_files(self):
    127          """copy pem files to /mnt on image host"""
    128          conn = self.host_ssh
    129          pkey_dest = "/mnt/" + os.path.basename(self.private_key)
    130          cert_dest = "/mnt/" + os.path.basename(self.cert)
    131          conn.put(self.private_key, pkey_dest)
    132          conn.put(self.cert, cert_dest)
    133
    134      @print_timing
    135 -    def _bundle_image(self):
    136          # run script to prepare the host
    137          conn = self.host_ssh
    138          config_dict = self.config_dict
    139          self._transfer_pem_files()
    140          self.clean_private_data()
    141          log.info('Creating the bundled image: (please be patient)')
    142          conn.execute('ec2-bundle-vol -d /mnt -k /mnt/%(private_key)s \
    143  -c /mnt/%(cert)s -p %(prefix)s -u %(userid)s -r %(arch)s -e /root/.ssh' % \
    144                       config_dict,
    145                       silent=False)
    146          self._cleanup_pem_files()
    147
    148      @print_timing
    149 -    def _upload_image(self):
    150          log.info('Uploading bundled image: (please be patient)')
    151          conn = self.host_ssh
    152          config_dict = self.config_dict
    153          conn.execute('ec2-upload-bundle -b %(bucket)s \
    154  -m /mnt/%(prefix)s.manifest.xml -a %(access_key)s -s %(secret_key)s' % \
    155                       config_dict, silent=False)
    156
    157 -    def _cleanup(self):
    158          #just in case...
    159          self._cleanup_pem_files()
    160          conn = self.host_ssh
    161          conn.execute('rm -f ~/.bash_history', silent=False)
    162
    163 -    def _cleanup_pem_files(self):
    164          log.info('Cleaning up...')
    165          # delete keys and remove bash history
    166          conn = self.host_ssh
    167          conn.execute('rm -f /mnt/*.pem /mnt/*.pem', silent=False)
    168
    169 -    def _register_image(self):
    170          # register image in s3 with ec2
    171          conn = self.ec2
    172          config_dict = self.config_dict
    173          return conn.register_image(
    174              self.prefix,
    175              description=self.description,
    176              image_location="%(bucket)s/%(prefix)s.manifest.xml" % config_dict,
    177              kernel_id=self.kernel_id,
    178              ramdisk_id=self.ramdisk_id,
    179              architecture=config_dict.get('arch'),
    180          )
    181
    182
    183 -class EBSImageCreator(ImageCreator):
    184      """
    185      Creates a new EBS image from a running instance
    186
    187      If the instance is an instance-store image, then this class will create a
    188      new volume, attach it to the instance, sync the root filesystem to the
    189      volume, detach the volume, snapshot it, and then create a new AMI from the
    190      snapshot
    191
    192      If the instance is EBS-backed, this class simply calls ec2.create_image
    193      which tells Amazon to create a new image in a single API call.
    194      """
    195
    196 -    def __init__(self, easy_ec2, instance_id, key_location, name,
    197                   description=None, snapshot_description=None,
    198                   kernel_id=None, ramdisk_id=None, **kwargs):
    199          super(EBSImageCreator, self).__init__(easy_ec2, instance_id,
    200                                                key_location, description,
    201                                                kernel_id, ramdisk_id)
    202          self.name = name
    203          self.description = description
    204          self.snapshot_description = snapshot_description or description
    205          self._snap = None
    206          self._vol = None
    207
    208      @print_timing
    209 -    def create_image(self, size=15):
    210          try:
    211              self.clean_private_data()
    212              if self.host.root_device_type == "ebs":
    213                  return self._create_image_from_ebs(size)
    214              return self._create_image_from_instance_store(size)
    215          except:
    216              log.error("Error occured while creating image")
    217              if self._snap:
    218                  log.error("Removing generated snapshot '%s'" % self._snap)
    219                  self._snap.delete()
    220              if self._vol:
    221                  log.error("Removing generated volume '%s'" % self._vol.id)
    222                  self._vol.detach(force=True)
    223                  self._vol.delete()
    224              raise
    225
    226 -    def _create_image_from_ebs(self, size=15):
    227          log.info("Creating EBS image...")
    228          imgid = self.ec2.create_image(self.host.id, self.name,
    229                                        self.description)
    230          log.info("Waiting for AMI %s to become available..." % imgid,
    231                   extra=dict(__nonewline__=True))
    232          img = self.ec2.get_image(imgid)
    233          s = Spinner()
    234          s.start()
    235          while img.state == "pending":
    236              time.sleep(15)
    237              if img.update() == "failed":
    238                  raise exception.AWSError(
    239                      "EBS image creation failed for AMI %s" % imgid)
    240          s.stop()
    241          return imgid
    242
    243 -    def _create_image_from_instance_store(self, size=15):
    244          host = self.host
    245          host_ssh = self.host_ssh
    246          log.info("Creating new EBS-backed image from instance-store instance")
    247          log.info("Creating new root volume...")
    248          vol = self._vol = self.ec2.create_volume(size, host.placement)
    249          log.info("Created new volume: %s" % vol.id)
    250          while vol.update() != 'available':
    251              time.sleep(5)
    252          dev = None
    253          for i in string.ascii_lowercase[::-1]:
    254              dev = '/dev/sd%s' % i
    255              if not dev in host.block_device_mapping:
    256                  break
    257          log.info("Attaching volume %s to instance %s on %s" %
    258                   (vol.id, host.id, dev))
    259          vol.attach(host.id, dev)
    260          while vol.update() != 'in-use':
    261              time.sleep(5)
    262          while not host_ssh.path_exists(dev):
    263              time.sleep(5)
    264          host_ssh.execute('mkfs.ext3 -F %s' % dev)
    265          mount_point = '/ebs'
    266          while host_ssh.path_exists(mount_point):
    267              mount_point += '1'
    268          host_ssh.mkdir(mount_point)
    269          log.info("Mounting %s on %s" % (dev, mount_point))
    270          host_ssh.execute('mount %s %s' % (dev, mount_point))
    271          log.info("Configuring /etc/fstab")
    272          host_ssh.remove_lines_from_file('/etc/fstab', '/mnt')
    273          fstab = host_ssh.remote_file('/etc/fstab', 'a')
    274          fstab.write('/dev/sdb1 /mnt auto defaults,nobootwait 0 0\n')
    275          fstab.close()
    276          log.info("Syncing root filesystem to new volume (%s)" % vol.id)
    277          host_ssh.execute(
    278              'rsync -avx --exclude %(mpt)s --exclude /root/.ssh / %(mpt)s' %
    279              {'mpt': mount_point})
    280          log.info("Unmounting %s from %s" % (dev, mount_point))
    281          host_ssh.execute('umount %s' % mount_point)
    282          log.info("Detaching volume %s from %s" % (dev, mount_point))
    283          vol.detach()
    284          while vol.update() != 'available':
    285              time.sleep(5)
    286          sdesc = self.snapshot_description
    287          snap = self._snap = self.ec2.create_snapshot(vol,
    288                                                       description=sdesc,
    289                                                       wait_for_snapshot=True)
    290          log.info("New snapshot created: %s" % snap.id)
    291          log.info("Removing generated volume %s" % vol.id)
    292          vol.delete()
    293          log.info("Creating root block device map using snapshot %s" % snap.id)
    294          bmap = self.ec2.create_root_block_device_map(snap.id,
    295                                                       add_ephemeral_drives=True)
    296          log.info("Registering new image...")
    297          img_id = self.ec2.register_image(name=self.name,
    298                                           description=self.description,
    299                                           architecture=host.architecture,
    300                                           kernel_id=self.kernel_id,
    301                                           ramdisk_id=self.ramdisk_id,
    302                                           root_device_name='/dev/sda1',
    303                                           block_device_map=bmap)
    304          return img_id
    305
    306
    307  # for backwards compatibility
    308  EC2ImageCreator = S3ImageCreator
    309