# Copyright (c) 2015-2016 Avere Systems, Inc.  All Rights Reserved.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
''' Abstraction for doing things on instances via Amazon

Cookbook/examples:

aws = vFXT.aws.Service(region='', subnet='', access_key='', secret_access_key='')
# or if on an AWS instance, we can autodetect settings
aws = vFXT.aws.Service.on_instance_init()

# Connection factory, has a thread specific copy
connection = aws.connection()
s3 = aws.connection(connection_type='s3')
iam = aws.connection(connection_type='iam')
vpc = aws.connection(connection_type='vpc')

instances = aws.find_instances({}) # filter dictionary
instances = aws.get_instances([])

instance = aws.get_instance('instance id')
aws.start(instance)
aws.stop(instance)
aws.restart(instance)
aws.destroy(instance)

aws.shelve(instance)
aws.unshelve(instance)

instance = aws.refresh(instance)

print aws.name(instance)
print aws.ip(instance)
print aws.fqdn(instance)
print aws.status(instance)

if aws.is_on(instance): pass
if aws.is_off(instance): pass
if aws.is_shelved(instance): pass

aws.wait_for_status(instance, aws.ON_STATUS, retries=aws.WAIT_FOR_STATUS)
aws.wait_for_service_checks(instance, retries=aws.WAIT_FOR_SERVICE_CHECKS)

aws.create_instance(machine_type, name, boot_disk_image, other_disks=None, tags=None, **options)
aws.create_cluster(self, cluster, **options)

aws.create_bucket(name)
aws.delete_bucket(name)

aws.load_cluster_information(cluster)

ip_count = 10
ip_addresses, netmask = aws.get_available_addresses(count=ip_count, contiguous=True)
aws.get_dns_servers(subnet_id)
aws.get_ntp_servers(subnet_id)
aws.get_default_router(subnet_id)

serializeme = aws.export()
newaws = vFXT.aws.Service(**serializeme)
'''

import threading
import Queue
import os
import time
import logging
import socket
import json
import urllib2
import filecmp
from itertools import cycle

import boto, boto.ec2, boto.vpc, boto.s3, boto.s3.connection, boto.iam, boto.utils
logging.getLogger('boto').setLevel(logging.CRITICAL)

from vFXT.cidr import Cidr
from vFXT.serviceInstance import ServiceInstance
from vFXT.service import *

log = logging.getLogger(__name__)

s3_region_locations = {
    'us-east-1':        boto.s3.connection.Location.DEFAULT,
    'eu-west-1':        boto.s3.connection.Location.EU,
    'us-west-1':        boto.s3.connection.Location.USWest,
    'us-west-2':        boto.s3.connection.Location.USWest2,
    'sa-east-1':        boto.s3.connection.Location.SAEast,
    'ap-northeast-1':   boto.s3.connection.Location.APNortheast,
    'ap-northeast-2':   'ap-northeast-2',
    'ap-southeast-1':   boto.s3.connection.Location.APSoutheast,
    'ap-southeast-2':   boto.s3.connection.Location.APSoutheast2,
    'ap-south-1':       'ap-south-1',
    'cn-north-1':       boto.s3.connection.Location.CNNorth1,
}

class Service(ServiceBase):
    '''AWS Service backend'''
    ON_STATUS="running"
    OFF_STATUS="stopped"
    PENDING_STATUS='pending'
    DESTROY_STATUS="terminated"
    NTP_SERVERS=['0.amazon.pool.ntp.org', '1.amazon.pool.ntp.org', '2.amazon.pool.ntp.org']
    DNS_SERVERS=[]
    MACHINE_DEFAULTS={
        "t2.micro":     {"data_disk_count":1, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":False, "node_count":3},
        "t2.small":     {"data_disk_count":1, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":False, "node_count":3},
        "t2.large":     {"data_disk_count":1, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":False, "node_count":3},
        "m1.small":     {"data_disk_count":1, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":False, "node_count":3},
        "c3.xlarge":    {"data_disk_count":8, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "c3.2xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "c3.4xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "c3.8xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":False, "node_count":3},
        "c4.xlarge":    {"data_disk_count":8, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "c4.2xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "c4.4xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "c4.8xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "r3.xlarge":    {"data_disk_count":8, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "r3.2xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "r3.4xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":True, "node_count":3},
        "r3.8xlarge":   {"data_disk_count":10, "data_disk_size":"200", 'data_disk_type':'gp2', "ebsoptimized":False, "node_count":3},
    }
    MACHINE_TYPES=MACHINE_DEFAULTS.keys()
    DEFAULTS_URL="http://avere-dist.s3-website-us-west-2.amazonaws.com/vfxtdefaults.json"
    S3TYPE_NAME='AMAZON'
    COREFILER_CRED_TYPE='s3'
    ARN="aws"
    IAM_HOST='iam.amazonaws.com'
    AWS_INSTANCE_HOST='169.254.169.254'
    S3URL_RE=re.compile('s3://(?P<bucket>[^\/]*)/(?P<path>.*)$')
    IAM_BUCKET_POLICY=['s3:GetLifecycleConfiguration',
                       's3:GetBucketLocation',
                       's3:ListBucket',
                       's3:ListBucketMultipartUploads',
                       's3:ListBucketVersions',
                       's3:PutLifecycleConfiguration']
    IAM_OBJECT_POLICY=['s3:AbortMultipartUpload',
                       's3:DeleteObject',
                       's3:GetObject',
                       's3:ListMultipartUploadParts',
                       's3:PutObject']
    IAM_POLICY=['ec2:AssignPrivateIpAddresses',
                'ec2:UnassignPrivateIpAddresses',
                'ec2:DescribeInstance*',
                'ec2:DescribeRouteTables',
                'ec2:DescribeSubnets',
                'ec2:ReplaceRoute',
                'ec2:CreateRoute',
                'ec2:DeleteRoute']
    ENDPOINT_TEST_HOSTS = ['s3.amazonaws.com']
    BOTO_503_RETRIES=10
    CREATE_INSTANCE_IAM_PROPAGATION_RETRIES=60
    ALLOCATE_PRIVATE_ADDRESSES=True

    def __init__(self, region, access_key, secret_access_key, **options):
        '''Constructor
            This performs an initial connection test and downloads the default
            data.

            If using a different S3 account, the S3 credentials can be provided
            long with the AWS credentials.

            Arguments:
                region (str): AWS region
                access_key (str): account access key
                secret_access_key (str): secret access key
                subnet (str or []): one or more subnets
                profile_name (str, optional): AWS profile name
                s3_access_key (str, optional): S3 account to use instead
                s3_secret_access_key (str, optional): S3 account to use instead
                s3_profile_name (str, optional): S3 profile name
                arn (str, optional): defaults to aws
                iam_host (str, optional): custom IAM host
                security_token (str, optional): AWS security token
                private_range (str, optional): private address range (cidr)
                security_groups (str, optional): default security groups

                proxy_uri (str, optional): URI of proxy resource (e.g. http://user:pass@172.16.16.20:8080)

                no_connection_test (bool, optional): skip connection test

        '''
        self.region               = region
        self.access_key           = access_key
        self.secret_access_key    = secret_access_key

        self.subnets              = options.get('subnet', None)
        self.profile_name         = options.get('profile_name', None)
        self.s3_access_key        = options.get('s3_access_key', self.access_key)
        self.s3_secret_access_key = options.get('s3_secret_access_key', self.secret_access_key)
        self.s3_profile_name      = options.get('s3_profile_name', self.profile_name)
        self.arn                  = options.get('arn') or self.ARN
        self.iam_host             = options.get('iam_host') or self.IAM_HOST
        self.security_token       = options.get('security_token', None)
        self.private_range        = options.get('private_range', None)
        self.proxy_uri            = options.get('proxy_uri', None)
        self.security_groups      = options.get('security_groups', None)

        self.local                = threading.local()

        if not self.subnets:
            raise vFXTConfigurationException("You must provide at least one subnet")

        self.subnets = [self.subnets] if isinstance(self.subnets, basestring) else self.subnets
        self.security_groups = self.security_groups.split(' ') if isinstance(self.security_groups, basestring) else self.security_groups

        if self.proxy_uri: # parse proxy uri into host and port (and user, password)
            self.set_proxy(self.proxy_uri)

            proxy_handler = urllib2.ProxyHandler({'http':self.proxy_uri, 'https':self.proxy_uri})
            opener = urllib2.build_opener(proxy_handler)
            urllib2.install_opener(opener)
        else:
            self.proxy = None

        # emit third party library version information
        log.debug("Using boto version {}".format(boto.__version__))

        if not options.get('no_connection_test', None):
            self.connection_test()

        log.debug("Fetching defaults from {}".format(self.DEFAULTS_URL))
        self.defaults = json.load(urllib2.urlopen(self.DEFAULTS_URL))

    @classmethod
    def get_instance_data(cls, source_address=None):
        '''Detect the instance data
            Arguments:
                source_address (str, optional): source address for data request

            This only works when running on an AWS instance.

            This is a service specific data structure.

            Well known keys that can be expected across services:
            machine_type (str): machine/instance type
            account_id (str): account identifier
            service_id (str): unique identifier for this host
            ssh_keys ([str]): ssh keys
            cluster_cfg (str): cluster configuration
        '''
        import httplib

        data = {}
        attributes = ['ami-id', 'ami-launch-index', 'ami-manifest-path',
                    'hostname', 'instance-action', 'instance-id', 'instance-type',
                    'local-hostname', 'local-ipv4', 'mac', 'profile', 'reservation-id',
                    'security-groups', 'placement/availability-zone', 'public-keys/0/openssh-key']
        mac_attrs = ['device-number', 'interface-id', 'local-hostname',
                    'local-ipv4s', 'mac', 'owner-id', 'security-group-ids',
                    'security-groups', 'subnet-id', 'subnet-ipv4-cidr-block',
                    'vpc-id', 'vpc-ipv4-cidr-block']

        if source_address:
            source_address = (source_address, 0)
        connection_host = cls.AWS_INSTANCE_HOST
        connection_port = httplib.HTTP_PORT

        conn = httplib.HTTPConnection(connection_host, connection_port, source_address=source_address, timeout=CONNECTION_TIMEOUT)

        try:

            for attr in attributes:
                conn.request('GET', '/latest/meta-data/{}'.format(attr))
                attr = attr.split('/')[-1] # flatten
                resp = conn.getresponse()
                if resp.status != 200:
                    data[attr] = ''
                    continue
                value = resp.read()
                try:
                    value = json.loads(value)
                except: pass
                if str(value).find('\n') > -1: # make a list of multi-values
                    value = str(value).split('\n')
                data[attr] = value

            conn.request('GET', '/latest/dynamic/instance-identity/document')
            resp = conn.getresponse()
            if resp.status == 200:
                data['document'] = json.loads(resp.read())

            conn.request('GET', '/latest/user-data')
            resp = conn.getresponse()
            data['user-data'] = ''
            if resp.status == 200:
                data['user-data'] = resp.read()

            data['network'] = {'interfaces':{'macs':{}}}
            conn.request('GET', '/latest/meta-data/network/interfaces/macs/')
            macs = []
            resp = conn.getresponse()
            if resp.status == 200:
                macs = [m[:-1] for m in resp.read().split('\n')]
            for mac in macs:
                attr_data = {}
                for attr in mac_attrs:
                    conn.request('GET', '/latest/meta-data/network/interfaces/macs/{}/{}'.format(mac, attr))
                    r = conn.getresponse().read()
                    try:
                        r = json.loads(r)
                    except: pass
                    if str(r).find('\n') > -1:
                        r = str(r).split('\n')
                    attr_data[attr] = r
                data['network']['interfaces']['macs'][mac] = attr_data

            conn.request('GET', '/latest/meta-data/iam/security-credentials/')
            creds = []
            resp = conn.getresponse()
            if resp.status == 200:
                creds = resp.read().split('\n')
            data['iam'] = {'security-credentials':{}}
            for cred in creds:
                conn.request('GET', '/latest/meta-data/iam/security-credentials/{}'.format(cred))
                data['iam']['security-credentials'][cred] = json.loads(conn.getresponse().read())

            conn.request('GET', '/latest/meta-data/iam/info')
            resp = conn.getresponse()
            data['arn'] = None
            if resp.status == 200:
                try:
                    r = json.loads(resp.read())
                    data['arn'] = r['InstanceProfileArn'].split(':')[1]
                except: pass

            data['region']      = data['document']['region']
            data['hostname']    = data['hostname'].split(' ')[0]
            data['machine_type']= data['instance-type']
            data['account_id']  = data['document'].get('accountId', 'unknown')
            data['service_id']  = data['instance-id']
            data['ssh_keys']    = data['openssh-key'] if data['openssh-key'] else []
            data['cluster_cfg'] = data['user-data']

        except Exception as e:
            raise vFXTConfigurationException('Unable to read instance metadata: {}'.format(e))
        finally:
            conn.close()

        return data

    @classmethod
    def on_instance_init(cls, source_address=None, no_connection_test=False, proxy_uri=None):
        '''Init an AWS service object from instance metadata
            Arguments:
                source_address (str, optional): source address for data request
                no_connection_test (bool, optional): skip connection tests, defaults to False
                proxy_uri (str, optional): URI of proxy resource

            This is only meant to be called on instance.  Otherwise will
            raise a vFXTConfigurationException exception.

            If instance credentials are sparse, the connection test will also
            fail with permission error.

            http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-ec2instance.html
            http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
            http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
            http://docs.aws.amazon.com/STS/latest/UsingSTS/Welcome.html
        '''
        instance_data = cls.get_instance_data(source_address=source_address)
        log.debug('Read instance data: {}'.format(instance_data))
        try:

            if 'iam' not in instance_data or not instance_data['iam']['security-credentials']:
                raise vFXTConfigurationException('Cannot read IAM information.  Check role policy permissions')

            region = instance_data['region']
            access_key = instance_data['iam']['security-credentials'].values()[0]['AccessKeyId']
            secret_key = instance_data['iam']['security-credentials'].values()[0]['SecretAccessKey']
            #vpc_id      = instance_data['network']['interfaces']['macs'].values()[0]['vpc-id']
            #instance_id = instance_data['instance-id']
            subnet_id   = instance_data['network']['interfaces']['macs'].values()[0]['subnet-id']
            token = instance_data['iam']['security-credentials'].values()[0]['Token']
            arn   = instance_data['arn']
            security_groups = instance_data['network']['interfaces']['macs'].values()[0]['security-group-ids']

            iam_host = boto.iam.connect_to_region(region).host
            srv = Service(region=region, access_key=access_key, secret_access_key=secret_key,
                          subnet=subnet_id, security_token=token, arn=arn, iam_host=iam_host,
                          no_connection_test=no_connection_test, proxy_uri=proxy_uri,
                          security_groups=security_groups)
            srv.local.instance_data = instance_data
            return srv
        except Exception as e:
            raise vFXTConfigurationException(e)

    def connection_test(self):
        '''Connection test

            Raises: vFXTConfigurationException
        '''
        log.debug("Performing connection test")

        # fail as fast as possible... boto does not make it easy
        boto_config = boto.config
        boto_config.has_section('Boto') or boto_config.add_section('Boto')
        exiting_timeout = boto_config.get('Boto', 'http_socket_timeout')
        exiting_retries = boto_config.get('Boto', 'num_retries')
        try:
            if not self.proxy: # proxy environments may block outgoing name resolution
                self.dns_check()

            boto_config.set('Boto', 'http_socket_timeout', str(CONNECTION_TIMEOUT))
            boto_config.set('Boto', 'num_retries', '0')
            conn = self.connection()
            _aws_do(conn.get_all_reservations, filters={'tag:invalid_tag':'. .'}) # invalid name for filter
        except Exception as e:
            log.debug(e)
            raise vFXTConfigurationException("Failed to establish connection to service: {}".format(e))
        finally:
            if exiting_timeout:
                boto_config.set('Boto', 'http_socket_timeout', exiting_timeout)
            else:
                boto_config.remove_option('Boto', 'http_socket_timeout')
            if exiting_retries:
                boto_config.set('Boto', 'num_retries', exiting_retries)
            else:
                boto_config.remove_option('Boto', 'num_retries')

    def connection(self, connection_type='ec2'):
        '''Connection factory, returns a new connection or thread local copy

            Arguments:
                connection_type (str, optional): connection type (ec2, s3, iam, vpc)
        '''
        try:
            cred_expiration = self.local.instance_data['iam']['security-credentials'].values()[0]['Expiration']
            if (int(time.mktime(time.strptime(cred_expiration, "%Y-%m-%dT%H:%M:%SZ")))-120) < int(time.time()):
                log.debug("Access token expired, forcing refresh")
                self.local.connections = {}
        except: pass

        access_key           = self.access_key
        secret_access_key    = self.secret_access_key
        security_token       = self.security_token
        s3_access_key        = self.s3_access_key
        s3_secret_access_key = self.s3_secret_access_key

        if not hasattr(self.local,'connections'):
            self.local.connections = {}
        if not self.local.connections.get(connection_type, False):
            if security_token: # on instance, get most up to date
                instance_data            = self.get_instance_data()
                creds                    = instance_data['iam']['security-credentials'].values()[0]
                self.local.instance_data = instance_data
                access_key               = creds['AccessKeyId']
                secret_access_key        = creds['SecretAccessKey']
                security_token           = creds['Token']
                s3_access_key            = creds['AccessKeyId']
                s3_secret_access_key     = creds['SecretAccessKey']

            proxy_settings = {}
            proxy_settings['proxy']      = self.proxy.hostname if self.proxy else None
            proxy_settings['proxy_port'] = self.proxy.port     if self.proxy else None
            proxy_settings['proxy_user'] = self.proxy.username if self.proxy else None
            proxy_settings['proxy_pass'] = self.proxy.password if self.proxy else None

            log.debug("Creating new {} object".format(connection_type))
            err_fmt = 'Unable to create connection to {}: {}'
            if connection_type is 'ec2':
                newconn = boto.ec2.connect_to_region(self.region,
                            aws_access_key_id=access_key,
                            aws_secret_access_key=secret_access_key,
                            profile_name=self.profile_name,
                            security_token=security_token, **proxy_settings)
                if not newconn:
                    raise vFXTConfigurationException(err_fmt.format('region', self.region))
            elif connection_type is 's3':
                newconn = boto.s3.connect_to_region(self.region,
                             aws_access_key_id=s3_access_key,
                             aws_secret_access_key=s3_secret_access_key,
                             profile_name=self.s3_profile_name,
                             security_token=security_token, **proxy_settings)
                if not newconn:
                    raise vFXTConfigurationException(err_fmt.format('region', self.region))
            elif connection_type is 'iam':
                newconn = boto.iam.connection.IAMConnection(
                            aws_access_key_id=access_key,
                            aws_secret_access_key=secret_access_key,
                            profile_name=self.profile_name,
                            host=self.iam_host,
                            security_token=security_token, **proxy_settings)
                if not newconn:
                    raise vFXTConfigurationException(err_fmt.format('IAM host', self.iam_host))
            elif connection_type is 'vpc':
                newconn = boto.vpc.connect_to_region(self.region,
                            aws_access_key_id=access_key,
                            aws_secret_access_key=secret_access_key,
                            profile_name=self.profile_name,
                            security_token=security_token, **proxy_settings)
                if not newconn:
                    raise vFXTConfigurationException(err_fmt.format('region', self.region))
            else:
                raise vFXTConfigurationException("Unknown connection type: {}".format(connection_type))

            self.local.connections[connection_type] = newconn

        return self.local.connections[connection_type]

    def find_instances(self, search=None):
        '''Returns all or filtered list of instances

            Arguments:
                search (dict): search query

            Search examples:
                {"subnet_id":subnet_id}
                {"private_ip_address":['10.1.1.1','10.1.1.2']}
                {"tag:Name":"*lg*"}
                {"tag:Owner":"tstark"}
        '''
        conn = self.connection()
        return [i for r in _aws_do(conn.get_all_reservations, filters=search) \
                for i in r.instances]

    def get_instances(self, instance_ids):
        '''Returns a list of instances with the given instance ID list

            Arguments:
                instance_ids ([str]): list of instance id strings

            Returns:
                [objs]: list of backend instance objects
        '''
        conn = self.connection()
        return [i for r in _aws_do(conn.get_all_reservations, instance_ids) \
                for i in r.instances]

    def get_instance(self, instance_id):
        '''Get a specific instance by instance ID

            Arguments:
                instance_id (str)

            Returns:
                obj or None
        '''
        instances = self.get_instances([instance_id])
        if instances:
            return instances[0]
        return None

    def wait_for_status(self, instance, status, retries=ServiceBase.WAIT_FOR_STATUS):
        '''Poll on a given instance for status

            Arguments:
                instance (obj): backend instance object
                status (str): status string to watch for
                retries (int, optional): number of retries

            Raises: vFXTServiceTimeout
        '''
        s = '...' # in case our instance is not yet alive
        while status != s:
            if retries % 10 == 0: # rate limit
                log.debug("Waiting for status: {} != {}".format(s, status))
            time.sleep(self.POLLTIME)
            try:
                s = _aws_do(instance.update)
            except Exception as e:
                log.debug('Ignored wait_for_status: {}'.format(e))
            retries -= 1
            if retries == 0:
                message = 'unknown'
                try:
                    message = instance.state_reason['message']
                except Exception as e: pass
                raise vFXTServiceTimeout("Failed waiting for {} on {} ({})".format(status, instance.id, message))

    def wait_for_service_checks(self, instance, retries=ServiceBase.WAIT_FOR_SERVICE_CHECKS):
        '''Wait for Amazon service checks to complete

            Arguments:
                instance: backend instance
                retries (int, optional): numb of retries

            Raises: vFXTServiceTimeout
        '''
        log.info("Checking health checks for instance {}".format(instance.id))
        instance_id = instance.id
        conn        = self.connection()
        statuses    = _aws_do(conn.get_all_instance_status, instance_id)
        if not statuses:
            return
        status = statuses[0]
        instance.reboots_tried = getattr(instance, 'reboots', 0) # temp attribute
        while status.system_status.status != 'ok' and status.instance_status.status != 'ok':
            time.sleep(self.POLLTIME)
            retries -= 1
            if retries == 0:
                if instance.reboots_tried == 3:
                    raise vFXTServiceTimeout("Failed waiting for {} status checks".format(instance_id))
                else:
                    instance.reboots_tried += 1
                    log.debug("Attempt {} to restart instance {}".format(instance.reboots_tried, instance_id))
                    self.restart(instance)
            try:
                statuses = _aws_do(conn.get_all_instance_status, instance_id)
                if statuses:
                    status = statuses[0]
            except Exception as e:
                log.debug('Ignored wait_for_service_checks: {}'.format(e))

    def can_stop(self, instance):
        ''' Some instance configurations cannot be stopped. Check if this is one.

            Arguments:
                instance: backend instance
        '''
        return True

    def stop(self, instance, wait=ServiceBase.WAIT_FOR_STOP):
        '''Stop an instance

            Arguments:
                instance: backend instance
        '''
        log.info("Stopping instance {}".format(self.name(instance)))
        instance.stop()
        self.wait_for_status(instance, self.OFF_STATUS, wait)

    def start(self, instance, wait=ServiceBase.WAIT_FOR_START):
        '''Start an instance

            Arguments:
                instance: backend instance
                wait (int): wait time
        '''
        log.info("Starting instance {}".format(self.name(instance)))
        instance.start()
        self.wait_for_status(instance, self.ON_STATUS, wait)

    def restart(self, instance, wait=ServiceBase.WAIT_FOR_RESTART):
        '''Restart an instance

            Arguments:
                instance: backend instance
                wait (int): wait time
        '''
        log.info("Restarting instance {}".format(self.name(instance)))
        # use reboot so our instance state and billing isn't changed
        instance.reboot()
        self.wait_for_status(instance, self.ON_STATUS, wait)

    def destroy(self, instance, wait=ServiceBase.WAIT_FOR_DESTROY):
        '''Destroy an instance

            Arguments:
                instance: backend instance
                wait (int): wait time
        '''

        # clean up any routes
        vpc = self.connection(connection_type='vpc')
        routes = [{'route_table_id':rt.id, 'destination_cidr_block':r.destination_cidr_block}
                    for rt in _aws_do(vpc.get_all_route_tables)
                    for r in rt.routes
                    if r.instance_id == instance.id]
        for route in routes:
            try:
                _aws_do(vpc.delete_route, **route)
            except Exception as e:
                log.debug("Ignored delete route failure: {}".format(e))

        log.info("Destroying instance {}".format(self.name(instance)))
        _aws_do(instance.terminate)
        self.wait_for_status(instance, self.DESTROY_STATUS, wait)

    def is_on(self, instance):
        '''Return True if the instance is currently on

            Arguments:
                instance: backend instance

            This will be false if the instance is off or terminated.
        '''
        s = _aws_do(instance.update)
        return self.OFF_STATUS != s and self.DESTROY_STATUS != s

    def is_off(self, instance):
        '''Return True if the instance is currently off

            Arguments:
                instance: backend instance

            This will be true if the instance is on or terminated.
        '''
        s = _aws_do(instance.update)
        return self.OFF_STATUS == s or self.DESTROY_STATUS == s

    def is_shelved(self, instance):
        '''Return True if the instance is currently shelved

            Arguments:
                instance: backend instance
        '''
        return 'shelved' in instance.tags

    def name(self, instance):
        '''Returns the instance name (may be different from instance id)

            Arguments:
                instance: backend instance
        '''
        return instance.tags.get('Name', instance.id)

    def instance_id(self, instance):
        '''Returns the instance id (may be different from instance name)

            Arguments:
                instance: backend instance
        '''
        return instance.id

    def status(self, instance):
        '''Return the instance status

            Arguments:
                instance: backend instance
        '''
        return _aws_do(instance.update)

    def refresh(self, instance):
        '''Refresh the instance from the Amazon backend

            Arguments:
                instance: backend instance
        '''
        try:
            _aws_do(instance.update,validate=True) # raise error if not found
            return instance
        except Exception as e:
            raise vFXTServiceFailure("Failed to find instance: {}".format(e))

    def ip(self, instance):
        '''Return the primary IP address of the instance

            Arguments:
                instance: backend instance
        '''
        return instance.private_ip_address

    def fqdn(self, instance):
        '''Provide the fully qualified domain name of the instance

            Arguments:
                instance: backend instance
        '''
        try:
            return socket.gethostbyaddr(instance.private_ip_address)[0]
        except:
            return instance.private_dns_name

    # storage/buckets
    def create_bucket(self, name, tags=None):
        '''Create a bucket

            Arguments:
                name (str): bucket name to create
                tags (dict): tags to apply to the bucket

            Raises: vFXTServiceFailure
        '''
        if not self.valid_bucketname(name):
            raise vFXTConfigurationException("{} is not a valid bucket name".format(name))

        s3 = self.connection(connection_type='s3')

        # we support having 2 connections (if s3 account is different from ec2
        ec2_grant_user = None
        if self.s3_access_key != self.access_key:
            ec2s3 = boto.s3.connection.S3Connection(self.access_key, self.secret_access_key,
                    security_token=self.security_token)
            ec2_grant_user = _aws_do(ec2s3.get_canonical_user_id)

        # tagging
        btags  = boto.s3.tagging.Tags()
        tagset = boto.s3.tagging.TagSet()
        if tags:
            for k,v in tags.iteritems():
                tagset.add_tag(k,v)
            tagset.add_tag('Name', name)
            btags.add_tag_set(tagset)


        s3_region = s3_region_locations[self.region] if self.region in s3_region_locations else self.region

        retries=ServiceBase.CLOUD_API_RETRIES
        while True:
            try:
                b = _aws_do(s3.create_bucket, name, location=s3_region)
                if ec2_grant_user:
                    log.info("Granting EC2 user bucket control")
                    _aws_do(b.add_user_grant, 'FULL_CONTROL', ec2_grant_user)
                if tags:
                    log.info("Tagging bucket {}".format(name))
                    _aws_do(b.set_tags, btags)
                return b
            except Exception as e:
                if retries == 0:
                    log.debug(e)
                    raise vFXTServiceFailure("Failed to create bucket {}: {}".format(name, e))
                retries -= 1

    def authorize_bucket(self, cluster, name, retries=ServiceBase.CLOUD_API_RETRIES):
        '''Perform any backend work for the bucket, and register a credential
        for it to the cluster

            Arguments:
                cluster (Cluster): cluster object
                name (str): bucket name
                retries (int): number of attempts to make

            Raises: vFXTServiceFailure
        '''
        iam         = self.connection(connection_type='iam')
        iamrole     = cluster.iamrole
        policy_name = 'policy_{}'.format(iamrole)

        try:
            response = _aws_do(iam.list_role_policies, iamrole)
            policies = response['list_role_policies_response']['list_role_policies_result']['policy_names']
            # if this role has only one policy, use it
            if len(policies) == 1:
                policy_name = policies[0]
            elif policy_name not in policies:
                raise Exception("Unable to determine which policy to use")
        except Exception as e:
            raise vFXTConfigurationException("Could not find role policy for {}: {}".format(iamrole, e))

        response = _aws_do(iam.get_role_policy, iamrole, policy_name)
        if not response:
            raise vFXTServiceFailure("Failed to find existing policy")
        policy = json.loads(urllib2.unquote(response['get_role_policy_response']['get_role_policy_result']['policy_document']))

        bucket_policy = {
            'Action': self.IAM_BUCKET_POLICY,
            'Resource': ['arn:{}:s3:::{}'.format(self.arn, name)],
            'Effect': 'Allow'
        }
        object_policy = {
            'Action': self.IAM_OBJECT_POLICY,
            'Resource': ['arn:{}:s3:::{}/*'.format(self.arn, name)],
            'Effect': 'Allow'
        }
        policy['Statement'].append(bucket_policy)
        policy['Statement'].append(object_policy)

        _aws_do(iam.put_role_policy, iamrole, policy_name, json.dumps(policy))

        xmlrpc = cluster.xmlrpc()
        # we cant reuse the existing default creds
        if self.s3_access_key != self.access_key:
            cred = 's3-{}'.format(cluster.name)

            # if it exists, use it
            if cred in [c['name'] for c in xmlrpc.corefiler.listCredentials()]:
                return cred

            log.debug("Creating credential {}".format(cred))
            create_retries = retries
            while True:
                try:
                    r = xmlrpc.corefiler.createCredential(cred, self.COREFILER_CRED_TYPE, {
                        'accessKey':self.s3_access_key,
                        'privateKey':self.s3_secret_access_key,
                    })
                    if r != 'success':
                        raise Exception(r)
                    return cred
                except Exception as e:
                    if create_retries == 0:
                        raise vFXTConfigurationException("Could not create credential {}: {}".format(cred, e))
                create_retries -= 1

        # XXX otherwise assume first credential (internalName = cloudCredential1)
        return xmlrpc.corefiler.listCredentials()[0]['name']


    def delete_bucket(self, name):
        '''Delete a bucket

            Arguments:
                name (str): bucket name

            Raises: vFXTServiceFailure
        '''
        s3 = self.connection(connection_type='s3')

        try:
            b = _aws_do(s3.get_bucket, name)
            _aws_do(b.delete)
        except Exception as e:
            raise vFXTServiceFailure("Failed to delete bucket {}: {}".format(name, e))

    def create_instance(self, machine_type, name, boot_disk_image, other_disks=None, tags=None, **options):
        '''Create and return an AWS instance

            Arguments:
                machine_type (str): AWS machine type
                name (str): name of the instance (placed in tag)
                boot_disk_image (str): AMI ID name
                other_disks (boto.ec2.blockdevicemapping.BlockDeviceMapping): custom disk layout
                tags (dict): tags to apply to instance (Name tag auto filled)
                user_data (dict, optional): user_data to pass to the instance (for cluster_cfg, etc)
                placement_group (str, optional): placement group to launch instances in
                subnet (str, optional): subnet to place the instance in
                private_ip_address (str, optional): primary private IP address
                role_name (str, optional): role/profile name
                ebs_optimized (bool, optional): instance type specific
                key_name (str, optional): ssh key-pair name
                security_group_ids (str, optional):  space delimited list of security groups
                disableSourceDestCheck (bool, optional): disables instance sourceDestCheck
                root_size (int, optional): root disk size in GB
                root_dev_name (string, optional): root disk device name (defaults to AMI default)
                wait_for_success (int, optional): wait time for the instance to report success (default WAIT_FOR_SUCCESS)

            Raises: vFXTConfigurationException, vFXTServiceFailure
        '''
        if not self.valid_instancename(name):
            raise vFXTConfigurationException("{} is not a valid instance name".format(name))
        if machine_type not in self.MACHINE_TYPES:
            raise vFXTConfigurationException("{} is not a valid instance type".format(machine_type))
        machine_defs = self.MACHINE_DEFAULTS[machine_type]
        conn         = self.connection()

        # setup disks
        bdm = other_disks
        if not bdm:
            bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
        root_opts = {}
        if options.get('root_size'):
            root_opts['size'] = options.get('root_size')
        root = boto.ec2.blockdevicemapping.BlockDeviceType(volume_type='gp2',
                    delete_on_termination=True, **root_opts)
        ami  = _aws_do(conn.get_image, boot_disk_image)
        if not ami:
            raise vFXTConfigurationException("{} is not a valid AMI".format(boot_disk_image))
        root_dev_name = options.get('root_dev_name', ami.root_device_name)
        bdm[root_dev_name] = root
        # fill in with blank disks if necessary
        base = ord('b')
        for idx in xrange(4):
            dev_name = "/dev/sd{:c}".format(base+idx)
            if dev_name not in bdm:
                dev      = boto.ec2.blockdevicemapping.BlockDeviceType(no_device=True)
                bdm[dev_name] = dev

        # general instance settings
        user_data       = options.get('user_data','')
        placement_group = options.get('placement_group', None)
        subnet_id       = options.get('subnet') or self.subnets[0]
        ip_address      = options.get('private_ip_address', None)
        role_name       = options.get('role_name', None)
        subnet          = self._get_subnet(subnet_id)
        az              = subnet.availability_zone
        ebs_optimized   = options.get('ebs_optimized', machine_defs['ebsoptimized'])
        key_name        = options.get('key_name', None)
        security_groups = options.get('security_group_ids', self.security_groups)

        if security_groups and isinstance(security_groups, basestring):
            security_groups = [_ for _ in security_groups.split(' ')]

        interfaces = None
        if options.get('enable_ephemeral', False):
            interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id=subnet_id, associate_public_ip_address=True)
            interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
            subnet_id = None # cannot specify this at instance level if doing it at interface level

        create_options = {
            'instance_type': machine_type,
            'subnet_id': subnet_id,
            'placement_group': placement_group,
            'block_device_map': bdm,
            'user_data': user_data,
            'placement': az,
            'private_ip_address': ip_address,
            'ebs_optimized': ebs_optimized,
            'instance_profile_name': role_name,
            'key_name': key_name,
            'security_group_ids': security_groups,
            'network_interfaces': interfaces,
        }
        log.debug("create_instance request body: {}, {}".format(boot_disk_image, create_options))

        # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-cli
        # "After you create an IAM role, it may take several seconds for the permissions to propagate."
        # XXX we might not have the instance profile or the roles may not yet be populated
        iam_propagation_msgs = ['has no associated IAM Roles', 'Invalid IAM Instance Profile']
        retries = self.CREATE_INSTANCE_IAM_PROPAGATION_RETRIES
        while retries > 0:
            try:
                r = _aws_do(conn.run_instances, boot_disk_image, **create_options)
                break
            except Exception as e:
                log.debug('Run instances failed: {}'.format(e))
                retries -= 1

                if retries > 0 and filter(lambda x: x in e.message, iam_propagation_msgs):
                    log.debug("Retrying awaiting IAM propagation: {}".format(e))
                    time.sleep(self.POLLTIME)
                    continue

                log.debug("Create instance failed: {}".format(e))
                raise vFXTServiceFailure(e)

        instance = r.instances[0]

        # use wait_for_status of pending prior to tagging
        self.wait_for_status(instance, self.PENDING_STATUS)

        try:
            # tags
            if not tags:
                tags = {}
            if 'Name' not in tags:
                tags['Name'] = name
            _aws_do(conn.create_tags, instance.id, tags)

            # tag volumes
            for vname, vdev in instance.block_device_mapping.iteritems():
                tagname = "{}-{}".format(tags['Name'], vname[vname.rfind("/")+1:])
                _aws_do(conn.create_tags, vdev.volume_id, {"Name":tagname})
        except Exception as e:
            log.exception(e)
            _aws_do(instance.terminate)
            self.wait_for_status(instance, self.DESTROY_STATUS, 600)
            raise vFXTServiceFailure(e)

        if options.get('disableSourceDestCheck'):
            dsd_retries = ServiceBase.CLOUD_API_RETRIES
            while True:
                try:
                    _aws_do(conn.modify_instance_attribute, instance.id, attribute='sourceDestCheck', value=False)
                    break
                except Exception as e:
                    time.sleep(self.POLLTIME)
                    dsd_retries -= 1
                    if dsd_retries == 0:
                        raise

        wait_for_success = options.get('wait_for_success') or self.WAIT_FOR_SUCCESS
        self.wait_for_status(instance, self.ON_STATUS, wait_for_success)
        return instance

    def create_node(self, node_name, cfg, node_opts, instance_options):
        '''Create a cluster node

            This is a frontend for create_instance that handles vFXT node specifics

            Arguments:
                node_name (str): name of the node
                cfg (str): configuration string to pass to the node
                node_opts (dict): node creation options
                instance_options (dict): options passed to create_instance

                node_opts include:
                    data_disk_size: size of data disks
                    data_disk_type: type of data disks
                    data_disk_count: number of data disks
                    data_disk_iops (optional)
                    tags (dict)
                    machine_type
                    role: (iam role dict from AWS)
                    root_image: AMI id

        '''
        bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
        tags = {}
        tags.update(node_opts.get('tags', {}))
        tags['Name'] = node_name

        if node_opts['data_disk_type'] == 'ephemeral':
            if 'ebs_optimized' in instance_options:
                instance_options['ebs_optimized'] = False
            base = ord('b')
            for idx in xrange(4): # ignore data_disk_size for ephemeral
                name     = 'ephemeral{}'.format(idx)
                dev_name = "/dev/sd{:c}".format(base+idx)
                dev      = boto.ec2.blockdevicemapping.BlockDeviceType(ephemeral_name=name)
                bdm[dev_name] = dev
        else:
            base = ord('f')
            for idx in xrange(node_opts['data_disk_count']):
                opts = {
                    'size':                  node_opts['data_disk_size'],
                    'volume_type':           node_opts['data_disk_type'],
                    'delete_on_termination': True,
                    'iops':                  node_opts['data_disk_iops'],
                    'encrypted':             True,
                }
                dev       = boto.ec2.blockdevicemapping.BlockDeviceType(**opts)
                name      = "/dev/sd{:c}".format(base+idx)
                bdm[name] = dev

        log.info("Creating node {}".format(node_name))
        n = self.create_instance(machine_type=node_opts['machine_type'],
                    name=node_name,
                    boot_disk_image=node_opts['root_image'],
                    other_disks=bdm,
                    tags=tags,
                    user_data=cfg,
                    role_name=node_opts['role']['role_name'],
                    **instance_options
        )
        log.info("Created {} ({})".format(n.id, n.private_ip_address))
        return n

    def create_cluster(self, cluster, **options):
        '''Create a vFXT cluster (calls create_node for each node)
            Typically called via vFXT.Cluster.create()

            Arguments:
                cluster (vFXT.cluster.Cluster): cluster object
                size (int, optional): size of cluster (node count)
                subnet (str or []): one or more subnets
                root_image (str, optional): AMI id (or machine type default)
                data_disk_size (int, optional): size of data disk (or machine type default)
                data_disk_count (int, optional): number of data disks (or machine type default)
                data_disk_type (str, optional): type of data disks (or machine type default)
                data_disk_iops (int, optional): IOPs value for EBS optimized disks (or machine type default, 250)
                tags (dict, optional): tags for instance (Name autofilled)
                iamrole (str, optional): iam role (otherwise one is created)
                config_expiration (int, optional): expiration time for cluster join configuration
                wait_for_state (str, optional): red, yellow, green cluster state (defaults to yellow)
                skip_cleanup (bool, optional): do not clean up on failure

                Additional arguments are passed through to create_node()

            Raises: vFXTConfigurationException, vFXTCreateFailure

        '''
        machine_type    = cluster.machine_type

        if machine_type not in self.MACHINE_TYPES:
            raise vFXTConfigurationException("{} is not a valid instance type".format(machine_type))

        machine_defs    = self.MACHINE_DEFAULTS[machine_type]
        cluster_size    = int(options.get('size', machine_defs['node_count']))

        # networking
        subnets = options.get('subnet') or self.subnets
        subnets = [subnets] if isinstance(subnets, basestring) else subnets
        cycle_subnets = cycle(subnets)

        # check subnet objects for mapPublicIpOnLaunch
        for subnet_id in subnets:
            if self._get_subnet(subnet_id).mapPublicIpOnLaunch == 'true':
                log.warn("Subnet {} has mapPublicIpOnLaunch enabled".format(subnet_id))

        # check that our subnets share a route table
        route_tables = set()
        vpcconn = self.connection('vpc')
        for subnet_id in subnets:
            subnet_route_tables = _aws_do(vpcconn.get_all_route_tables, filters={'association.subnet-id':subnet_id})
            if not subnet_route_tables:
                route_tables.add('main') # main is the default route table
            else:
                route_tables.update(filter(None, [_.id for _ in subnet_route_tables]))
        if len(route_tables) > 1:
            raise vFXTConfigurationException("All subnets must share the same route table")

        ip_count   = (cluster_size*2)+1 # cluster + mgmt
        avail,mask = self.get_available_addresses(count=ip_count, contiguous=True)

        cluster.mgmt_ip = avail.pop(0)
        private_ips     = avail[0:cluster_size]
        cluster_ips     = avail[cluster_size:]

        # if we are going to span multiple subnets we can not assign the private ip address
        if len(subnets) > 1:
            private_ips = [None]*cluster_size

        # store for cluster config
        cluster.mgmt_netmask     = mask
        cluster.cluster_ip_start = cluster_ips[0]
        cluster.cluster_ip_end   = cluster_ips[-1]

        # disks
        root_image      = options.get('root_image') or self.defaults[self.region]['current']
        data_disk_size  = options.get('data_disk_size')  or machine_defs['data_disk_size']
        data_disk_count = options.get('data_disk_count') or machine_defs['data_disk_count']
        data_disk_type  = options.get('data_disk_type')  or machine_defs['data_disk_type']
        data_disk_iops  = options.get('data_disk_iops')  or machine_defs.get('data_disk_iops', 250)
        if data_disk_type != 'io1':
            data_disk_iops = None

        # tags
        tags = options.pop('tags', {})

        # role
        iamrole = options.get('iamrole', None)
        role    = None
        if iamrole: # verify
            role = self._get_iamrole(iamrole)
        else:
            role = self._create_iamrole('avere_cluster_role_{}_{}'.format(int(time.time()), cluster.name))
        cluster.iamrole = role['role_name']

        # config
        cfg     = cluster.cluster_config(expiration=options.get('config_expiration', None))
        joincfg = cluster.cluster_config(joining=True, expiration=options.get('config_expiration', None))
        cfg     +=  '''\n[EC2 admin credential]''' \
                    '''\nname={0}''' \
                    '''\nrole={0}''' \
                    .format(role['role_name'])
        log.debug("Generated cluster config: {}".format(cfg))
        log.debug("Generated cluster join config: {}".format(joincfg))

        # for point to point addressing we need to disable Source/Dest checks
        if mask == '255.255.255.255':
            options['disableSourceDestCheck'] = True

        try:
            name = '{}-{}'.format(cluster.name, 1)
            opts = {'data_disk_type':data_disk_type, 'data_disk_count':data_disk_count,
                    'tags': tags.copy(), 'data_disk_size':data_disk_size,
                    'data_disk_iops':data_disk_iops, 'machine_type':machine_type,
                    'root_image':root_image, 'role':role}
            inst_opts = options.copy()
            inst_opts['subnet'] = next(cycle_subnets)
            inst_opts['private_ip_address'] = private_ips.pop(0)
            n    = self.create_node(name, cfg, node_opts=opts, instance_options=inst_opts)
            cluster.nodes.append(ServiceInstance(service=self, instance=n))

            # post first node setup, prior to creating the rest of the cluster
            if not options.get('skip_configuration'):
                cluster.first_node_configuration(wait_for_state=options.get('wait_for_state', 'yellow'))

            nodeq   = Queue.Queue()
            failq   = Queue.Queue()
            threads = []
            def cb(nodenum, inst_opts, nodeq, failq):
                '''callback'''
                try:
                    name = '{}-{}'.format(cluster.name, nodenum)
                    n = self.create_node(name, joincfg, node_opts=opts, instance_options=inst_opts)
                    nodeq.put(n)
                except Exception as e:
                    failq.put(e)

            for node_num in xrange(1,cluster_size):
                inst_opts = options.copy()
                inst_opts['subnet'] = next(cycle_subnets)
                inst_opts['private_ip_address'] = private_ips.pop(0)
                t = threading.Thread(target=cb, args=(node_num+1, inst_opts, nodeq, failq,))
                t.start()
                threads.append(t)

            for t in threads:
                t.join()

            while True:
                try:
                    n = nodeq.get_nowait()
                    cluster.nodes.append(ServiceInstance(service=self, instance=n))
                except Queue.Empty:
                    break

            failed = []
            while True:
                try:
                    failed.append(failq.get_nowait())
                except Queue.Empty:
                    break
            if failed:
                raise Exception(failed)

        except Exception as e:
            log.debug(e)
            if not options.get('skip_cleanup', False):
                for n in cluster.nodes: # cleanup
                    try:
                        n.destroy()
                    except Exception as e:
                        log.debug(e)
                self.post_destroy_cluster(cluster)
            raise vFXTCreateFailure(e)

    def post_destroy_cluster(self, cluster):
        '''Post cluster destroy cleanup'''
        try:
            if hasattr(cluster, 'iamrole'):
                iamrole = cluster.iamrole
                # Note we don't want to just delete a role, unless we created it
                if iamrole.startswith('avere_cluster_role_'):
                    self._delete_iamrole(iamrole)
        except Exception as e:
            log.debug("Cleanup failed: {}".format(e))

    def add_cluster_nodes(self, cluster, count, **options):
        '''Add nodes to the cluster (delegates to create_node())

            Arguments:
                cluster (vFXT.cluster.Cluster): cluster object
                count (int): number of nodes to add
                skip_cleanup (bool, optional): do not clean up on failure
                private_addresses ([], optional): list of instance private addresses
                **options: passed to create_node()

            Raises: exceptions from create_node()
        '''
        conn = self.connection()
        subnets = options.get('subnet') or cluster.subnets or self.subnets
        subnets = [subnets] if isinstance(subnets, basestring) else subnets
        # make sure to use unused subnets first
        unused_subnets = [s for s in self.subnets if s not in subnets]
        unused_subnets.extend([s for s in subnets if s not in unused_subnets])
        cycle_subnets = cycle(unused_subnets)

        # if we are preallocating our private_addresses
        private_addresses = options.pop('private_addresses', [None]*count)
        assert len(private_addresses) == count, "Not enough private addresses"

        # extend our service subnets if necessary
        for sn in subnets:
            if sn not in self.subnets:
                self.subnets.append(sn)

        instance        = cluster.nodes[0].instance
        bdm             = instance.block_device_mapping
        non_root_vols   = [_aws_do(conn.get_all_volumes, bdm[vol].volume_id)[0] \
                        for vol in bdm if vol != instance.root_device_name]
        machine_type    = cluster.machine_type
        data_disk_count = len(non_root_vols)
        data_disk_type  = non_root_vols[0].type
        data_disk_size  = non_root_vols[0].size
        data_disk_iops  = None
        root_image      = instance.image_id
        role            = self._get_iamrole(cluster.iamrole)
        tags            = instance.tags
        security_groups = [_.id for _ in instance.groups]

        # overrides
        if options.get('machine_type'):
            machine_type = options.pop('machine_type')
        if options.get('security_group_ids'):
            options['security_group_ids'] = security_groups
        if options.get('root_image'):
            root_image = options.pop('root_image')
        if options.get('data_disk_size'):
            data_disk_size = options.pop('data_disk_size')
        if options.get('data_disk_type'):
            data_disk_type = options.pop('data_disk_type')

        opts         = {'data_disk_type':data_disk_type, 'data_disk_count':data_disk_count,
                        'tags': tags.copy(), 'data_disk_size':data_disk_size,
                        'data_disk_iops':data_disk_iops, 'machine_type':machine_type,
                        'root_image':root_image, 'role':role}

        # Requires cluster be online
        # XXX assume our node name always ends in the node number
        max_node_num = max([int(i.name().split('-')[-1]) for i in cluster.nodes])

        joincfg     = cluster.cluster_config(joining=True)

        nodeq   = Queue.Queue()
        failq   = Queue.Queue()
        threads = []
        def cb(nodenum, inst_opts, nodeq, failq):
            '''callback'''
            try:
                name = '{}-{}'.format(cluster.name, nodenum)
                n = self.create_node(name, joincfg, node_opts=opts, instance_options=inst_opts)
                nodeq.put(n)
            except Exception as e:
                failq.put(e)

        for node_num in xrange(max_node_num, max_node_num+count):
            next_node_num = node_num + 1
            inst_opts = options.copy()
            inst_opts['subnet'] = next(cycle_subnets)
            inst_opts['private_ip_address'] = private_addresses.pop()
            t  = threading.Thread(target=cb, args=(next_node_num, inst_opts, nodeq, failq,))
            t.start()
            threads.append(t)

        for t in threads:
            t.join()

        nodes = []
        while True:
            try:
                n = nodeq.get_nowait()
                nodes.append(ServiceInstance(service=self, instance=n))
            except Queue.Empty:
                break

        failed = []
        while True:
            try:
                failed.append(failq.get_nowait())
            except Queue.Empty:
                break
        if failed:
            if not options.get('skip_cleanup', False):
                for n in nodes:
                    n.destroy()
            raise Exception(failed)

        cluster.nodes.extend(nodes)

    def load_cluster_information(self, cluster, **options):
        '''Loads cluster information from the service and cluster itself
        '''
        conn       = self.connection()
        xmlrpc     = cluster.xmlrpc()

        # make sure mgmt_ip is set to the valid address (in case we used
        # a node address to get in)
        cluster.mgmt_ip = xmlrpc.cluster.get()['mgmtIP']['IP']

        node_ips = set([ip['IP']
                        for name in xmlrpc.node.list()
                        for n in [xmlrpc.node.get(name)[name]]
                        if 'clusterIPs' in n
                        for ip in n['clusterIPs'] ])

        expr = {"private_ip_address":list(node_ips)}
        instances = [i for r in _aws_do(conn.get_all_reservations, filters=expr) for i in r.instances]

        # we should always find the private addresses in the cluster IP list
        #if not instances or len(instances) != len(node_ips): # maybe point 2 point
        #    vpc = self.connection(connection_type='vpc')
        #    route_instances = [self.get_instance(r.instance_id)
        #                          for rt in vpc.get_all_route_tables()
        #                          for r in rt.routes
        #                          if r.instance_id
        #                            and r.destination_cidr_block.endswith('/32')
        #                            and r.destination_cidr_block.split('/')[0] in node_ips ]
        #    instances = route_instances

        if instances:
            cluster.nodes = []
            for i in instances:
                cluster.nodes.append(ServiceInstance(service=self, instance=i))

            cluster.subnets      = list(set([i.subnet_id for i in instances]))
            # XXX assume all instances have the same settings
            cluster.iamrole      = instances[0].instance_profile['arn'].split('/')[-1]
            cluster.vpc_id       = instances[0].vpc_id
            cluster.region       = instances[0].region
            cluster.machine_type = instances[0].instance_type

    def can_shelve(self, instance):
        ''' Some instance configurations cannot be shelved. Check if this is one.

            Arguments:
                instance: backend instance
        '''
        return True

    def shelve(self, instance):
        ''' shelve the instance; shut it down, detach and delete
            all non-root block devices

            Arguments:
                instance: backend instance
            Raises: vFXTServiceFailure
        '''
        conn     = self.connection()

        if instance.state == self.DESTROY_STATUS:
            return

        bdm = instance.block_device_mapping
        if not bdm:
            raise vFXTServiceFailure("Failed to get block device mapping for {}".format(instance.id))

        non_root_vols = [_aws_do(conn.get_all_volumes, bdm[vol].volume_id)[0] \
                            for vol in bdm if vol != instance.root_device_name]

        if not non_root_vols:
            log.info("No non-root volumes for instance {}, already shelved?".format(instance.id))
            return

        log.debug("Found non-root volumes: {}".format(non_root_vols))

        if self.is_on(instance):
            self.stop(instance)

        # XXX assume all volume attributes are uniform
        tag = {
            'shelved': "{}|{}|{}".format(
                len(non_root_vols),
                non_root_vols[0].size,
                non_root_vols[0].type)
        }

        errors          = ShelveErrors()
        detach_failed   = []
        for nrv in non_root_vols:
            log.info("{}|{}: detaching and deleting {}".format(
                instance.tags['Name'],
                instance.id,
                nrv.id))
            _aws_do(conn.detach_volume, nrv.id, instance.id)
            try:
                self.wait_for_status(nrv, "available", retries=self.WAIT_FOR_SUCCESS)
                _aws_do(nrv.delete)
            except:
                detach_failed.append(nrv.id)

        if detach_failed:
            errors['notdetached'] = ",".join(detach_failed)

        if errors:
            # XXX hopefully we don't exceed 255 characters
            tag['shelved'] += '|{}'.format(errors)

        _aws_do(conn.create_tags, instance.id, tag)
        log.debug("Creating instance tags: {}".format(tag))

    def unshelve(self, instance, count_override=None, size_override=None, type_override=None):
        ''' bring our instance back to life.  This requires a tag called
            shelved that contains the number of disks and their size/type

            Arguments:
                instance: backend instance
                count_override (int, optional): number of data disks
                size_override (int, optional): size of data disks
                type_override (str, optional): type of data disks

            Raises: vFXTServiceFailure
        '''
        conn        = self.connection()

        # assume we've previously killed the data disks and set a tag
        if "shelved" not in instance.tags:
            log.info( "{} does not have shelved tag, skipping".format(instance.id))
            return
        # XXX assume instance is already stopped
        if instance.state != self.OFF_STATUS:
            log.info("{} is not stopped, skipping".format(instance.id))
            return

        try:
            attrs = instance.tags['shelved'].split("|")
            vol_count, vol_size, vol_type = attrs[0:3]
            log.debug("Split {}, {}, {} from {}".format(
                vol_count, vol_size, vol_type, instance.tags['shelved']))
        except:
            log.error("{} does not have data in the shelved tag".format(instance.id))
            return

        # verify our non-ephemeral disks (/dev/sd[f-p]) labels are available
        if '/dev/sdf' in instance.block_device_mapping:
            log.info("{} appears to already have data disks, skipping".format(instance.id))
            return

        if count_override:
            vol_count = count_override
        if size_override:
            vol_size = size_override
        if type_override:
            vol_type = type_override

        volumes_created = []
        try:
            # -- we have to support ephemeral disks
            # create an ephemeral instance (or find one) to examine
            # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
            drive_id_base = ord('f')
            for i in xrange(int(vol_count)):
                drive_name = drive_id_base + i
                dev_name   = "/dev/sd{:c}".format(drive_name)

                log.debug("{}|{}: creating {} volume {} of type {} in {}".format(
                    instance.tags['Name'],
                    instance.id,
                    vol_size,
                    dev_name,
                    vol_type,
                    instance.placement))
                vol = _aws_do(conn.create_volume, vol_size,
                                         instance.placement,
                                         encrypted=True, # XXX hardcoded?
                                         volume_type=vol_type)
                volumes_created.append(vol)

                tags = {}
                tags['Name'] = "{}-sd{:c}".format(instance.tags['Name'], drive_name)

                # get Owner, Department if available
                if 'Owner' in instance.tags:
                    tags['Owner']       = instance.tags['Owner']
                if 'Department' in instance.tags:
                    tags['Department']  = instance.tags['Department']

                log.debug("Creating volume tags: {}".format(tags))
                _aws_do(conn.create_tags, vol.id, tags)

                self.wait_for_status(vol, 'available')
                _aws_do(vol.attach, instance.id, dev_name)
                _aws_do(instance.modify_attribute, 'blockDeviceMapping', [ "{}=1".format(dev_name)])
                self.wait_for_status(vol, 'in-use')

        except boto.exception.BotoServerError as e:
            log.debug(e)
            log.error("Error while creating volumes, undoing what we did")
            for vol in volumes_created:
                if vol.attach_data and vol.attach_data.instance_id:
                    log.debug("Detaching volume {} from instance {}".format(vol.id, vol.attach_data.instance_id))
                    _aws_do(conn.detach_volume, vol.id, vol.attach_data.instance_id)
                    self.wait_for_status(vol, "available")
                log.debug("Deleting volume {}".format(vol.id))
                vol.delete()
            raise vFXTServiceFailure(e)

        _aws_do(conn.delete_tags, instance.id, ['shelved'])
        _aws_do(instance.start)

    def get_available_addresses(self, count=1, contiguous=False, addr_range=None, in_use=None):
        '''Returns a list of available addresses for the given range
            Arguments:
                count (int, optional): number of addresses required
                contiguous (bool=False): addresses must be contiguous
                addr_range (str, optional): address range cidr block
                in_use ([str], optional): list of addresses known to be used

            Returns:
                ([], str): tuple of address list and netmask str

            Raises: vFXTConfigurationException
        '''
        netmask    = None
        addr_range = addr_range or self.private_range
        if not addr_range:
            if len(self.subnets) == 1: # fall back to a subnet if only one
                addr_range = self._get_subnet(self.subnets[0]).cidr_block
            else: # lots of subnets, determine a private range outside vpc XXX review
                # this requires permission to view vpc
                vpc_cidr   = Cidr(self._subnet_to_vpc().cidr_block)
                addr_range = '{}/{}'.format(Cidr.to_address(vpc_cidr.end()+1),vpc_cidr.bits)
                netmask = '255.255.255.255'
        else:
                netmask = '255.255.255.255'

        used = self.in_use_addresses(addr_range)
        if in_use:
            used.extend(in_use)
            used = list(set(used))

        try:
            addr_cidr  = Cidr(addr_range)
            avail = addr_cidr.available(count, contiguous, used)
            if not netmask:
                netmask = addr_cidr.netmask
            return (avail, netmask)
        except Exception as e:
            log.exception(e)
            raise vFXTConfigurationException(e)

    def get_dns_servers(self, subnet_id=None):
        '''Get DNS server addresses

            Arguments:
                subnet_id (str): subnet id (optional if given to constructor)
            Returns:
                [str]: list of DNS server addresses
        '''
        subnet_id = subnet_id or self.subnets[0]
        # self.DNS_SERVERS is empty, we look to DHCP options
        try:
            opts = self._get_dhcp_options(subnet_id)
            dns  = opts['domain-name-servers']
            if dns and dns[0] == 'AmazonProvidedDNS':
                vpc_cidr = self._subnet_to_vpc(subnet_id).cidr_block
                default_addr = Cidr.to_address(Cidr(vpc_cidr).start()+2)
                dns = [default_addr]
            return dns
        except Exception as e:
            raise vFXTServiceFailure("Failed to determine DNS configuration: {}".format(e))

    def get_ntp_servers(self):
        '''Get NTP server addresses
            Returns:
                [str]: list of NTP server addresses
        '''
        return    self.NTP_SERVERS

    def get_default_router(self, subnet_id=None):
        '''Get default route address

            Arguments:
                subnet_id (str): subnet id (optional if given to constructor)
            Returns:
                str: address of default router
        '''
        subnet_id = subnet_id or self.subnets[0]
        subnet    = self._get_subnet(subnet_id)
        c         = Cidr(subnet.cidr_block)
        return    c.to_address(c.start() | 1)

    def export(self):
        '''Export the service object in an easy to serialize format
            Returns:
                {}: serializable dictionary
        '''
        export = {
            'region':self.region,
            'access_key':self.access_key,
            'secret_access_key':self.secret_access_key,
        }
        if self.s3_access_key != self.access_key:
            export['s3_access_key'] = self.s3_access_key
            export['s3_secret_access_key'] = self.s3_secret_access_key
            if self.s3_profile_name:
                export['s3_profile_name'] = self.s3_profile_name
        if self.subnets:
            export['subnet'] = self.subnets
        if self.profile_name:
            export['profile_name'] = self.profile_name
        if self.security_token:
            export['security_token'] = self.security_token
        if self.private_range:
            export['private_range'] = self.private_range
        if self.proxy_uri:
            export['proxy_uri'] = self.proxy_uri
        if self.iam_host:
            export['iam_host'] = self.iam_host
        if self.arn:
            export['arn'] = self.arn

        return export


    # AWS specific

    def _subnet_to_vpc(self, subnet_id=None):
        '''Lookup VPC by subnet

            Arguments:
                subnet_id (str): subnet id (optional if given to constructor)
            Returns:
                obj: VPC object
        '''
        subnet_id = subnet_id or self.subnets[0]
        subnet    = self._get_subnet(subnet_id)
        vpc       = subnet.connection.get_all_vpcs(vpc_ids=[subnet.vpc_id])[0]
        return    vpc

    def _get_dhcp_options(self, subnet_id=None):
        '''Get DHCP options

            Arguments:
                subnet_id (str): subnet id (optional if given to constructor)
            Returns:
                {}: dict of DHCP options
        '''
        subnet_id = subnet_id or self.subnets[0]
        vpc       = self._subnet_to_vpc(subnet_id)
        conn      = vpc.connection
        optid     = vpc.dhcp_options_id
        r         = _aws_do(conn.get_all_dhcp_options, dhcp_options_ids=[optid])
        return    r[0].options

    def _get_subnet(self, subnet_id=None):
        '''Get subnet object by ID

            Arguments:
                subnet_id (str): subnet id (optional if given to constructor)
            Returns:
                obj: subnet object
        '''
        subnet_id = subnet_id or self.subnets[0]
        vpc = self.connection(connection_type='vpc')
        subnet = _aws_do(vpc.get_all_subnets, subnet_ids=[subnet_id])[0]
        return subnet

    def in_use_addresses(self, cidr_block):
        '''Return a list of in use addresses within the specified cidr

            Arguments:
                cidr_block (str)
        '''
        vpcconn = self.connection(connection_type='vpc')
        c       = Cidr(cidr_block)
        used    = set()

        # XXX whole lotta expensiveness
        try:
            addrs = [r.destination_cidr_block.split('/')[0]
                    for rt in _aws_do(vpcconn.get_all_route_tables)
                    for r in rt.routes
                    if r.instance_id and
                    c.contains(r.destination_cidr_block.split('/')[0])]
            used.update(addrs)
        except Exception as e:
            log.error('Failed to determine in use addresses by routes: {}'.format(e))
            raise vFXTServiceFailure('Failed to determine in use addresses by routes: {}'.format(e))

        # search all instance private ip addresses that fail within the specified block
        # XXX whole lotta expensiveness
        try:
            conn = self.connection()
            addrs = [addr.private_ip_address
                        for r in _aws_do(conn.get_all_reservations)
                        for i in r.instances
                        for iface in i.interfaces
                        for addr in iface.private_ip_addresses
                        if c.contains(addr.private_ip_address)]
            used.update(addrs)
        except Exception as e:
            log.error('Failed to determine in use addresses by instances: {}'.format(e))
            raise vFXTServiceFailure('Failed to determine in use addresses by instances: {}'.format(e))

        return list(used)

    def _get_iamrole(self, name, retries=ServiceBase.CLOUD_API_RETRIES):
        '''Retrieve the IAM role object

            Arguments:
                name (str): IAM role name
                retries (int, optional): number of retries
            Returns:
                obj: IAM role object
        '''
        iam = self.connection(connection_type='iam')

        while True:
            try:
                role = _aws_do(iam.get_role, name)
                log.debug("Fetched role {}".format(name))
                return role['get_role_response']['get_role_result']['role']
            except Exception as e:
                if retries == 0:
                    raise vFXTServiceTimeout(e)
                retries -= 1

    def _create_iamrole(self, name):
        '''Create an IAM role

            Arguments:
                name (str): name of the IAM role to create
        '''
        iam = self.connection(connection_type='iam')

        policy = {
            'Version': '2012-10-17',
            'Statement': [
                {
                    'Action': self.IAM_POLICY,
                    'Resource': '*',
                    'Effect': 'Allow'
                },
            ]
        }

        role_policy = {
            "Version": "2012-10-17",
            "Statement": [{
                "Effect": "Allow",
                "Principal": { "Service": "ec2.amazonaws.com" },
                "Action": "sts:AssumeRole"
            }]
        }

        log.info("Creating instance profile {}".format(name))
        profile     = _aws_do(iam.create_instance_profile, name) #pylint: disable=unused-variable
        log.debug("Creating instance profile policy {}: {}".format(name, role_policy))
        role        = _aws_do(iam.create_role, name, json.dumps(role_policy))
        policy_name = 'policy_{}'.format(name)
        log.debug("Adding role policy {}: {}".format(policy_name, policy))
        _aws_do(iam.put_role_policy, name, policy_name, json.dumps(policy))
        log.info("Adding role {} to instance profile {}".format(name, name))
        _aws_do(iam.add_role_to_instance_profile, name, name)
        time.sleep(5) # XXX Wow, sync time between IAM service and EC2
        return role['create_role_response']['create_role_result']['role']

    def _delete_iamrole(self, name):
        '''Delete an IAM role and cleanup role/policies applied to it

            Arguments:
                name (str): name of the IAM role to delete
        '''
        iam = self.connection(connection_type='iam')

        log.info("Removing role {} from instance profile".format(name))
        _aws_do(iam.remove_role_from_instance_profile, name, name)
        log.info("Removing instance profile {}".format(name))
        _aws_do(iam.delete_instance_profile, name)

        resp = _aws_do(iam.list_role_policies, name)
        role_policies = resp['list_role_policies_response']['list_role_policies_result']['policy_names']
        for role_policy in role_policies:
            log.info("Removing role {} policy".format(role_policy))
            _aws_do(iam.delete_role_policy, name, role_policy)

        log.info("Removing role {}".format(name))
        _aws_do(iam.delete_role, name)

    def add_instance_address(self, instance, address, **options):
        '''Add a new instance private address or route

            Arguments:
                instance: backend instance
                address (str): IP address
                allow_reassignment (bool, optional): defaults to True
                route_tables (list, optional): list of additional route table IDs to add the address to

            If the address is not contained within the instances default subnet, point to point
            routes are added.

            Raises: vFXTServiceFailure
        '''
        try:
            addr = Cidr('{}/32'.format(address)) # validates
            existing = self.instance_in_use_addresses(instance)
            if addr.address in existing:
                raise vFXTConfigurationException("Address {} is already associated with instance {}".format(addr.address, self.name(instance)))

            # if part of the subnet, add to private addresses
            subnet = self._get_subnet(instance.subnet_id)
            subnet_block = Cidr(subnet.cidr_block)
            if subnet_block.contains(addr.address):
                conn = self.connection()
                _aws_do(conn.assign_private_ip_addresses, instance.interfaces[0].id,
                    private_ip_addresses=[addr.address],
                    allow_reassignment=options.get('allow_reassignment', True))
                log.debug("Added private address {}".format(addr.address))

            else: # otherwise we will need a route
                conn = self.connection(connection_type='vpc')
                route_tables = self._get_route_tables()
                route_tables.extend(options.get('route_tables', []))

                for rt_id in route_tables:
                    try:
                        _aws_do(conn.create_route, rt_id, '{}/32'.format(addr.address), instance_id=instance.id)
                        log.debug("Added routed address {} to {}".format(addr.address, rt_id))
                        continue
                    except Exception as e:
                        if not 'RouteAlreadyExists' in str(e):
                            raise
                    _aws_do(conn.replace_route, rt_id, '{}/32'.format(addr.address), instance_id=instance.id)
                    log.debug("Replaced routed address {} in {}".format(addr.address, rt_id))

        except vFXTConfigurationException:
            raise
        except Exception as e:
            raise vFXTServiceFailure("Failed to create new address for instance {}: {}".format(self.name(instance), e))

    def remove_instance_address(self, instance, address):
        '''Remove an instance private address or route

            Arguments:
                instance: backend instance
                address (str): IP address

            Raises: vFXTServiceFailure
        '''
        try:
            addr = Cidr('{}/32'.format(address)) # validates
            instance_addresses = self.instance_in_use_addresses(instance, 'instance')
            routed_addresses = self.instance_in_use_addresses(instance, 'routes')

            if addr.address in instance_addresses:
                conn = self.connection()
                _aws_do(conn.unassign_private_ip_addresses, instance.interfaces[0].id, [addr.address])
                log.debug("Removed private address {}".format(addr.address))
            elif addr.address in routed_addresses:
                conn = self.connection(connection_type='vpc')
                route_addr = '{}/32'.format(addr.address)
                route_tables = [rt.id
                    for rt in _aws_do(conn.get_all_route_tables)
                    for r in rt.routes
                    if r.destination_cidr_block == route_addr]
                for rt_id in route_tables:
                    log.debug("Removed routed address {} from route table {}".format(addr.address, rt_id))
                    _aws_do(conn.delete_route, rt_id, route_addr)
            else:
                raise vFXTServiceFailure("Address {} is not associated with instance {}".format(addr.address, self.name(instance)))
        except Exception as e:
            raise vFXTServiceFailure("Failed to remove address from instance {}: {}".format(self.name(instance), e))

    def instance_in_use_addresses(self, instance, category='all'):
        '''Get the in use addresses for the instance
            Arguments:
                instance (object)
                category (str): all (default), instance, routes

            To obtain the public instance address, use 'public' category.  This
            is not included with 'all'.
        '''
        addresses = set()
        if category in ['all','instance']:
            private_addrs = [addr.private_ip_address for iface in instance.interfaces for addr in iface.private_ip_addresses]
            addresses.update(set(private_addrs))

        if category in ['all','routes']:
            vpcconn     = self.connection(connection_type='vpc')
            route_addrs = [r.destination_cidr_block.split('/')[0] \
                            for rt in _aws_do(vpcconn.get_all_route_tables) \
                            for r in rt.routes \
                            if r.instance_id and
                                r.instance_id == instance.id and
                                r.destination_cidr_block.endswith('/32')
                            ]
            addresses.update(set(route_addrs))

        # for special requests
        if category == 'public':
            if instance.ip_address:
                addresses.add(instance.ip_address)

        return list(addresses)

    def _get_route_tables(self, subnets=None):
        '''Returns a list of associated route tables with the
            subnets specified

            Arguments:
                subnets ([], optional): defaults to service subnets

            Returns list
        '''
        subnets = subnets if subnets else self.subnets
        subnets = [self._get_subnet(subnet_id) for subnet_id in subnets]
        vpc_id = subnets[0].vpc_id
        vpcconn = self.connection(connection_type='vpc')
        try:
            return [a.route_table_id \
                for rt in _aws_do(vpcconn.get_all_route_tables, filters={'vpc-id':vpc_id})
                for a in rt.associations
                if a.main or a.subnet_id in subnets]
        except Exception as e:
            log.debug('Failed trying to lookup route tables: {}'.format(e))
        return []

    def _region_names(self):
        '''Get a list of region names
            Returns: list
        '''
        conn = self.connection()
        return [r.name for r in _aws_do(conn.get_all_regions)]

    def _s3_fetch(self, url, filename):
        '''Retrieve the object from S3, writing it to the passed in file location

            Arguments:
                url (str): s3:// url
                filename (str): name of destination file (absolute path)

            Returns: Nothing
            Raises: Exception
        '''
        s3   = self.connection(connection_type='s3')

        log.debug("Fetching {} to {}".format(url, filename))
        m = self.S3URL_RE.match(url)
        if not m:
            raise vFXTConfigurationException("Unknown url format: {}".format(url))

        bucket_name, path = m.groups()
        sig_path     = path + '.sig'
        sig_filename = filename + '.sig'
        bucket       = _aws_do(s3.get_bucket, bucket_name)
        obj          = _aws_do(bucket.get_key, path)
        if not obj:
            raise vFXTConfigurationException("No such object: {}".format(url))

        # check sig
        if os.access(sig_filename, os.F_OK) and os.access(filename, os.F_OK):
            sig_filename_tmp = sig_filename + '.tmp'
            with open(sig_filename_tmp, 'w') as f:
                k = bucket.get_key(sig_path)
                if k:
                    k.get_contents_to_file(f)
            sig_cmp = filecmp.cmp(sig_filename, sig_filename_tmp)
            os.unlink(sig_filename_tmp)
            if sig_cmp:
                log.debug('Signature {} up to date'.format(sig_path))
                return # cached, no download necessary
        # fetch sig
        sig_obj = _aws_do(bucket.get_key, sig_path)
        if sig_obj:
            with open(sig_filename, 'w') as f:
                sig_obj.get_contents_to_file(f)
        # fetch object
        with open(filename, 'w') as f:
            obj.get_contents_to_file(f)

    def _destroy_installation_image(self, name):
        '''Destroy an installation image

            Arguments:
            name (str): name of the AMI
        '''
        log.info("Destroying installation image {}".format(name))
        conn = self.connection()
        try:
            image = _aws_do(conn.get_image, name).id
        except Exception as e:
            log.debug(e)
            log.debug("Failed to get image {} by id, trying by name".format(name))
            try:
                image = _aws_do(conn.get_all_images, filters={'name':name})[0].id
            except Exception as e:
                log.debug(e)
                raise vFXTServiceFailure("Failed to find installation image {}".format(name))
        try:
            _aws_do(conn.deregister_image, image, delete_snapshot=True)
        except Exception as e:
            raise vFXTServiceFailure("Failed to destroy image {}: {}".format(name, e))

    def _cache_to_disk_config(self, cache_size, machine_type=None, disk_type=None):
        '''For a given cache size, output the default data disk count and size

            Arguments:
                cache_size (int): vFXT cluster node cache size
                machine_type (str, optional): vFXT cluster node machine type
                disk_type (str, optional): vFXT cluster node disk type

            Returns:
                tuple (disk count, size per disk)
        '''
        assert cache_size > 0
        count = 10 if cache_size >= 1000 else 1
        return tuple([count, int(cache_size/count)])


def _aws_do(function, *args, **kwargs):
    '''Retry boto operation with exponential backoff
        Arguments:
            function (function): function to call
            *args, **kwargs: arguments to pass to function

        Calls vFXT.service.backoff() with the error counter
    '''
    errors = 0
    while True:
        try:
            return function(*args, **kwargs)
        except boto.exception.BotoServerError as e:
            log.debug("_aws_do exception: {}".format(e))
            errors += 1
            # probably could be only 503 but boto also looks for 500
            if e.status < 500: raise

            time.sleep(backoff(errors))

            # give up after our max retry count
            if errors > Service.BOTO_503_RETRIES: raise


