# Copyright (c) 2015-2017 Avere Systems, Inc.  All Rights Reserved.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
''' Abstraction for doing things on instances via Google Compute

Cookbook/examples:

# With JSON key file provided by Google service account
gce = vFXT.gce.Service(network, zone, key_file=path_to_json)
# or with a P12 key file provided by Google service account
gce = vFXT.gce.Service(network, zone, client_email, project, key_file=path_to_p12)
# or if on a GCE compute instance, we can autodetect service account and settings
gce = vFXT.gce.Service.on_instance_init()

# Connection factory, has a thread specific copy
connection = gce.connection()
compute_conn = gce.connection(connection_type='compute')
storage_conn = gce.connection(connection_type='storage')

instances = gce.find_instances('') # filter string
instances = gce.get_instances([])

instance = gce.get_instance('instance id')
gce.start(instance)
gce.stop(instance)
gce.restart(instance)
gce.destroy(instance)

gce.shelve(instance)
gce.unshelve(instance)

instance = gce.refresh(instance)

print gce.name(instance)
print gce.ip(instance)
print gce.fqdn(instance)
print gce.status(instance)

if gce.is_on(instance): pass
if gce.is_off(instance): pass
if gce.is_shelved(instance): pass

gce.wait_for_status(instance, gce.ON_STATUS, retries=gce.WAIT_FOR_STATUS)

gce.create_instance(machine_type, name, boot_disk_image, other_disks=None, **options)
gce.create_cluster(self, cluster, **options)

gce.create_bucket(name)
gce.delete_bucket(name)

gce.load_cluster_information(cluster)

ip_count = 12
ip_addresses, mask = gce.get_available_addresses(count=ip_count, contiguous=True)
gce.get_dns_servers()
gce.get_ntp_servers()
gce.get_default_router()

serializeme = gce.export()
newgce = vFXT.gce.Service(**serializeme)
'''

import httplib2
import logging
import time
import threading
import Queue
import urllib2
import json
import socket
import re
import os
import filecmp
from itertools import cycle

import googleapiclient.discovery
import oauth2client.client #pylint: disable=unused-import
import googleapiclient
logging.getLogger('googleapiclient').setLevel(logging.CRITICAL)

from vFXT.cidr import Cidr
from vFXT.serviceInstance import ServiceInstance
from vFXT.service import *

log = logging.getLogger(__name__)

class Service(ServiceBase):
    '''GCE Service backend'''
    ON_STATUS="RUNNING"
    OFF_STATUS="TERMINATED"
    NTP_SERVERS=['169.254.169.254']
    DNS_SERVERS=['169.254.169.254']
    GCE_URL="https://www.googleapis.com/compute/v1/projects"
    GCE_INSTANCE_HOST='169.254.169.254'
    CONTROL_ADDR=None
    MACHINE_DEFAULTS={
        'f1-micro':         {'data_disk_size':200,'data_disk_type':'pd-standard','data_disk_count':1,'node_count':1,'root_disk_type':'pd-standard'},
        'g1-small':         {'data_disk_size':200,'data_disk_type':'pd-standard','data_disk_count':1,'node_count':1,'root_disk_type':'pd-standard'},
        'n1-highcpu-2':     {'data_disk_size':200,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':1,'root_disk_type':'pd-ssd'},
        'n1-highcpu-4':     {'data_disk_size':200,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-highcpu-8':     {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-highcpu-16':    {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-highcpu-32':    {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-highmem-2':     {'data_disk_size':200,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':1,'root_disk_type':'pd-ssd'},
        'n1-highmem-4':     {'data_disk_size':200,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-highmem-8':     {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-highmem-16':    {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-highmem-32':    {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-standard-1':    {'data_disk_size':200,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':1,'root_disk_type':'pd-ssd'},
        'n1-standard-2':    {'data_disk_size':200,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':1,'root_disk_type':'pd-ssd'},
        'n1-standard-4':    {'data_disk_size':200,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-standard-8':    {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-standard-16':   {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
        'n1-standard-32':   {'data_disk_size':250,'data_disk_type':'pd-ssd','data_disk_count':1,'node_count':3,'root_disk_type':'pd-ssd'},
    }
    MACHINE_TYPES=MACHINE_DEFAULTS.keys()
    DEFAULTS_URL="http://storage.googleapis.com/avere-dist/vfxtdefaults.json"
    DEFAULT_SCOPES=['https://www.googleapis.com/auth/compute',
                    'https://www.googleapis.com/auth/devstorage.full_control',
                    'https://www.googleapis.com/auth/userinfo.email']
    S3TYPE_NAME='GOOGLE'
    COREFILER_CRED_TYPE='s3'
    INSTANCENAME_RE=re.compile(r'[a-z]([-a-z0-9]*[a-z0-9])?')
    GSURL_RE=re.compile(r'gs://([^\/]*)/(.*)$')
    STORAGE_CLASSES=['STANDARD', 'NEARLINE', 'DURABLE_REDUCED_AVAILABILITY', 'MULTI_REGIONAL', 'REGIONAL', 'COLDLINE']
    ENDPOINT_TEST_HOSTS = ['www.googleapis.com']
    DISABLE_SSL_CERTIFICATE_VALIDATION=False
    OFFLINE_DEFAULTS={
      'version': '1',
      'clustermanager': {
        'maxNumNodes': 20,
        'instanceTypes': [ 'n1-highmem-8', 'n1-highmem-32' ],
        'cacheSizes': [
          {'label':'250-persistent-SSD', 'size': 250, 'type': 'pd-ssd'},
          {'label':'375-local-SSD', 'size': 375, 'type': 'local-ssd'},
          {'label':'1000-persistent-SSD', 'size': 1000, 'type': 'pd-ssd'},
          {'label':'1500-local-SSD', 'size': 1500, 'type': 'local-ssd'},
          {'label':'3000-local-SSD', 'size': 3000, 'type': 'local-ssd'},
          {'label':'4000-persistent-SSD', 'size': 4000, 'type': 'pd-ssd'},
          {'label':'8000-persistent-SSD', 'size': 8000, 'type': 'pd-ssd'}
        ]
      }
    }

    def __init__(self, network_id, zone, client_email=None, project_id=None,
                 key_file=None, key_data=None, access_token=None, s3_access_key=None,
                 s3_secret_access_key=None, private_range=None, proxy_uri=None,
                 no_connection_test=False, subnetwork_id=None, on_instance=False,
                 use_environment_for_auth=False):
        '''Constructor

            This performs an initial connection test and downloads the default
            data.

            Either a base64 encoded key string or the path to the service account
            P12/JSON key file must be provided.

            If the JSON key file is provided, client_email, project_id, and key are
            read from it.  Otherwise client_email and project_id must be specified.

            If an access token is provided, that will be used in place of the
            client_email + key/key_file.  This is only useful if running on a
            GCE instance.

            Arguments:
                network_id (str): network ID
                zone (str or []): one or more zones names

                client_email (str, optional): client email
                project_id (str, optional): project ID
                key_file (str, optional): file path to P12/JSON key file
                access_token (str, optional): existing access token

                private_range (str, optional): private address range (cidr)
                subnetwork_id (str, optional): subnetwork ID

                proxy_uri (str, optional): URI of proxy resource (e.g. http://user:pass@172.16.16.20:8080)

                no_connection_test (bool, optional): skip connection test
        '''
        self.defaults     = {}
        self.client_email = client_email
        self.key_file     = key_file
        self.key_data     = key_data
        self.access_token = access_token
        self.project_id   = project_id
        self.zones        = [zone] if isinstance(zone, basestring) else zone
        self.network_id   = network_id
        self.local        = threading.local()
        self.s3_access_key        = s3_access_key
        self.s3_secret_access_key = s3_secret_access_key
        self.private_range = private_range
        self.subnetwork_id = subnetwork_id
        self.proxy_uri     = proxy_uri
        self.on_instance   = on_instance
        self.use_environment_for_auth = use_environment_for_auth

        if not any([key_file, key_data, access_token, use_environment_for_auth]):
            raise vFXTConfigurationException("You must provide a keyfile or auth token")
        if self.key_data:
            try:
                self.client_email = self.key_data['client_email']
                self.project_id   = self.key_data['project_id']
            except KeyError:
                raise vFXTConfigurationException("Invalid key data: {}".format(self.key_data))
        elif self.key_file and self.key_file.endswith('.json'):
            with open(self.key_file, 'rb') as f:
                log.debug("Reading key data from {}".format(self.key_file))
                key_data = f.read()
                self.key_data = json.loads(key_data)
                try:
                    self.client_email = self.key_data['client_email']
                    self.project_id   = self.key_data['project_id']
                except KeyError:
                    raise vFXTConfigurationException("Invalid key file: {}".format(self.key_file))

        if not use_environment_for_auth:
            if not all([self.client_email, self.project_id]):
                raise vFXTConfigurationException("You must provide a keyfile or specify client_email and project_id")

        # emit third party library version information
        log.debug("Using googleapiclient version {}".format(googleapiclient.__version__))
        log.debug("Using oauth2client version {}".format(oauth2client.__version__))

        if self.proxy_uri: # parse proxy uri into host and port (and user, password)
            self.set_proxy(self.proxy_uri)

            proxy_handler = urllib2.ProxyHandler({'http':self.proxy_uri, 'https':self.proxy_uri})
            opener = urllib2.build_opener(proxy_handler)
            urllib2.install_opener(opener)
        else:
            self.proxy = None

        if not no_connection_test:
            self.connection_test()

            if self.subnetwork_id:
                subnetwork_names = [_['name'] for _ in self._get_subnetworks()]
                if self.subnetwork_id not in subnetwork_names:
                    err = "Invalid subnetwork: {} (available {})".format(self.subnetwork_id, ','.join(subnetwork_names))
                    raise vFXTConfigurationException(err)

        log.debug("Fetching defaults from {}".format(self.DEFAULTS_URL))
        load_defaults(self)

    @classmethod
    def get_instance_data(cls, source_address=None):
        '''Detect the instance data
            Arguments:
                source_address (str, optional): source address for data request

            This only works when running on a GCE instance.

            This is a service specific data structure.

            Well known keys that can be expected across services:
            machine_type (str): machine/instance type
            account_id (str): account identifier
            service_id (str): unique identifier for this host
            ssh_keys ([str]): ssh keys
            cluster_cfg (str): cluster configuration

            https://cloud.google.com/compute/docs/metadata
        '''
        import httplib

        if source_address:
            source_address = (source_address, 0)
        connection_host = cls.GCE_INSTANCE_HOST
        connection_port = httplib.HTTP_PORT

        conn          = httplib.HTTPConnection(connection_host, connection_port, source_address=source_address, timeout=CONNECTION_TIMEOUT)
        instance_data = {}
        headers       = {'Metadata-Flavor': 'Google'}
        attrs         = {
          'project-id':'/computeMetadata/v1/project/project-id',
          'numeric-project-id':'/computeMetadata/v1/project/numeric-project-id',
          'zone-id':'/computeMetadata/v1/instance/zone',
          'network-id':'/computeMetadata/v1/instance/network-interfaces/0/network',
          'ip':'/computeMetadata/v1/instance/network-interfaces/0/ip',
          'access-token':'/computeMetadata/v1/instance/service-accounts/default/token',
          'scopes':'/computeMetadata/v1/instance/service-accounts/default/scopes',
          'email':'/computeMetadata/v1/instance/service-accounts/default/email',
          'hostname':'/computeMetadata/v1/instance/hostname',
          'tags':'/computeMetadata/v1/instance/tags',
          'id':'/computeMetadata/v1/instance/id',
          'machine-type':'/computeMetadata/v1/instance/machine-type',
          'metadata_keys':'/computeMetadata/v1/instance/attributes/', # gives list b/c of trailing /
        }

        try:

            for k,v in attrs.iteritems():
                conn.request('GET', '{}'.format(v), headers=headers)
                response = conn.getresponse()
                if response.status == 200:
                    content = response.read()
                    try:
                        instance_data[k] = json.loads(content)
                    except ValueError as e:
                        instance_data[k] = content

            instance_data['metadata'] = {}
            for key in [_ for _ in instance_data['metadata_keys'].split('\n') if _]: # filter empty entries
                path = '{}{}'.format(attrs['metadata_keys'], key)
                conn.request('GET', '{}'.format(path), headers=headers)
                response = conn.getresponse()
                if response.status == 200:
                    content = response.read()
                    try:
                        instance_data['metadata'][key] = json.loads(content)
                    except ValueError as e:
                        instance_data['metadata'][key] = content

            if 'access-token' in instance_data:
                instance_data['access_token']  = instance_data['access-token']['access_token']
                instance_data['expires_in']    = instance_data['access-token']['expires_in']
                instance_data['token_expires'] = int(time.time()) + instance_data['expires_in'] - 120
            else: # try and support cloud shell
                # XXX right now this doesn't work... missing proper network-id (sets to default)
                try:
                    s = socket.socket()
                    s.connect(('localhost', int(os.getenv('DEVSHELL_CLIENT_PORT', '0'))))
                    s.sendall('2\n[]')
                    data = json.loads(s.recv(1024).split('\n')[1])
                    instance_data['email'] = data[0]
                    instance_data['project-id'] = data[1]
                    instance_data['access_token'] = data[2]
                    instance_data['token_expires'] = int(data[3]) - 120
                except Exception as e:
                    log.error('Failed to extract configuration for cloud shell: {}'.format(e))
                    raise

            instance_data['machine_type'] = instance_data['machine-type'].split('/')[-1]
            instance_data['account_id'] = instance_data['numeric-project-id']
            instance_data['service_id'] = instance_data['hostname'].split('.')[0]
            instance_data['ssh_keys'] = []
            if 'sshKeys' in instance_data['metadata']: # deprecated
                # prune username: from the key data
                instance_data['ssh_keys'] = [_.split(':')[-1] for _ in instance_data['metadata']['sshKeys'].split('\n')]
            if 'ssh-keys' in instance_data['metadata']:
                # prune username: from the key data
                instance_data['ssh_keys'].extend([_.split(':')[-1] for _ in instance_data['metadata']['ssh-keys'].split('\n')])

            instance_data['cluster_cfg'] = '' if 'cluster_cfg' not in instance_data['metadata'] \
                                        else instance_data['metadata']['cluster_cfg']\
                                            .replace('\\n','\n').replace(' ','\n').replace('_','\n').replace('@','=').decode('base64')
        except Exception as e:
            raise vFXTServiceMetaDataFailure('Unable to read instance metadata: {}'.format(e))
        finally:
            conn.close()

        return instance_data

    @classmethod
    def environment_init(cls, **kwargs):
        '''Init an GCE service object using the local environment credentials

            Arguments:
                **kwargs: arguments passed through to __init__
        '''
        kwargs['use_environment_for_auth'] = True
        return Service(**kwargs)

    @classmethod
    def on_instance_init(cls, source_address=None, no_connection_test=False, proxy_uri=None):
        '''Init a GCE service object from instance metadata
            Arguments:
                source_address (str, optional): source address for data request
                no_connection_test (bool, optional): skip connection tests, defaults to False
                proxy_uri (str, optional): URI of proxy resource

            This is only meant to be called on instance.  Otherwise will
            raise a vFXTConfigurationException exception.
        '''
        instance_data = cls.get_instance_data(source_address=source_address)
        log.debug('Read instance data: {}'.format(instance_data))

        if source_address:
            Service.CONTROL_ADDR = source_address

            import httplib
            import ssl
            import httplib2.socks
            class HTTPSConnectionFromSource(httplib2.HTTPSConnectionWithTimeout):
                """
                An override of httplib2's HTTPSConnectionWithTimeout that forces
                connections to come from our controlAddr.  __init__ is essentially
                a copy of the httplib2 version.
                """
                def __init__(self, host, port=None, key_file=None, cert_file=None,
                             strict=None, timeout=None, proxy_info=None,
                             ca_certs=None, disable_ssl_certificate_validation=False):
                    log.debug("Making connection to {} from {}".format(host, Service.CONTROL_ADDR))
                    httplib.HTTPSConnection.__init__(self, host, port=port,
                                                     key_file=key_file,
                                                     cert_file=cert_file, strict=strict, timeout=timeout,
                                                     source_address=(Service.CONTROL_ADDR, 0))
                    self.timeout = timeout
                    self.proxy_info = proxy_info
                    if ca_certs is None:
                        ca_certs = httplib2.CA_CERTS
                    self.ca_certs = ca_certs
                    self.disable_ssl_certificate_validation = disable_ssl_certificate_validation

                def connect(self):
                    "Connect to a host on a given (SSL) port."
                    if self.proxy_info and self.proxy_info.isgood():
                        sock = httplib2.socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
                        proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple()
                        sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)
                    else:
                        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

                    if self.source_address:
                        sock.bind(self.source_address)
                    if self.timeout:
                        sock.settimeout(float(self.timeout))

                    sock.connect((self.host, self.port))
                    # XXX set ssl_version in ssl.wrap_socket call
                    self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file)

            httplib2.SCHEME_TO_CONNECTION['https'] = HTTPSConnectionFromSource


        # check scopes for one of the following
        required_scopes = ['https://www.googleapis.com/auth/compute', 'https://www.googleapis.com/auth/cloud-platform']
        if not any([_ in required_scopes for _ in instance_data['scopes'].split('\n')]):
            raise vFXTConfigurationException("Compute R/W or Full Access required for this instance")

        try:
            project_id   = instance_data['project-id']
            zone_id      = instance_data['zone-id'].split('/')[-1]
            network_id   = instance_data['network-id'].split('/')[-1]
            access_token = instance_data['access_token']
            client_email = instance_data['email']
            srv = Service(network_id=network_id, client_email=client_email,
                          project_id=project_id, zone=zone_id,
                          access_token=access_token, no_connection_test=no_connection_test,
                          proxy_uri=proxy_uri, on_instance=True)
            srv.local.instance_data = instance_data
            region      = srv._zone_to_region(zone_id)
            # no subnetwork in metadata... figure out which subnetwork owns our address
            subnetworks = srv._get_subnetworks(region)
            if subnetworks:
                for subnetwork in subnetworks:
                    if Cidr(subnetwork['ipCidrRange']).contains(instance_data['ip']):
                        srv.subnetwork_id = subnetwork['name']
                if not srv.subnetwork_id:
                    srv.subnetwork_id = subnetworks[0]['name']
            return srv
        except (vFXTServiceFailure, vFXTServiceConnectionFailure) as e:
            raise
        except Exception as e:
            raise vFXTConfigurationException(e)

    def connection_test(self):
        '''Connection test

            Raises: vFXTConfigurationException
        '''
        log.debug("Performing connection test")

        try:
            if not self.proxy: # proxy environments may block outgoing name resolution
                self.dns_check()

            conn = self.connection(retries=0)
            resp = conn.projects().get(project=self.project_id).execute()
            for q in resp['quotas']:
                if q['usage']/q['limit'] > 0.9:
                    log.warn("QUOTA ALERT: Using {} of {} {}".format(int(q['usage']), int(q['limit']), q['metric']))
        except Exception as e:
            if isinstance(e, IOError):
                log.exception(e)
            raise vFXTServiceConnectionFailure("Failed to establish connection to service: {}".format(e))

    def check(self, percentage=0.6, instances=0, machine_type=None, data_disk_type=None, data_disk_size=None, data_disk_count=None):
        '''Check quotas and API access

            Arguments:
            percentage (float, optional): percentage as a decimal
            instances (int, optional): Number of planned for instances to account for
            machine_type (str, optional): Machine type
            data_disk_type (str, optional): Data disk type
            data_disk_size (int, optional): Data disk size
            data_disk_count (int, optional): Data disk count
        '''
        core_count = 0
        if machine_type and instances:
            machine_type_cores = 1
            try:
                machine_type_cores = int(machine_type.split('-')[-1])
            except ValueError: pass
            core_count = instances * machine_type_cores

        ssd_count = 0
        local_ssd_count = 0
        if all([data_disk_type, data_disk_count, data_disk_size]):
            if data_disk_type == 'local-ssd':
                local_ssd_count = data_disk_count * data_disk_size
            else:
                ssd_count = data_disk_count * data_disk_size

        conn = self.connection()
        project_quotas = conn.projects().get(project=self.project_id).execute()['quotas']
        for q in project_quotas:
            usage = int(q.get('usage') or 0)
            limit = int(q.get('limit') or 0)
            metric = q.get('metric').lower().capitalize().replace('_', ' ')
            if limit and usage/limit > percentage:
                log.warn("QUOTA ALERT: Using {} of {} {} for the project".format(usage, limit, metric))
            else:
                log.debug("Using {} of {} {} for the project".format(usage, limit, metric))

        region = self._zone_to_region(self.zones[0])
        region_quotas = conn.regions().get(project=self.project_id, region=region).execute()['quotas']
        for q in region_quotas:
            usage = int(q.get('usage') or 0)
            limit = int(q.get('limit') or 0)
            metric = q.get('metric')

            if metric == 'CPUS':
                usage += core_count
            if metric == 'SSD_TOTAL_GB':
                usage += ssd_count
            if metric == 'LOCAL_SSD_TOTAL_GB':
                usage += local_ssd_count

            metric = metric.lower().capitalize().replace('_', ' ')
            if limit and usage/limit > percentage:
                log.warn("QUOTA ALERT: Using {} of {} {} for the region".format(usage, limit, metric))
            else:
                log.debug("Using {} of {} {} for the region".format(usage, limit, metric))

    def _auth_http(self, scopes=None):
        '''Simple wrapper for the HTTP object credential authorization

            Do not call this directly, use connection() instead.

            Arguments:
                scopes ([str], optional): list of scopes to request, defaults to DEFAULT_SCOPES
        '''
        creds = None
        scopes = scopes or self.DEFAULT_SCOPES
        if self.access_token:
            from oauth2client.client import AccessTokenCredentials
            # we check for access_token presence but use the threading.local copy
            creds = AccessTokenCredentials(self.local.access_token, 'vFXT UserAgent/0.1')
        elif self.use_environment_for_auth:
            creds = oauth2client.client.GoogleCredentials.get_application_default()
        else:
            try:
                from oauth2client.service_account import ServiceAccountCredentials
                if self.key_data:
                    creds = ServiceAccountCredentials.from_json_keyfile_dict(self.key_data, scopes)
                elif self.key_file.endswith('.p12'):
                    creds = ServiceAccountCredentials.from_p12_keyfile(self.client_email, self.key_file, scopes)
                else:
                    raise vFXTConfigurationException("Unknown key file type: {}".format(self.key_file))
            except Exception as e:
                log.debug('Failed importing oauth2client.service_account (oauth2client 2.x), falling back to oauth2client 1.x: {}'.format(e))
                with open(self.key_file, 'rb') as f:
                    key = f.read()
                if self.key_file.endswith('.json'):
                    key = json.loads(key)['private_key']
                from oauth2client.client import SignedJwtAssertionCredentials
                creds = SignedJwtAssertionCredentials(self.client_email, key, scopes)

        proxy = None
        if self.proxy_uri:
            proxy = httplib2.proxy_info_from_url(self.proxy_uri)
        else:
            try:
                proxy = httplib2.proxy_info_from_environment()
            except Exception as e:
                log.debug("httplib2.proxy_info_from_environment(): {}".format(e))
        # maybe set this for those proxies that don't support CONNECT?
        # proxy.proxy_type = httplib2.socks.PROXY_TYPE_HTTP_NO_TUNNEL

        http = httplib2.Http(proxy_info=proxy,
                disable_ssl_certificate_validation=self.DISABLE_SSL_CERTIFICATE_VALIDATION, timeout=CONNECTION_TIMEOUT)
        return creds.authorize(http)

    def connection(self, connection_type='compute', version='v1', retries=CONNECTION_TIMEOUT, scopes=None):
        '''Connection factory, returns a new connection or thread local copy

            Arguments:
                connection_type (str, optional): connection type (compute, storage)
                version (str, optional): currently unused
                retries (int, optional): number of retries, default to vFXT.service.CONNECTION_TIMEOUT
                scopes ([str], optional): list of scopes to request, defaults to DEFAULT_SCOPES
        '''
        try:
            if self.local.instance_data['token_expires'] < int(time.time()):
                log.debug("Access token expired, forcing refresh")
                self.local.connections = {}
        except: pass

        if not hasattr(self.local, 'connections'):
            self.local.connections = {}

        connection_sig = '{}_{}'.format(connection_type, version)
        if not self.local.connections.get(connection_sig, False):
            if self.access_token:
                self.local.instance_data = self.get_instance_data()
                self.local.access_token = self.local.instance_data['access_token']

            log.debug("Creating new {} connection object".format(connection_type))
            connection_attempts = 0
            while True:
                try:
                    self.local.connections[connection_sig] = googleapiclient.discovery.build(connection_type, version, http=self._auth_http(scopes=scopes))
                    break
                except Exception as e:
                    if connection_attempts == retries:
                        raise vFXTServiceConnectionFailure("Failed to establish connection to service: {}".format(e))
                    log.debug("Retrying failed connection attemp: {}".format(e))
                    connection_attempts += 1
                    time.sleep(backoff(connection_attempts))

        return self.local.connections[connection_sig]

    def find_instances(self, search=None, all_regions=True):
        '''Returns all or filtered list of instances

            Arguments:
                search (str, optional): search query
                all_regions (bool, optional): search all regions, not just the current

            Search examples:
                field [ne|eq] expression
                'name eq instance-name'
                'name eq (name1|name2|name3)'
                'name eq prefix.*$'
        '''
        conn = self.connection()
        instances = []
        for zone in self._zone_names(all_regions):
            r =  _gce_do(conn.instances().list, project=self.project_id, filter=search, zone=zone)
            if r and 'items' in r:
                instances.extend(r['items'])
        return instances

    def get_instances(self, instance_ids, all_regions=True):
        '''Returns a list of instances with the given instance ID list

            Arguments:
                instance_ids ([str]): list of instance id strings
                all_regions (bool, optional): search all regions, not just the current

            Returns:
                [objs]: list of backend instance objects
        '''
        id_str = '|'.join(instance_ids)
        search = 'name eq {}'.format(id_str)
        conn   = self.connection()
        instances = []
        for zone in self._zone_names(all_regions):
            r = _gce_do(conn.instances().list, project=self.project_id, filter=search, zone=zone)
            if r and 'items' in r:
                instances.extend(r['items'])
        return instances

    def get_instance(self, instance_id, all_regions=True):
        '''Get a specific instance by instance ID

            Arguments:
                instance_id (str)
                all_regions (bool, optional): search all regions, not just the current

            Returns:
                obj or None
        '''
        conn   = self.connection()
        for zone in self._zone_names(all_regions):
            try:
                return _gce_do(conn.instances().get, project=self.project_id, instance=instance_id, zone=zone)
            except:
                pass
        return None

    def wait_for_status(self, instance, status, retries=ServiceBase.WAIT_FOR_STATUS):
        '''Poll on a given instance for status

            Arguments:
                instance (obj): backend instance object
                status (str): status string to watch for
                retries (int, optional): number of retries

            Raises: vFXTServiceTimeout
        '''
        s = '...' # in case our instance is not yet alive
        errors = 0
        while status != s:
            if retries % 10 == 0: # rate limit
                log.debug("Waiting for status: {} != {}".format(s, status))
            time.sleep(self.POLLTIME)
            try:
                instance = self.refresh(instance)
                s = self.status(instance)
            except Exception as e:
                log.debug('Ignored: {}'.format(e))
                errors += 1
                time.sleep(backoff(errors))
            retries -= 1
            if retries == 0:
                raise vFXTServiceTimeout("Timed out waiting for {} on {}".format(status, instance['name']))

    def _wait_for_operation(self, response, msg='operation to complete', retries=ServiceBase.WAIT_FOR_OPERATION, op_type='zoneOperations', zone=None):
        '''Wait for an operation to complete by polling the response

            Arguments:
                response (obj): response object from a prior service query
                msg (str, optional): string to debug log for this operation
                retries (int, optional): number of retries
                op_type (str, optional): zoneOperations, globalOperations, ...

            Raises: vFXTServiceFailure
        '''
        conn        = self.connection()
        op          = conn.__getattribute__(op_type)
        errors      = 0
        while response['status'] != 'DONE':
            try:
                time.sleep(self.POLLTIME)
                if retries % 10 == 0:
                    log.debug("Waiting for {}: {}".format(msg, response['status']))
                operation   = response['name']
                args        = {'project':self.project_id, 'operation': operation}
                if op_type == 'zoneOperations':
                    args['zone'] = zone or self.zones[0]
                response    = _gce_do(op().get, **args)
            except googleapiclient.errors.HttpError as e:
                if int(e.resp['status']) < 500:
                    if 'httpErrorMessage' in response:
                        raise vFXTServiceFailure("{}: {}".format(response['httpErrorMessage'], response['error']['errors'][0]['message']))
                    else:
                        raise vFXTServiceFailure(e)
                errors += 1
                time.sleep(backoff(errors))
            retries -= 1
            if retries == 0:
                raise vFXTServiceTimeout("Failed waiting for operation to complete")

        if 'httpErrorMessage' in response:
            log.debug("response {}".format(response))
            raise vFXTServiceFailure("{}: {}".format(response['httpErrorMessage'], response['error']['errors'][0]['message']))

    def can_stop(self, instance):
        ''' Some instance configurations cannot be stopped. Check if this is one.

            Arguments:
                instance: backend instance
        '''
        if 'SCRATCH' in [d['type'] for d in instance['disks']]:
            raise vFXTConfigurationException("Cannot stop instance {} with local-ssd disks".format(self.name(instance)))
        return True

    def stop(self, instance, wait=ServiceBase.WAIT_FOR_STOP):
        '''Stop an instance

            Arguments:
                instance: backend instance
        '''
        if not self.can_stop(instance):
            raise vFXTConfigurationException("Node configuration prevents them from being stopped")
        log.info("Stopping instance {}".format(self.name(instance)))
        conn        = self.connection()
        zone        = instance['zone'].split('/')[-1]
        response    = _gce_do(conn.instances().stop, project=self.project_id, zone=zone, instance=instance['name'])
        log.debug(response)
        self.wait_for_status(instance, self.OFF_STATUS, retries=wait)

    def start(self, instance, wait=ServiceBase.WAIT_FOR_START):
        '''Start an instance

            Arguments:
                instance: backend instance
                wait (int): wait time
        '''
        log.info("Starting instance {}".format(self.name(instance)))
        conn        = self.connection()
        zone        = instance['zone'].split('/')[-1]
        response    =  _gce_do(conn.instances().start, project=self.project_id, zone=zone, instance=instance['name'])
        log.debug(response)
        self.wait_for_status(instance, self.ON_STATUS, retries=wait)

    def restart(self, instance, wait=ServiceBase.WAIT_FOR_RESTART):
        '''Restart an instance

            Arguments:
                instance: backend instance
                wait (int): wait time
        '''
        if not self.can_stop(instance):
            raise vFXTConfigurationException("Node configuration prevents them from being restarted")
        log.info("Restarting instance {}".format(self.name(instance)))
        # GCE does not have a reboot option, only reset (which is not what we want)
        self.stop(instance)
        self.start(instance)

    def destroy(self, instance, wait=ServiceBase.WAIT_FOR_DESTROY, keep_root_disk=False):
        '''Destroy an instance

            Arguments:
                instance: backend instance
                wait (int): wait time
                keep_root_disk (bool, optional): keep the root disk
        '''
        conn        = self.connection()
        zone        = instance['zone'].split('/')[-1]

        root_disk = None
        if keep_root_disk:
            root_disks = [ d for d in instance['disks'] if 'boot' in d and d['boot']]
            if not root_disks:
                raise vFXTServiceFailure("Failed to find root disk")
            root_disk = root_disks[0]
            self._disable_disk_auto_delete(instance, root_disk['deviceName'])

        log.info("Destroying instance {}".format(self.name(instance)))
        response =  _gce_do(conn.instances().delete, project=self.project_id, zone=zone, instance=instance['name'])
        # we wait because we cannot destroy resources still attached to the instance
        self._wait_for_operation(response, msg='instance to be destroyed', retries=wait, zone=zone)

        for d in instance['disks']:
            # skip our root if we want to keep it
            if root_disk and d['deviceName'] == root_disk['deviceName']:
                continue
            try: # need to delete any leftover disks
                resp = _gce_do(conn.disks().delete, project=self.project_id, zone=zone, disk=d['deviceName'])
                self._wait_for_operation(resp, msg='disk to be deleted', zone=zone)
            except: pass

        expr   = "nextHopInstance eq .*/{}$".format(instance['name'])
        routes = _gce_do(conn.routes().list, project=self.project_id,filter=expr)
        if not routes or 'items' not in routes:
            return
        for route in routes['items']:
            try: # need to delete any leftover routes
                resp = _gce_do(conn.routes().delete, project=self.project_id, route=route['name'])
                self._wait_for_operation(resp, msg='route to be deleted', zone=zone)
            except: pass

    def is_on(self, instance):
        '''Return True if the instance is currently on

            Arguments:
                instance: backend instance
        '''
        return instance['status'] != self.OFF_STATUS

    def is_off(self, instance):
        '''Return True if the instance is currently off

            Arguments:
                instance: backend instance
        '''
        return instance['status'] == self.OFF_STATUS

    def is_shelved(self, instance):
        '''Return True if the instance is currently shelved

            Arguments:
                instance: backend instance
        '''
        try:
            metadata = instance['metadata']['items']
            return 'shelved' in [v for opts in metadata for v in opts.values()]
        except:
            return False

    def name(self, instance):
        '''Returns the instance name (may be different from instance id)

            Arguments:
                instance: backend instance
        '''
        return instance['name']

    def instance_id(self, instance):
        '''Returns the instance id (may be different from instance name)

            Arguments:
                instance: backend instance
        '''
        return instance['name']

    def status(self, instance):
        '''Return the instance status

            Arguments:
                instance: backend instance
        '''
        return instance['status']

    def refresh(self, instance):
        '''Refresh the instance from the Google backend

            Arguments:
                instance: backend instance
        '''
        i = self.get_instance(instance['name'])
        if not i:
            raise vFXTConfigurationException("Failed to find instance: {}".format(instance['name']))
        return i

    def ip(self, instance):
        '''Return the primary IP address of the instance

            Arguments:
                instance: backend instance
        '''
        return instance['networkInterfaces'][0]['networkIP']

    def fqdn(self, instance): # XXX revisit
        '''Provide the fully qualified domain name of the instance

            Arguments:
                instance: backend instance
        '''
        name = instance['name']
        return "{}.c.{}.internal".format(name, self.project_id)

    def can_shelve(self, instance):
        ''' Some instance configurations cannot be shelved. Check if this is one.

            Arguments:
                instance: backend instance
        '''
        if 'SCRATCH' in [d['type'] for d in instance['disks']]:
            log.error("Cannot shelve {} with local-ssd disks".format(instance['name']))
            return False
        return True

    def shelve(self, instance):
        ''' shelve the instance; shut it down, detach and delete
            all non-root block devices

            Arguments:
                instance: backend instance
            Raises: vFXTServiceFailure
        '''
        conn     = self.connection()
        zone     = instance['zone'].split('/')[-1]

        if not self.can_shelve(instance):
            raise vFXTConfigurationException("Node configuration prevents them from being shelved")

        if self.is_on(instance):
            self.stop(instance)
            instance = self.refresh(instance)

        disks          = instance['disks']
        non_root_disks = [ d for d in disks if 'boot' not in d or not d['boot'] ]

        if not non_root_disks:
            log.info("No non-root volumes for instance {}, already shelved?".format(instance['name']))
            return

        log.debug("Found non-root volumes: {}".format(non_root_disks))

        disk_srv      = conn.disks()
        errors        = ShelveErrors()
        detach_failed = []
        disk_size     = None
        disk_type     = None
        for nrd in non_root_disks:
            name = nrd['source'].split('/')[-1]
            data = _gce_do(disk_srv.get, project=self.project_id, zone=zone, disk=name)

            log.info("{}: detaching and deleting {}".format(instance['name'], name))

            try:
                r = _gce_do(conn.instances().detachDisk, project=self.project_id, zone=zone, instance=instance['name'], deviceName=nrd['deviceName'])
                self._wait_for_operation(r, msg='disk to be detached', zone=zone)
                r = _gce_do(disk_srv.delete, project=self.project_id, zone=zone, disk=name)
                self._wait_for_operation(r, msg='disk to be deleted', zone=zone)
            except:
                detach_failed.append(name)

            # XXX assume all volume attributes are uniform
            disk_size = data['sizeGb']
            disk_type = data['type']

        if detach_failed:
            errors['notdetached'] = ','.join(detach_failed)

        shelved = "{}|{}|{}".format(len(non_root_disks), disk_size, disk_type)

        if errors:
            shelved += '|{}'.format(errors)

        log.debug("Creating shelved metadata: {}".format(shelved))
        instance = self.refresh(instance)
        self._set_metadata(instance, "shelved", shelved)

    def unshelve(self, instance, count_override=None, size_override=None, type_override=None, **options): #pylint: disable=unused-argument
        ''' bring our instance back to life.  This requires a metadata tag called
            shelved that contains the number of disks and their size/type

            Arguments:
                instance: backend instance
                count_override (int, optional): number of data disks
                size_override (int, optional): size of data disks
                type_override (str, optional): type of data disks

            Raises: vFXTServiceFailure
        '''
        conn     = self.connection()
        zone     = instance['zone'].split('/')[-1]

        # assume we've previously killed the data disks and set a tag
        if not self._get_metadata(instance, "shelved"):
            log.info( "{} does not have shelved tag, skipping".format(instance['name']))
            return

        # XXX assume instance is already stopped
        if self.is_on(instance):
            log.info("{} is not stopped, skipping".format(instance['name']))
            return

        try:
            attrs = self._get_metadata(instance, "shelved").split('|')
            vol_count, vol_size, vol_type = attrs[0:3]
        except:
            log.error("{} does not have data in the shelved tag".format(instance['name']))
            return

        if len(instance['disks']) > 1:
            log.info("{} appears to already have data disks, skipping".format(instance['name']))
            return

        if count_override:
            vol_count = count_override
        if size_override:
            vol_size = size_override
        if type_override:
            vol_type = type_override

        i_srv           = conn.instances()
        d_srv           = conn.disks()
        disks_created   = []
        try:
            for i in range(int(vol_count)):
                disk_name   = "{}-data-{}".format(instance['name'], i+1)
                body        = {'name':disk_name, "sizeGb":long(vol_size), 'type':vol_type}

                log.info("{}: creating {} volume {}".format(instance['name'], vol_size, disk_name))
                r = _gce_do(d_srv.insert, project=self.project_id, zone=zone, body=body)
                self._wait_for_operation(r, msg='disk to be created', zone=zone)

                d = _gce_do(d_srv.get, project=self.project_id, zone=zone, disk=disk_name)
                disks_created.append(d)

                body        = {'deviceName':disk_name, "source":d['selfLink'], "autoDelete": True}
                log.info("{}: attaching disk {}".format(instance['name'], disk_name))
                r = _gce_do(i_srv.attachDisk, project=self.project_id, zone=zone, instance=instance['name'], body=body)
                self._wait_for_operation(r, msg='disk to be attached', zone=zone)
        except Exception as e:
            log.debug(e)
            log.error("Error while creating volumes, undoing what we did")
            instance = self.refresh(instance)
            for d in disks_created:
                if d['name'] in [dev['deviceName'] for dev in instance['disks']]:
                    r = _gce_do(i_srv.detachDisk, project=self.project_id, zone=zone, instance=instance['name'], deviceName=d['name'])
                    self._wait_for_operation(r, msg='disk to be detached', zone=zone)
                r = _gce_do(d_srv.delete, project=self.project_id, zone=zone, disk=d['name'])
                self._wait_for_operation(r, msg='disk to be deleted', zone=zone)
            raise vFXTServiceFailure(e)

        self.start(instance)
        instance = self.refresh(instance)
        self._delete_metadata(instance, 'shelved')

    # No GCE equivalent
    def wait_for_service_checks(self, instance, retries=ServiceBase.WAIT_FOR_SERVICE_CHECKS): pass

    # storage/buckets
    def create_bucket(self, name, **options):
        '''Create a bucket

            Arguments:
                name (str): bucket name to create
                storage_class (str, optional): storage class of MULTI_REGIONAL, REGIONAL,
                    STANDARD, NEARLINE, COLDLINE, and DURABLE_REDUCED_AVAILABILIT
                region (str, optional): region for the bucket if using REGIONAL (defaults to service default region)

            Raises: vFXTServiceFailure
        '''
        if not self.valid_bucketname(name):
            raise vFXTConfigurationException("{} is not a valid bucket name".format(name))
        storage_service = self.connection(connection_type='storage')

        storage_class = options.get('storage_class') or 'STANDARD'
        if not storage_class in self.STORAGE_CLASSES:
            raise vFXTConfigurationException("{} is not a valid storage class".format(storage_class))

        body = {'name':name, 'storageClass': storage_class}
        if storage_class == 'REGIONAL':
            region = options.get('region') or self._zone_to_region(self.zones[0])
            body['location'] = region
        log.debug("Bucket create request {}".format(body))
        return _gce_do(storage_service.buckets().insert,project=self.project_id, body=body)

    def delete_bucket(self, name):
        '''Delete a bucket

            Arguments:
                name (str): bucket name

            Raises: vFXTServiceFailure
        '''
        try:
            storage_service = self.connection(connection_type='storage')
            _gce_do(storage_service.buckets().delete, bucket=name)
        except Exception as e:
            raise vFXTServiceFailure("Failed to delete bucket {}: {}".format(name, e))

    def authorize_bucket(self, cluster, name, retries=ServiceBase.CLOUD_API_RETRIES):
        '''Perform any backend work for the bucket, and register a credential
        for it to the cluster

            No authorization is currently performed for GCE.

            Arguments:
                cluster (Cluster): cluster object
                name (str): bucket name
                retries (int): number of attempts to make

            Raises: vFXTServiceFailure
        '''
        xmlrpc = cluster.xmlrpc()
        # see if we have s3 interop credentials
        if self.s3_access_key and self.s3_secret_access_key:
            log.debug("Found s3 access keys")
            cred = 's3-{}'.format(cluster.name)

            # if it exists, use it
            if cred in [c['name'] for c in xmlrpc.corefiler.listCredentials()]:
                return cred

            log.debug("Creating credential {}".format(cred))
            create_retries = retries
            while True:
                try:
                    r = xmlrpc.corefiler.createCredential(cred, self.COREFILER_CRED_TYPE, {
                        'accessKey':self.s3_access_key,
                        'privateKey':self.s3_secret_access_key,
                    })
                    if r != 'success':
                        raise Exception(r)
                    return cred

                except Exception as e:
                    if create_retries == 0:
                        raise vFXTConfigurationException("Could not create credential {}: {}".format(cred, e))
                create_retries -= 1

        # otherwise use the first default
        return xmlrpc.corefiler.listCredentials()[0]['name']

    # networking
    def get_default_router(self, subnetwork=None):
        '''Get default route address

            Arguments:
                subnetwork (str, optional): subnetwork name

            Returns:
                str: address of default router
        '''
        network = self._get_network()

        if 'gatewayIPv4' in network:
            return network['gatewayIPv4']

        if 'subnetworks' in network:
            region = self._zone_to_region(self.zones[0])
            region_gateways = [] # subnetwork gateways associated with our region
            subnetwork = subnetwork or self.subnetwork_id

            # try to find a direct match if we have a subnetwork
            subnetworks = self._get_subnetworks(region)
            for sn in subnetworks:
                if subnetwork and sn['name'] == subnetwork:
                    return sn['gatewayAddress']
                region_gateways.append(sn['gatewayAddress'])

            # otherwise pick one associated with our region
            if region_gateways:
                return region_gateways[0]

            raise vFXTConfigurationException("Unable to determine default router for this configuration")

    def get_dns_servers(self):
        '''Get DNS server addresses

            Returns:
                [str]: list of DNS server addresses
        '''
        dns = []
        dns.extend(self.DNS_SERVERS)
        dns.insert(0, self.get_default_router())
        return dns

    def get_ntp_servers(self):
        '''Get NTP server addresses

            Returns:
                [str]: list of NTP server addresses
        '''
        return self.NTP_SERVERS

    def in_use_addresses(self, cidr_block, category='all'):
        '''Return a list of in use addresses within the specified cidr

            Arguments:
                cidr_block (str)
                category (str): all, interfaces, routes
        '''
        conn      = self.connection()
        c         = Cidr(cidr_block)
        addresses = set()

        if category in ['all', 'interfaces']:
            for instance in self.find_instances(all_regions=False):
                for interface in instance['networkInterfaces']:
                    interface_address = interface.get('networkIP')
                    if not interface_address:
                        continue
                    if c.contains(interface_address):
                        addresses.add(interface_address)

        if category in ['all', 'routes']:
            search = 'destRange eq .*/32' # only point to point addresses
            resp   = _gce_do(conn.routes().list, project=self.project_id, filter=search)
            if resp and 'items' in resp:
                for route in resp['items']:
                    addr = route['destRange'].split('/')[0]
                    if c.contains(addr):
                        addresses.add(addr)

        return list(addresses)

    def get_available_addresses(self, count=1, contiguous=False, addr_range=None, in_use=None):
        '''Returns a list of available addresses for the given range
            Arguments:
                count (int, optional): number of addresses required
                contiguous (bool=False): addresses must be contiguous
                addr_range (str, optional): address range cidr block
                in_use ([str], optional): list of addresses known to be used

            Returns:
                ([], str): tuple of address list and netmask str

            Raises: vFXTConfigurationException
        '''
        network       = self._get_network()
        network_range = None
        range_offset  = 1
        if 'IPv4Range' in network:
            network_range = network['IPv4Range']
        elif 'subnetworks' in network:
            # find the last block from the subnetworks
            subnetworks = self._get_subnetworks()
            addrs = sorted([Cidr(_['ipCidrRange']) for _ in subnetworks], key=lambda x: x.addr)
            network_range = str(addrs[-1])
            # for auto setup subnetworks, we have to go beyond the /16 padding
            if network['autoCreateSubnetworks']:
                range_offset = 65536
        else:
            if not addr_range:
                raise vFXTConfigurationException("Unable to determine network/subnetwork range")
        # find an unused range, either provided or default network_range + range_offset
        addr_range = addr_range or self.private_range or \
                        "{}/{}".format(Cidr.to_address(Cidr(network_range).end()+range_offset),
                                        network_range.split('/')[-1])
        used          = self.in_use_addresses(addr_range)
        if in_use:
            used.extend(in_use)
            used = list(set(used))
        addr_cidr  = Cidr(addr_range)

        try:
            avail   = addr_cidr.available(count, contiguous, used)
            netmask = "255.255.255.255" # hardcoded for gce /32
            return (avail, netmask)
        except Exception as e:
            raise vFXTConfigurationException(e)

    def export(self):
        '''Export the service object in an easy to serialize format
            Returns:
                {}: serializable dictionary
        '''
        data = {
            'zone':self.zones,
            'network_id':self.network_id,
            'client_email':self.client_email,
            'project_id':self.project_id
        }
        if self.key_file:
            data['key_file'] = self.key_file
        if self.key_data:
            data['key_data'] = self.key_data
        if self.access_token:
            data['access_token'] = self.access_token
        if self.s3_access_key:
            data['s3_access_key'] = self.s3_access_key
        if self.s3_secret_access_key:
            data['s3_secret_access_key'] = self.s3_secret_access_key
        if self.private_range:
            data['private_range'] = self.private_range
        if self.proxy_uri:
            data['proxy_uri'] = self.proxy_uri
        if self.subnetwork_id:
            data['subnetwork_id'] = self.subnetwork_id
        return data

    def create_instance(self, machine_type, name, boot_disk_image, other_disks=None, **options):
        '''Create and return a GCE instance

            Arguments:
                machine_type (str): GCE machine type
                name (str): name of the instance
                boot_disk_image (str): the name of the disk image for the root disk
                boot_disk (str, optional): the name of an existing disk for use as the root disk (instead of a disk image)
                other_disks ([], optional): GCE disk definitions
                metadata (dict, optional): metadata tags to apply to instance
                disk_type (str, optional): type of disk to use for root disk
                root_size (int, optional): root disk size in GB
                tags ([], optional): list of GCE tags to apply to the instance
                zone (str, optional): create in custom zone
                auto_public_address (bool, optional): auto assign a public address (defaults to False)
                private_ip_address (str, optional): primary private IP address
                wait_for_success (int, optional): wait time for the instance to report success (default WAIT_FOR_SUCCESS)
                service_account (str, optional): Service account name to start the instance with (defaults to the default service account)
                scopes ([], optional): List of service scopes for the instance (default DEFAULT_SCOPES)
                subnetwork (str, optional): subnetwork path (projects/project-foo/regions/us-east1/subnetworks/foo)

            Raises: vFXTConfigurationException, vFXTServiceFailure

            Service failures here are uncaught exceptions and should be handled
            by the caller.

            boot_disk_image format is the image name for local images (or the more
            formal global/images/my-private-image).  For images from other projects,
            the format is projects/<project>/global/images/<image name>.  The full
            URL also is accepted.
        '''
        if not self.valid_instancename(name):
            raise vFXTConfigurationException("{} is not a valid instance name".format(name))
        if machine_type not in self.MACHINE_TYPES:
            raise vFXTConfigurationException("{} is not a valid instance type".format(machine_type))
        if self.get_instance(name):
            raise vFXTConfigurationException("{} exists".format(name))

        machine_defs = self.MACHINE_DEFAULTS[machine_type]

        conn       = self.connection()
        network    = self._get_network()
        zone       = options.get('zone') or self.zones[0]
        subnetwork = options.get('subnetwork') or None
        disk_type  = options.get('disk_type') or machine_defs['root_disk_type']
        root_size  = options.get('root_size') or None
        metadata   = options.get('metadata', None)
        disk_type  = _gce_do(conn.diskTypes().get, project=self.project_id, zone=zone, diskType=disk_type)

        boot_image = {}

        try:
            boot_image = _gce_do(conn.images().get, project=self.project_id, image=boot_disk_image)
        except:
            log.debug("Could not find boot_disk_image in our list of images, assuming public/other")
            boot_image['selfLink'] = boot_disk_image

        # gce instance defaults
        # https://cloud.google.com/compute/docs/reference/latest/instances#resource
        body = {}
        body['name'] = name
        body['machineType'] = "{}/{}/zones/{}/machineTypes/{}".format(self.GCE_URL,self.project_id, zone, machine_type)
        body['disks'] = [{
                'autoDelete': True,
                'boot': True,
                'type': 'PERSISTENT',
                'deviceName': '{}-boot'.format(name),
            }
        ]
        if options.get('boot_disk'):
            # fetch the disk
            boot_disk = _gce_do(conn.disks().get, project=self.project_id, zone=zone, disk=options.get('boot_disk'))
            body['disks'][0]['autoDelete'] = False
            body['disks'][0]['source'] = boot_disk['selfLink']
        else:
            body['disks'][0]['initializeParams'] = {
                'diskName': '{}-boot'.format(name),
                'diskType': disk_type['selfLink'],
                'sourceImage': boot_image['selfLink']
            }
            if root_size:
                body['disks'][0]['initializeParams']['diskSizeGb'] = long(root_size)
        if other_disks:
            body['disks'].extend(other_disks)

        body['networkInterfaces'] = [{'network':network['selfLink']}]

        if subnetwork:
            sn = self._get_subnetwork(subnetwork)
            body['networkInterfaces'][0]['subnetwork'] = sn['selfLink']
            body['networkInterfaces'][0]['network'] = sn['network']
        else:
            subnetwork_region = self._zone_to_region(zone)
            subnetworks = [_ for _ in self._get_subnetworks(subnetwork_region)]
            if self.subnetwork_id:
                _subnetworks = [_ for _ in subnetworks if _['name'] == self.subnetwork_id]
                if _subnetworks:
                    body['networkInterfaces'][0]['subnetwork'] = _subnetworks[0]['selfLink']
                else:
                    raise vFXTConfigurationException("Unable to find subnetwork {}".format(self.subnetwork_id))
            elif subnetworks: # no subnetwork specified, but we have them so use one
                log.warning("No subnetwork specified, picking {}".format(subnetworks[0]['selfLink']))
                body['networkInterfaces'][0]['subnetwork'] = subnetworks[0]['selfLink']

        # optional ephemeral address
        if options.get('auto_public_address', False):
            nat_name = '{}-nat'.format(name)
            nat_config = [{'kind':'compute#accessConfig', 'type':'ONE_TO_ONE_NAT', 'name':nat_name}]
            body['networkInterfaces'][0]['accessConfigs'] = nat_config
        if options.get('private_ip_address', False):
            body['networkInterfaces'][0]['networkIP'] = options.get('private_ip_address')

        scopes = options.get('scopes') or self.DEFAULT_SCOPES
        if not isinstance(scopes, list) or not all([_.startswith('http') for _ in scopes]):
            raise vFXTConfigurationException("Invalid scopes: {}".format(scopes))
        body['serviceAccounts'] = [{
            'email': options.get('service_account') or 'default',
            'scopes' : scopes
        }]
        body['canIpForward'] = True
        body['tags'] = {'items':[]}
        body['metadata'] = {'items':[]}

        if 'tags' in options:
            body['tags']['items'].extend(options['tags'])

        if metadata:
            # google wants a list of this dict :-/
            pairs = [{'key':k, 'value': v} for k,v in metadata.iteritems()]
            body['metadata']['items'].extend(pairs)

        log.debug("create_instance request body: {}".format(body))

        try:
            r = _gce_do(conn.instances().insert, project=self.project_id, zone=zone, body=body)
            wait_for_success = options.get('wait_for_success') or self.WAIT_FOR_SUCCESS
            self._wait_for_operation(r, msg='instance to be created', retries=wait_for_success, zone=zone)
            retries = self.CLOUD_API_RETRIES
            while retries > 0:
                n = self.get_instance(name)
                if n:
                    return n
                retries -= 1
                time.sleep(self.POLLTIME)
            raise vFXTServiceFailure("Unable to locate the created instance {}".format(name))
        except Exception as e:
            raise vFXTServiceFailure("Create instance failed: {}".format(e))

    def create_node(self, node_name, cfg, node_opts, instance_options):
        '''Create a cluster node

            This is a frontend for create_instance that handles vFXT node specifics

            Arguments:
                node_name (str): name of the node
                cfg (str): configuration string to pass to the node
                node_opts (dict): node creation options
                instance_options (dict): options passed to create_instance

                node_opts include:
                    data_disk_size: size of data disks (in MB)
                    data_disk_type: disk type of data disk (pd-standard, pd-ssd, local-ssd)
                    data_disk_count: number of data disks
                    data_disk_nvme (bool): use NVME instead of SCSI
                    metadata (dict)
                    machine_type
                    root_image: disk image name
                    disk_type: root disk type

        '''
        conn = self.connection()
        if self.get_instance(node_name):
            raise vFXTNodeExistsException("Node {} exists".format(node_name))

        use_local_ssd = False
        if node_opts['data_disk_type'] == 'local-ssd':
            use_local_ssd = True
            # local-ssd sizes cannot be anything but 375
            node_opts['data_disk_size'] = 375
            if int(node_opts['data_disk_count']) > 8:
                raise vFXTConfigurationException("{} is larger than 8, the maximum for number of local-ssd disks".format(node_opts['data_disk_count']))

        zone = instance_options.get('zone') or self.zones[0]
        data_disk_url = _gce_do(conn.diskTypes().get, project=self.project_id, zone=zone, diskType=node_opts['data_disk_type'])['selfLink']
        data_disk_disks = []
        try:
            node_meta = node_opts.get('metadata', {})
            node_meta['cluster_cfg'] = cfg

            for idx in range(node_opts['data_disk_count']):
                data_disk_name = "{}-data-{}".format(node_name, idx+1)
                data_disk_disk = {}
                # local-ssd can only be created atomically with the instance so
                # we only define it here
                if use_local_ssd:
                    data_disk_disk = {
                        'autoDelete':True,
                        'type':'SCRATCH',
                        'interface': 'NVME' if node_opts.get('data_disk_nvme') else 'SCSI',
                        'deviceName':data_disk_name,
                        'initializeParams': {
                            'diskType':data_disk_url,
                            'diskSizeGb':node_opts['data_disk_size'],
                        },
                    }
                else: # otherwise, create the data disks before the instance
                    body = {'name':data_disk_name, 'sizeGb':long(node_opts['data_disk_size']), 'type':data_disk_url}
                    log.info("Creating data disk {} for {}".format(idx+1, node_name))
                    log.debug("data disk request body: {}".format(body))
                    r    = _gce_do(conn.disks().insert, project=self.project_id, zone=zone, body=body)
                    self._wait_for_operation(r, msg='disk to be created', zone=zone)
                    created_disk = _gce_do(conn.disks().get, project=self.project_id, zone=zone, disk=data_disk_name)
                    data_disk_disk    = {'autoDelete':True,'type':'PERSISTENT','source':created_disk['selfLink'], 'deviceName': data_disk_name}

                data_disk_disks.append(data_disk_disk)

            log.info("Creating node {}".format(node_name))
            n = self.create_instance(machine_type=node_opts['machine_type'],
                            name=node_name,
                            boot_disk_image=node_opts['root_image'],
                            disk_type=node_opts['disk_type'],
                            other_disks=data_disk_disks,
                            metadata=node_meta,
                            **instance_options
            )
            log.info("Created {} ({})".format(n['selfLink'], n['networkInterfaces'][0]['networkIP']))
            return n

        except (vFXTServiceFailure, vFXTConfigurationException) as e:
            log.debug(e)
            n = self.get_instance(node_name)
            if n:
                self.destroy(n)
            elif data_disk_disks:
                for data_disk_disk in data_disk_disks:
                    if data_disk_disk['type'] != 'PERSISTENT': # only created disks
                        continue
                    try:
                        log.debug("Removing data disk {}".format(data_disk_disk['deviceName']))
                        r = _gce_do(conn.disks().delete, project=self.project_id, zone=zone, disk=data_disk_disk['deviceName'])
                        self._wait_for_operation(r, msg='disk to be deleted', zone=zone)
                    except Exception as disk_e:
                        log.error("Failed to remove data disk: {}".format(disk_e))
            raise

    def create_cluster(self, cluster, **options):
        '''Create a vFXT cluster (calls create_node for each node)
            Typically called via vFXT.Cluster.create()

            Arguments:
                cluster (vFXT.cluster.Cluster): cluster object
                size (int, optional): size of cluster (node count)
                root_image (str, optional): root disk image name
                disk_type (str, optional): root disk type
                data_disk_size (int, optional): size of data disk (or machine type default)
                data_disk_count (int, optional): number of data disks (or machine type default)
                data_disk_type (str, optional): type of data disks (or machine type default)
                metadata (dict, optional): metadata for instance
                config_expiration (int, optional): expiration time for cluster join configuration
                wait_for_state (str, optional): red, yellow, green cluster state (defaults to yellow)
                skip_cleanup (bool, optional): do not clean up on failure
                zones ([str], optional): one or more zones
                management_address (str, optional): management address for the cluster
                instance_addresses ([], optional): list of instance addresses to use (passed to create_cluster(private_ip_address))
                address_range_start (str, optional): The first of a custom range of addresses to use for the cluster
                address_range_end (str, optional): The last of a custom range of addresses to use for the cluster
                address_range_netmask (str, optional): cluster address range netmask

                Additional arguments are passed through to create_node()

            Raises: vFXTConfigurationException, vFXTCreateFailure

            root_image format is the image name for local images (or the more
            formal global/images/my-private-image).  For images from other projects,
            the format is projects/<project>/global/images/<image name>.  The full
            URL also is accepted.
        '''
        zones = options.get('zones') or self.zones
        zones = [zones] if isinstance(zones, basestring) else zones
        cycle_zones = cycle(zones)
        # extend our service zones if necessary
        for z in zones:
            if z not in self.zones:
                self.zones.append(z)

        machine_type    = cluster.machine_type
        if machine_type not in self.MACHINE_TYPES:
            raise vFXTConfigurationException("{} is not a valid instance type".format(machine_type))
        zone_machine_types = self._zone_machine_types()
        if not all([_ for _ in zones if _ in zone_machine_types.keys() and machine_type in zone_machine_types[_]]):
            err = "{} is not available in all requested zones: {}".format(machine_type, ', '.join(zones))
            raise vFXTConfigurationException(err)

        machine_defs    = self.MACHINE_DEFAULTS[machine_type]
        cluster_size    = int(options.get('size', machine_defs['node_count']))

        ip_count = cluster_size + (1 if not options.get('management_address') else 0)
        custom_ip_config_reqs = ['address_range_start', 'address_range_end']
        if all([options.get(_) for _ in custom_ip_config_reqs]):
            log.debug("Using overrides for cluster management and address range")
            mask = options.get('address_range_netmask') or '255.255.255.255'
            avail = Cidr.expand_address_range(options.get('address_range_start'), options.get('address_range_end'))
            if len(avail) < ip_count:
                raise vFXTConfigurationException("Not enough addresses provided, require {}".format(ip_count))
        else:
            avail, mask     = self.get_available_addresses(count=ip_count, contiguous=True)

        cluster.mgmt_ip = options.get('management_address') or avail.pop(0)
        cluster_ips     = avail

        # store for cluster config
        cluster.mgmt_netmask     = mask
        cluster.cluster_ip_start = cluster_ips[0]
        cluster.cluster_ip_end   = cluster_ips[-1]

        cfg     = cluster.cluster_config(expiration=options.get('config_expiration', None))
        log.debug("Generated cluster config: {}".format(cfg))

        # gce needs them base64 encoded
        cfg     = ''.join(cfg.encode('base64').split()).strip()

        disk_type   = options.get('disk_type')  or machine_defs['root_disk_type']
        root_image  = options.get('root_image') or self._get_default_image()

        data_disk_size  = options.get('data_disk_size')  or machine_defs['data_disk_size']
        data_disk_count = options.get('data_disk_count') or machine_defs['data_disk_count']
        data_disk_type  = options.get('data_disk_type')  or machine_defs['data_disk_type']
        data_disk_nvme  = options.get('data_disk_nvme', False)

        metadata    = options.pop('metadata', {})

        instance_addresses = options.pop('instance_addresses', [None] * cluster_size)
        if len(instance_addresses) != cluster_size:
            raise vFXTConfigurationException("Not enough instance addresses provided, require {}".format(cluster_size))

        try:
            # create the initial node
            name = '{}-{}'.format(cluster.name, 1)
            opts = {'data_disk_count': data_disk_count, 'data_disk_size': data_disk_size,
                    'data_disk_type': data_disk_type, 'metadata': metadata.copy(),
                    'machine_type': machine_type, 'root_image': root_image,
                    'disk_type': disk_type, 'data_disk_nvme': data_disk_nvme}
            options['zone'] = next(cycle_zones)
            options['private_ip_address'] = instance_addresses.pop()
            n    = self.create_node(name, cfg, node_opts=opts, instance_options=options)
            cluster.nodes.append(ServiceInstance(service=self, instance=n))

            if not options.get('skip_configuration'):
                cluster.first_node_configuration(wait_for_state=options.get('wait_for_state', 'yellow'))
            options.update(opts)
            options['instance_addresses'] = instance_addresses
            self.add_cluster_nodes(cluster, cluster_size - 1, **options)
        except vFXTNodeExistsException as e:
            log.error("Failed to create node: {}".format(e))
            raise
        except Exception as e:
            log.error("Failed to create node: {}".format(e))
            if not options.get('skip_cleanup', False):
                cluster.destroy(quick_destroy=True)
            raise vFXTCreateFailure(e)

    def post_destroy_cluster(self, cluster):
        '''Post cluster destroy cleanup'''
        pass

    def add_cluster_nodes(self, cluster, count, **options):
        '''Add nodes to the cluster (delegates to create_node())

            Arguments:
                cluster (vFXT.cluster.Cluster): cluster object
                count (int): number of nodes to add
                skip_cleanup (bool, optional): do not clean up on failure
                **options: passed to create_node()

            Raises: exceptions from create_node()
        '''
        zones = options.get('zone') or cluster.zones or self.zones
        zones = [zones] if isinstance(zones, basestring) else zones
        # make sure to use unused zones first
        unused_zones = [z for z in self.zones if z not in zones]
        unused_zones.extend([z for z in zones if z not in unused_zones])
        cycle_zones = cycle(unused_zones)

        # extend our service zones if necessary
        for z in zones:
            if z not in self.zones:
                self.zones.append(z)

        instance_addresses = options.pop('instance_addresses', [None] * count)
        if len(instance_addresses) != count:
            raise vFXTConfigurationException("Not enough instance addresses provided, require {}".format(count))

        # look at cluster.nodes[0].instance
        instance       = cluster.nodes[0].instance
        instance_zone  = instance['zone'].split('/')[-1]
        disks          = instance['disks']
        root_disk      = [d for d in disks if 'boot' in d and d['boot']][0]
        non_root_disks = [d for d in disks if 'boot' not in d or not d['boot']]
        data_disk_count= options.get('data_disk_count', len(non_root_disks))
        if data_disk_count == 0:
            raise vFXTConfigurationException("Cannot determine data disk configuration")
        if 'items' in instance['tags'] and not options.get('tags'):
            options['tags'] = instance['tags']['items']
        if 'email' in instance['serviceAccounts'] and 'service_account' not in options:
            options['service_account'] = instance['serviceAccounts']['email']

        metadata       = {opt['key']:opt['value'] for opt in instance['metadata']['items']}
        # overrides
        opts = {'data_disk_count': data_disk_count, 'metadata': metadata, 'machine_type':cluster.machine_type}
        overrides = ['machine_type', 'data_disk_size', 'data_disk_type', 'root_image', 'disk_type']
        for o in overrides:
            if o in options:
                opts[o] = options.pop(o)

        if 'metadata' in options: # even if empty
            opts['metadata'].update(options.pop('metadata') or {})

        conn = None

        # root disk info
        if 'root_image' not in opts or 'disk_type' not in opts:
            conn = self.connection()
            disk_name      = root_disk['source'].split('/')[-1]
            disk_data      = _gce_do(conn.disks().get, project=self.project_id, zone=instance_zone, disk=disk_name)
            disk_type      = disk_data['type'].split('/')[-1]
            root_image     = disk_data['sourceImage'] if 'sourceImage' in disk_data else self._get_default_image()
            if 'root_image' not in opts:
                opts['root_image'] = root_image
            if 'disk_type' not in opts:
                opts['disk_type'] = disk_type

        # data info
        if 'data_disk_size' not in opts or 'data_disk_type' not in opts:
            data_disk_size = None
            data_disk_type = None
            if not conn:
                conn = self.connection()
            if non_root_disks[0]['type'] == 'SCRATCH':
                data_disk_type = _gce_do(conn.diskTypes().get, project=self.project_id, zone=instance_zone, diskType='local-ssd')['selfLink'].split('/')[-1]
                # there is not API to query the size of a non-persistent disk
                data_disk_size = 375
            else:
                disk_name      = non_root_disks[0]['source'].split('/')[-1]
                disk_data      = _gce_do(conn.disks().get, project=self.project_id, zone=instance_zone, disk=disk_name)
                data_disk_size = disk_data['sizeGb']
                data_disk_type = disk_data['type'].split('/')[-1]
            if 'data_disk_size' not in opts:
                opts['data_disk_size'] = data_disk_size
            if 'data_disk_type' not in opts:
                opts['data_disk_type'] = data_disk_type

        # Requires cluster be online
        # XXX assume our node name always ends in the node number
        max_node_num = max([int(i.name().split('-')[-1]) for i in cluster.nodes])

        joincfg = cluster.cluster_config(joining=True, expiration=options.get('config_expiration', None))
        joincfg = ''.join(joincfg.encode('base64').split()).strip()

        nodeq   = Queue.Queue()
        failq   = Queue.Queue()
        threads = []
        def cb(nodenum, inst_opts, nodeq, failq):
            '''callback'''
            try:
                name = '{}-{}'.format(cluster.name, nodenum)
                n    = self.create_node(name, joincfg, node_opts=opts, instance_options=inst_opts)
                nodeq.put(n)
            except Exception as e:
                failq.put(e)

        for node_num in xrange(max_node_num, max_node_num+count):
            next_node_num = node_num + 1
            inst_opts = options.copy()
            inst_opts['zone'] = next(cycle_zones)
            inst_opts['private_ip_address'] = instance_addresses.pop()
            t = threading.Thread(target=cb, args=(next_node_num, inst_opts, nodeq, failq,))
            t.start()
            threads.append(t)

        for t in threads:
            t.join()

        nodes = []
        while True:
            try:
                n = nodeq.get_nowait()
                nodes.append(ServiceInstance(service=self, instance=n))
            except Queue.Empty:
                break

        failed = []
        while True:
            try:
                failed.append(failq.get_nowait())
            except Queue.Empty:
                break
        if failed:
            if not options.get('skip_cleanup', False):
                for n in nodes:
                    n.destroy()
            raise Exception(failed)

        cluster.nodes.extend(nodes)

    def load_cluster_information(self, cluster, **options):
        '''Loads cluster information from the service and cluster itself
        '''
        xmlrpc = cluster.xmlrpc()

        # make sure mgmt_ip is set to the valid address (in case we used
        # a node address to get in)
        cluster.mgmt_ip = xmlrpc.cluster.get()['mgmtIP']['IP']

        node_ips = set([n['primaryClusterIP']['IP']
                        for name in xmlrpc.node.list()
                        for n in [xmlrpc.node.get(name)[name]]
                        if 'primaryClusterIP' in n])

        # lookup nodes that have one of our primary IP addresses.. for now we
        # have to fetch all the instances and search here instead of server side
        # https://code.google.com/p/google-compute-engine/issues/detail?id=533
        nodes = [n for n in self.find_instances(all_regions=False) if self.ip(n) in node_ips]
        if nodes:
            cluster.nodes        = [ServiceInstance(self, instance=n) for n in nodes]
            cluster.zones        = list(set([node['zone'].split('/')[-1] for node in nodes]))
            # XXX assume all instances have the same settings
            n                    = nodes[0]
            cluster.machine_type = n['machineType'].split('/')[-1]
            cluster.project_id   = n['zone'].split('/')[-3]
            cluster.network_id   = n['networkInterfaces'][0]['network'].split('/')[-1]

    # gce specific
    def _get_network(self):
        '''Get the network object'''
        conn        = self.connection()
        return _gce_do(conn.networks().get, project=self.project_id, network=self.network_id)

    def _get_subnetworks(self, region=None):
        '''Get the subnetworks from the network object

            Arguments:
                region (str, optional): return only the subnetworks in the provided region
        '''
        network = self._get_network()
        if 'subnetworks' not in network:
            return []
        subnetworks = []
        conn = self.connection()
        for sn in network['subnetworks']:
            sn_region = sn.split('/')[-3]
            sn_name = sn.split('/')[-1]
            subnetwork = _gce_do(conn.subnetworks().get,
                            project=self.project_id,
                            region=sn_region, subnetwork=sn_name)
            subnetworks.append(subnetwork)
        if region:
            subnetworks = [_ for _ in subnetworks if _['region'].endswith(region)]
        return subnetworks

    def _get_metadata(self, instance, key):
        '''Retrieve the value of a key from the instance metadata

            Arguments:
                instance (obj): backend instance object
                key (str): key to lookup

            Returns: value or None
        '''
        if 'items' not in instance['metadata']:
            return None

        items       = instance['metadata']['items']
        try:
            value = [i['value'] for i in items if i['key'] == key][0]
            log.debug("Fetched metadata {}={}".format(key, value))
            return value
        except:
            log.debug("No such metadata key:{}".format(key))
            return None

    def _delete_metadata(self, instance, key):
        '''Delete a key from the instance metadata and sync with the backend

            Arguments:
                instance (obj): backend instance object
                key (str): key to delete
        '''
        if 'items' not in instance['metadata']:
            return None

        conn        = self.connection()
        metadata    = instance['metadata']
        zone        = instance['zone'].split('/')[-1]
        items       = metadata['items']
        existing = [ idx for idx, i in enumerate(items) if i['key'] == key]
        if existing:
            del metadata['items'][existing[0]]
        response = _gce_do(conn.instances().setMetadata,
                    project=self.project_id,
                    zone=zone,
                    instance=instance['name'],
                    body=metadata)
        self._wait_for_operation(response, msg='metadata to be deleted', zone=zone)

    def _set_metadata(self, instance, key, value):
        '''Set a key from the instance metadata and sync with the backend

            Arguments:
                instance (obj): backend instance object
                key (str): key to set
                value (str): value of key
        '''
        if 'items' not in instance['metadata']:
            instance['metadata']['items'] = []
        conn        = self.connection()
        zone        = instance['zone'].split('/')[-1]
        metadata    = instance['metadata']
        items       = metadata['items']
        existing = [ (idx, i['value']) for idx, i in enumerate(items) if i['key'] == key]
        if existing:
            idx, oldvalue = existing[0]
            metadata['items'][idx]['value'] = value
            log.debug("Updated metadata key:{}={} (from {})".format(key,value, oldvalue))
        else:
            metadata['items'].append(dict(key=key, value=value))
            log.debug("Setting metadata key:{}={}".format(key,value))

        response = _gce_do(conn.instances().setMetadata,
                    project=self.project_id,
                    zone=zone,
                    instance=instance['name'],
                    body=metadata)
        self._wait_for_operation(response, msg='metadata to be set', zone=zone)

    def valid_bucketname(self, name):
        '''Validate the instance name

            Returns: bool
        '''
        if not ServiceBase.valid_bucketname(self, name):
            return False
        if name.startswith('goog'):
            return False

        disallowed = ['google', 'gogle', 'googgle', 'g00gle', 'goog1e']
        if all([s not in name for s in disallowed]):
            return True

        return False

    def valid_instancename(self, name):
        '''Validate the instance name

            Returns: bool
        '''
        if not ServiceBase.valid_instancename(self, name):
            return False
        if len(name) > 63 or len(name) < 1:
            return False
        if self.INSTANCENAME_RE.match(name):
            return True
        return False

    def _zone_machine_types(self):
        '''Get a mapping of zones and their supported machine types
            Returns: dict zone:[types]
        '''
        response = _gce_do(self.connection().machineTypes().aggregatedList, project=self.project_id)
        return {zone_name.split('/')[1]: [mt['name'] for mt in zone_data.get('machineTypes', [])] for zone_name, zone_data in response['items'].items()}

    def _zone_names(self, all_regions=True):
        '''Get a list of zone names
            Arguments:
                all_regions (bool, optional): return zones for all regions, True
            Returns: list
        '''
        if not hasattr(self.local, '_zone_names'):
            conn = self.connection()
            self.local._zone_names = [_ for _ in self.zones]
            regions = _gce_do(conn.regions().list,project=self.project_id)['items']
            all_zones = [zone.split('/')[-1] for region in regions if 'zones' in region for zone in region['zones']]
            self.local._zone_names.extend([_ for _ in all_zones if _ not in self.zones])
        if all_regions:
            return self.local._zone_names
        else:
            region = self._zone_to_region(self.zones[0])
            return [_ for _ in self.local._zone_names if _.startswith(region)]

    def _zone_to_region(self, zone):
        '''Return the name of the region for a given zone

            This is typically just zone[:-2]

            Arguments:
                zone (str): name of the zone
        '''
        conn = self.connection()
        regions = _gce_do(conn.regions().list,project=self.project_id)['items']
        for r in regions:
            zone_url = '{}/{}/zones/{}'.format(self.GCE_URL, self.project_id, zone)
            if zone_url in r['zones']:
                return r['name']
        raise vFXTConfigurationException("Invalid zone: {}".format(zone))

    def _gs_get_object(self, bucket, obj, fh, chunksize=1024*1024):
        '''Fetch an object from a bucket
            Arguments:
                bucket (str): bucket name
                obj (str): object name
                fh: filehandle (any io.IOBase derived filehandle, even StringIO
                chunksize: size of download chunks
        '''
        log.debug("Fetching {} from bucket {}".format(obj, bucket))
        c = self.connection(connection_type='storage')
        req = c.objects().get_media(bucket=bucket, object=obj)
        downloader = googleapiclient.http.MediaIoBaseDownload(fh, req, chunksize)

        done = False
        errors = 0
        while not done:
            try:
                status, done = downloader.next_chunk()
                if status:
                    log.debug("{:>3}% of {} downloaded".format(status.progress() *100, obj))
            except googleapiclient.http.HttpError as e:
                if int(e.resp['status']) < 500:
                    raise vFXTServiceFailure("Failed to fetch object {}: {}".format(obj,e))
                errors += 1
                time.sleep(backoff(errors))
            except Exception as e:
                errors += 1
                time.sleep(backoff(errors))

    def _gs_fetch(self, url, filename):
        '''Retrieve the object from google storage, writing it to the passed in file location
            Arguments:
                url (str): gs:// url
                filename (str): name of destination file (absolute path)

            Returns: Nothing

            Raises:
            googleapiclient.errors.HttpError
        '''
        bkt = None
        obj = None

        log.debug("Fetching {} to {}".format(url, filename))
        try:
            m = self.GSURL_RE.match(url)
            if not m:
                raise Exception('Match failed')
            bkt = m.groups()[0]
            obj = m.groups()[1]
            if not bkt and obj:
                raise Exception('Both bucket and object not parsed')
        except Exception as e:
            log.debug("Failed parsing google storage url: {}".format(e))
            raise vFXTConfigurationException("Invalid google storage URL: {}".format(url))

        sig_file = filename + '.sig'
        sig_obj  = obj + '.sig'

        try:
            # does sig exist
            if os.access(sig_file, os.F_OK) and os.access(filename, os.F_OK):
                tmp = sig_file + '.tmp'
                with open(tmp, 'w') as f:
                    self._gs_get_object(bkt, sig_obj, f)
                sig_cmp = filecmp.cmp(sig_file, tmp)
                os.unlink(tmp)
                if sig_cmp: # we got it
                    log.debug("Signature {} up to date".format(sig_obj))
                    return # bail, nothing to be done
        except googleapiclient.errors.HttpError:
            pass

        # fetch sig for future comparison
        try:
            with open(sig_file, 'w') as f:
                self._gs_get_object(bkt, sig_obj, f)
        except Exception as e:
            log.debug(e)
            try:
                os.unlink(sig_file)
            except Exception as cleanup_e:
                log.debug("Failed to cleanup {}: {}".format(sig_file, cleanup_e))

        # get the actual file
        try:
            with open(filename, 'w') as f:
                self._gs_get_object(bkt, obj, f)
        except Exception as e:
            log.debug(e)
            try:
                os.unlink(filename)
            except Exception as cleanup_e:
                log.debug("Failed to cleanup {}: {}".format(filename, cleanup_e))
            raise

    def _destroy_installation_image(self, image):
        '''Destroy an installation image

            Arguments:
            name (str): name of the cloud image
        '''
        log.info("Destroying image {}".format(image))
        try:
            r = _gce_do(self.connection().images().delete, project=self.project_id, image=image)
            self._wait_for_operation(r, msg='image to be deleted', op_type='globalOperations')
        except Exception as e:
            raise vFXTServiceFailure("Failed to destroy image {}: {}".format(image, e))

    def add_instance_address(self, instance, address, **options):
        '''Add a new route to the instance

            Arguments:
                instance: backend instance
                address (str): IP address
                allow_reassignment (bool, optional): defaults to True
                priority (int, optional): priority (lower value is higher), defaults to 900

        '''
        conn = self.connection()
        try:
            addr = Cidr('{}/32'.format(address)) # validates
            dest = '{}/32'.format(addr.address)

            # check for existing
            dest_filter = 'destRange eq {}'.format(dest)
            resp = _gce_do(conn.routes().list, project=self.project_id, filter=dest_filter)
            if 'items' in resp:
                existing = resp['items']
                network_selflink = self._get_network()['selfLink']
                # if we are the next hop instance for a route in our current network
                if instance['selfLink'] in [_['nextHopInstance'] for _ in existing if _['network'] == network_selflink and 'nextHopInstance' in _]:
                    raise vFXTConfigurationException("Instance already has a route for this address: {}".format(dest))
                if not options.get('allow_reassignment', True):
                    raise vFXTConfigurationException("Route already assigned: {}".format(existing))

                for route in existing:
                    log.debug("Deleting route {}".format(route['name']))
                    resp = _gce_do(conn.routes().delete, project=self.project_id, route=route['name'])
                    self._wait_for_operation(resp, msg='route to be deleted', op_type='globalOperations')

            # add the route
            body = {
                'name': '{}-{}'.format(self.name(instance), addr.address.replace('.','-')),
                'network': self._get_network()['selfLink'],
                'nextHopInstance': instance['selfLink'],
                'destRange': dest,
                'priority': options.get('priority') or 900,
            }
            log.debug('Adding instance address body {}'.format(body))
            resp = _gce_do(conn.routes().insert,project=self.project_id, body=body)
            self._wait_for_operation(resp, msg='route to be created', op_type='globalOperations')
        except vFXTConfigurationException as e:
            raise
        except Exception as e:
            raise vFXTServiceFailure("Failed to create route: {}".format(e))

    def remove_instance_address(self, instance, address):
        '''Remove an instance route address

            Arguments:
                instance: backend instance
                address (str): IP address

            Raises: vFXTServiceFailure
        '''
        conn = self.connection()
        try:
            addr = Cidr('{}/32'.format(address)) # validates
            expr = 'destRange eq {}/32'.format(addr.address)
            routes = _gce_do(conn.routes().list, project=self.project_id,filter=expr)
            if not routes or 'items' not in routes:
                raise vFXTConfigurationException("No route was found for {}".format(addr.address))
            for route in routes['items']:
                if instance['selfLink'] != route['nextHopInstance']:
                    log.warning("Skipping route destined for other host: {} -> {}".format(address, route['nextHopInstance']))
                    continue
                log.debug("Deleting route {}".format(route['name']))
                resp = _gce_do(conn.routes().delete, project=self.project_id, route=route['name'])
                self._wait_for_operation(resp, msg='route to be deleted', op_type='globalOperations')
        except vFXTConfigurationException as e:
            raise
        except Exception as e:
            raise vFXTServiceFailure("Failed to delete route: {}".format(e))

    def instance_in_use_addresses(self, instance, category='all'):
        '''Get the in use addresses for the instance

            Arguments:
                instance: backend instance
                category (str): all, instance, routes

            To obtain the public instance address, use 'public' category.  This
            is not included with 'all'.
        '''
        addresses = set()
        if category in ['all','instance']:
            for interface in instance['networkInterfaces']:
                interface_address = interface.get('networkIP')
                if not interface_address:
                    continue
                addresses.add(interface_address)

        if category in ['all','routes']:
            search = 'nextHopInstance eq .*/{}'.format(instance['name'])
            conn   = self.connection()
            resp   = _gce_do(conn.routes().list, project=self.project_id, filter=search)
            if resp and 'items' in resp:
                network_selflink = self._get_network()['selfLink']
                for route in resp['items']:
                    addr = route['destRange'].split('/')[0]
                    if addr == '0.0.0.0': # gw/default
                        continue
                    # if this route is for a different network, ignore it
                    if route['network'] != network_selflink:
                        continue
                    addresses.add(addr)

        # for special requests
        if category == 'public':
            for interface in instance['networkInterfaces']:
                try:
                    nat_addresses = [_['natIP'] for _ in interface['accessConfigs']]
                    addresses.update(set(nat_addresses))
                except: pass

        return list(addresses)

    def _add_tag(self, instance, tag):
        '''Add a tag to an instance
        '''
        conn     = self.connection()
        zone     = instance['zone'].split('/')[-1]
        instance = self.refresh(instance)
        tags = instance['tags']
        if 'items' not in tags:
            tags['items'] = []

        tags['items'].append(tag)

        response = _gce_do(conn.instances().setTags,
                    project=self.project_id,
                    zone=zone,
                    instance=instance['name'],
                    body=tags)
        self._wait_for_operation(response, msg='tags to be set', zone=zone)

    def _remove_tag(self, instance, tag):
        '''Remove a tag from an instance
        '''
        conn     = self.connection()
        zone     = instance['zone'].split('/')[-1]
        instance = self.refresh(instance)
        tags = instance['tags']
        if 'items' not in tags:
            tags['items'] = []

        if tag not in tags['items']:
            return
        tags['items'] = [_ for _ in tags['items'] if _ != tag]

        response = _gce_do(conn.instances().setTags,
                    project=self.project_id,
                    zone=zone,
                    instance=instance['name'],
                    body=tags)
        self._wait_for_operation(response, msg='tags to be set', zone=zone)

    def _cache_to_disk_config(self, cache_size, machine_type=None, disk_type=None): #pylint: disable=unused-argument
        '''For a given cache size, output the default data disk count and size

            Arguments:
                cache_size (int): vFXT cluster node cache size
                machine_type (str, optional): vFXT cluster node machine type
                disk_type (str, optional): vFXT cluster node disk type

            Returns:
                tuple (disk count, size per disk)
        '''
        if disk_type == 'local-ssd':
            if cache_size > 3000: # 375GB max 8 disks
                raise vFXTConfigurationException("{} is larger than 3000GB, the maximum size for local-ssd disks".format(cache_size))
            count = int(cache_size/375)
            if (cache_size % 375) != 0:
                count += 1
            return (count, 375)
        return tuple([1, cache_size])

    def _get_default_image(self):
        '''Get the default image from the defaults

            This may not be available if we are unable to fetch the defaults.
        '''
        try:
            return self.defaults['machineimages']['current']
        except:
            raise vFXTConfigurationException("You must provide a root disk image.")

    def _disable_disk_auto_delete(self, instance, disk_name):
        '''Disable disk auto delete flag

            Arguments:
                instance: backend instance
                disk_name (str): disk name
        '''
        conn = self.connection()
        instance_zone = instance['zone'].split('/')[-1]
        op = _gce_do(conn.instances().setDiskAutoDelete,
                project=self.project_id,
                zone=instance_zone,
                instance=instance['name'],
                deviceName=disk_name,
                autoDelete=False)
        self._wait_for_operation(op, msg='auto delete attribute to be disabled', zone=instance_zone)

    def _get_subnetwork(self, subnetwork):
        '''Get the labeled subnetwork

            Arguments:
            subnetwork (str): subnetwork identifier

            This must be in one of the following form:
            "foo"
            "projects/project-foo/regions/us-east1/subnetworks/foo"
        '''
        parts = subnetwork.split('/')
        if len(parts) not in [1, 6]:
            raise vFXTConfigurationException("Invalid subnetwork name: {}".format(subnetwork))

        conn = self.connection()
        try:
            if len(parts) == 1:
                subnetwork_region = self._zone_to_region(self.zones[0])
                subnetworks = [_ for _ in self._get_subnetworks(subnetwork_region) if _['name'] == subnetwork]
                if not subnetworks:
                    raise Exception('No such subnetwork')
                return subnetworks[0]
            elif len(parts) == 6:
                project = parts[1]
                region  = parts[3]
                name    = parts[5]
                return _gce_do(conn.subnetworks().get, project=project, region=region, subnetwork=name)
        except Exception as e:
            log.debug("Failed to find subnetwork {}: {}".format(subnetwork, e))
            raise vFXTConfigurationException("Failed to find subnetwork {}: {}".format(subnetwork, e))

    def _who_has_ip(self, address):
        instances = self.find_instances(all_regions=False)
        for i in instances:
            if address in self.instance_in_use_addresses(i):
                return i
        return None

def _gce_do(f, retries=ServiceBase.CLOUD_API_RETRIES, **options):
    '''GCE function call wrapper with variable retries
        Arguments:
            f (function): function to call
            retries (int, optional): number of retries
            **options: options to pass to the function

        Returns: Returns the function call

        Raises: vFXTServiceFailure, vFXTServiceTimeout
    '''
    errors = 0
    while True:
        try:
            return f(**options).execute()
        except googleapiclient.errors.HttpError as e:
            if int(e.resp['status']) < 500:
                raise vFXTServiceFailure(e)
            errors += 1
            time.sleep(backoff(errors))
            if retries == 0:
                raise vFXTServiceTimeout('{} failed, exhausted retries: {}'.format(f.func_name, e))
        except Exception as e:
            log.debug("Unknown GCE retry-able function call failure: {}".format(e))
        retries -= 1
        if retries == 0:
            raise vFXTServiceTimeout('{} failed, exhausted retries: {}'.format(f.func_name, e))
        time.sleep(Service.POLLTIME)


