# -*- coding: utf-8 -*-
import http.client
import ssl
import json
from io import StringIO
import mimetypes
import os
from pathlib import Path

"""
"
" The client interact with Waston Analytic Engine
"
"""


class AnalyticEngineClient():
    
    def __init__(self, host, uid=None, pwd=None, token=None, verbose=True):
        """
        @param::token: authentication token in string
        @param::host: host url in string
        return catalog client instance
        """
        if host == None:
            raise Exception('The host url is required.') 
        else:
            self.host = host
        
#         if instance_display_name == None:
#             raise Exception('Analytics Engine display name is required.') 
#         else:
#             self.instance_display_name = instance_display_name
        
        if token != None:
            self.token = token
        elif uid != None and pwd !=None:
            self.__get_auth_token__(uid,pwd)
        elif os.environ.get('USER_ACCESS_TOKEN', None) != None:
            self.token = os.environ['USER_ACCESS_TOKEN']
        else:
            raise Exception('The uid/pwd and authentication token can not be empty at the same time.')
            
        
        
        # retrieve auth token
#         if token == None:
#             self.__get_auth_token__(uid,pwd)
#         else:
#             self.token = token   
            
#         if self.token != None:
#             self.__get_jobs_auth_token__(self.token, self.instance_display_name)
#         else:
#             raise Exception('Something went wrong, during getting jobs auth token') 
        # debug info
        if verbose:
            print('Initialize Cloud Pak For Data: sucessfully!')
    
    
    
    def get_all_instances(self):
        """
        return all the analytics instance details
        """
        method = '/zen-data/v2/serviceInstance?type=spark'
        response = self.__GET__(method)
        return self.__jsonify__(response)
    
    def get_all_volumes(self):
        """
        return all the analytics instance details
        """
        method = '/zen-data/v2/serviceInstance?type=volumes'
        response = self.__GET__(method)
        return self.__jsonify__(response)

    def get_volume_status(self, volume_id=None, volume_name=None):
        """
        return the status of the volume
        """
        if volume_id == None and volume_name == None :
            raise Exception("Both volume_name and volume_id can't be None, need atleast one.")

        method = '/zen-data/v2/serviceInstance?type=volumes'
        response = self.__GET__(method)
        response = json.loads(response)
        if len(response["requestObj"]) == 0:
            raise Exception("No volume found")
        
        for val in response["requestObj"]:
            if val["ID"] == volume_id or val["ServiceInstanceDisplayName"] == volume_name:
                result = {"status": val["ProvisionStatus"]}
                return result
        
        raise Exception("No volume found")
    
    def get_all_storage_class(self):
        """
        returns all the available storage classes in the cluster.
        """
        method = '/zen-data/v2/storageclasses'
        response = self.__GET__(method)
        result = json.loads(response)
        if "requestObj" in result and len(result["requestObj"]) >0:
            result = [val["metadata"]["name"] for val in result["requestObj"]]
            return self.__jsonify__(json.dumps(result))
        
        return self.__jsonify__(response)
    
    
    def get_instance_id(self, instance_display_name):
        """
        @param string::instance_display_name: display name on the AE instance
        returns instance id for the AE instance 
        """
        method = '/zen-data/v2/serviceInstance?type=spark'
        response = self.__GET__(method)
        response_dict = json.loads(response)
        id = None
        if len(response_dict["requestObj"]) == 0:
            return self.__jsonify__(json.dumps({"id":id}))
        else:
            for val in response_dict["requestObj"]:
                if val["ServiceInstanceDisplayName"] == instance_display_name:
                    id = val["ID"]
                    return self.__jsonify__(json.dumps({"id":id}))
    
        return self.__jsonify__(json.dumps({"id":id}))
    

    def get_instance_details(self, instance_display_name=None, instance_id=None ):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        returns instance id for the AE instance 
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None")
        method = '/zen-data/v2/serviceInstance?type=spark'
        response = self.__GET__(method)
        response_dict = json.loads(response)
        result = {}
        if len(response_dict["requestObj"]) == 0:
            return self.__jsonify__(json.dumps(result))
        else:
            for val in response_dict["requestObj"]:
                if val["ServiceInstanceDisplayName"] == instance_display_name or val["ID"] == instance_id:
                    result = val
                    return self.__jsonify__(json.dumps(result))
        return self.__jsonify__(json.dumps(result))
    
    
    def get_instance_resource_quota(self,instance_display_name=None, instance_id=None):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        @param int::cpu_quota: Max CPU can be used by AE instance
        @param string::memory_quota: Max Memory can be used by AE instance
        returns Spark jobs end point url
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        
        history_server_ui_end_point = self.get_history_server_ui_end_point(instance_display_name, instance_id)
        spark_instance_end_point = history_server_ui_end_point['history_server_ui_endpoint'].rstrip('/historyui')
        spark_instance_end_point = spark_instance_end_point.replace(self.host, "")
        response = self.__GET__(spark_instance_end_point)
        return self.__jsonify__(json.dumps(response))
    
    
    def update_instance_resource_quota(self,instance_display_name=None, instance_id=None,cpu_quota=None,memory_quota=None):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        @param int::cpu_quota: Max CPU can be used by AE instance
        @param string::memory_quota: Max Memory can be used by AE instance
        returns Spark jobs end point url
        """
        
        if instance_display_name == None and instance_id == None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        if cpu_quota == None or memory_quota == None:
            raise Exception("cpu_quota or memory_quota can't be None, need both.")
        
        history_server_ui_end_point = self.get_history_server_ui_end_point(instance_display_name, instance_id)
        spark_instance_end_point = history_server_ui_end_point['history_server_ui_endpoint'].rstrip('/historyui')
        resource_quota_end_point = '/{}/resource_quota'.format(spark_instance_end_point.replace(self.host, ""))
        payload = {
                "cpu_quota": cpu_quota,
                "memory_quota":memory_quota
            }
        
        response = self.__PUT__(resource_quota_end_point,payloads=json.dumps(payload))
        return response

    
    def get_spark_end_point(self, instance_display_name=None, instance_id=None ):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        returns Spark jobs end point url
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        method = '/zen-data/v2/serviceInstance?type=spark'
        response = self.__GET__(method)
        response_dict = json.loads(response)
        result = {"spark_jobs_endpoint": None}
        if len(response_dict["requestObj"]) == 0:
            return self.__jsonify__(json.dumps(result))
        else:
            for val in response_dict["requestObj"]:
                if val["ServiceInstanceDisplayName"] == instance_display_name or val["ID"] == instance_id:
                    if "$HOST" in val["CreateArguments"]["connection-info"]["Spark jobs endpoint"]:
                        result["spark_jobs_endpoint"] = val["CreateArguments"]["connection-info"]["Spark jobs endpoint"].replace("$HOST", self.host)
                    else:
                        result["spark_jobs_endpoint"] = val["CreateArguments"]["connection-info"]["Spark jobs endpoint"]
                    return self.__jsonify__(json.dumps(result))
        return self.__jsonify__(json.dumps(result))
    
    
    def get_history_server_end_point(self, instance_display_name=None, instance_id=None ):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        returns AE instance spark jobs history server endpoint
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        method = '/zen-data/v2/serviceInstance?type=spark'
        response = self.__GET__(method)
        response_dict = json.loads(response)
        result = {"history_server_endpoint": None}
        if len(response_dict["requestObj"]) == 0:
            return self.__jsonify__(json.dumps(result))
        else:
            for val in response_dict["requestObj"]:
                if val["ServiceInstanceDisplayName"] == instance_display_name or val["ID"] == instance_id:
                    
                    if "$HOST" in val["CreateArguments"]["connection-info"]["History server endpoint"]:
                        result["history_server_endpoint"] = val["CreateArguments"]["connection-info"]["History server endpoint"].replace("$HOST", self.host)
                    else:
                        result["history_server_endpoint"] = val["CreateArguments"]["connection-info"]["History server endpoint"]
                    return self.__jsonify__(json.dumps(result))
        return self.__jsonify__(json.dumps(result))
    
    
    def get_history_server_ui_end_point(self, instance_display_name=None, instance_id=None ):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        returns AE instance spark jobs history UI server
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        method = '/zen-data/v2/serviceInstance?type=spark'
        response = self.__GET__(method)
        response_dict = json.loads(response)
        result = {"history_server_endpoint": None}
        if len(response_dict["requestObj"]) == 0:
            return self.__jsonify__(json.dumps(result))
        else:
            for val in response_dict["requestObj"]:
                if val["ServiceInstanceDisplayName"] == instance_display_name or val["ID"] == instance_id:
                    
                    if "$HOST" in val["CreateArguments"]["connection-info"]["Spark jobs endpoint"]:
                        result["history_server_ui_endpoint"] = val["CreateArguments"]["connection-info"]["View history server"].replace("$HOST", self.host)
                    else:
                        result["history_server_ui_endpoint"] = val["CreateArguments"]["connection-info"]["View history server"]
                    return self.__jsonify__(json.dumps(result))
        return self.__jsonify__(json.dumps(result))
    
    def start_history_server(self, instance_display_name=None, instance_id=None ):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        returns AE instance jobs history server
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        
        history_url= self.get_history_server_end_point(instance_display_name, instance_id)
        history_url= history_url["history_server_endpoint"].replace(self.host, "")
        response = self.__POST__(history_url)
        return self.__jsonify__(json.dumps(response))
    
    def stop_history_server(self, instance_display_name=None, instance_id=None ):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        returns AE instance jobs history server
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        
        history_url= self.get_history_server_end_point(instance_display_name, instance_id)
        history_url= history_url["history_server_endpoint"].replace(self.host, "")
        response = self.__DELETE__(history_url)
        return self.__jsonify__(json.dumps(response))
    
    def submit_word_count_job(self, instance_display_name=None, instance_id=None ):
         return self.submit_job(instance_display_name, application_arguments=["/opt/ibm/spark/examples/src/main/resources/people.txt"], application="/opt/ibm/spark/examples/src/main/python/wordcount.py")
    
    def submit_job(self, instance_display_name, instance_id=None, env ={}, volumes=[], size={}, conf={}, application_arguments = [], application_jar=None, main_class=None, application=None, params_json={}   ):
        """
        This method used to submit jobs to AE instance
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        @param dict::env: set env params
                    {
                         "PYSPARK_PYTHON": "<path>",
                         "PYTHONPATH": "<path>"
                     }
        @param list::volumes: volumes details to be mounted to the instance
                    [{
                    "volume_name": "volume anme",
                    "source_path": "",
                    "mount_path": "/mount-path"
                    }]
        @param dict::size: set executors and drivers size
                { 
                  "num_workers": 1, 
                  "worker_size": { 
                      "cpu": 1, 
                      "memory": "4g"
                  }, 
                  "driver_size": { 
                      "cpu": 1, 
                      "memory": "4g" 
                  } 
              }
        @param list::application_arguments: arguments to be passed into the spark application
        @param str::application_jar: path of file to be execcuted by spark engine
        @param str::main_class: main calss module to be used
            example: org.apache.spark.deploy.SparkSubmit
        
        returns instance id for the AE instance 
        @param dict::params_json: all the params can be sent as a json, which will be directly sent in spark submit
        
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None")
            
        spark_jobs_endpoint = self.get_spark_end_point(instance_display_name, instance_id)
        spark_jobs_endpoint= spark_jobs_endpoint["spark_jobs_endpoint"].replace(self.host, "")
        self.job_token = self.__get_jobs_auth_token__(self.token, instance_display_name)
        
        ###
        # comment out by kai, not used varaible
        #type = "spark"
        
        if len(params_json) != 0:
            payload =params_json
        else:
            payload = {
                "engine": {
                    "type": "spark"
                }
            }

            if env != {}:
                payload["engine"]["env"] = env

            if volumes != []:
                payload["engine"]["volumes"] = volumes

            if size != {}:
                payload["engine"]["size"] = size
                
            if conf != {}:
                payload["engine"]["conf"] = conf
                
            if application_arguments != None:
                payload["application_arguments"] = application_arguments
            if application_jar != None:
                payload["application_jar"] = application_jar
            if main_class != None:
                payload["main_class"] = main_class
            if application != None:
                payload["application"] = application
        
        
        
        headers = {
            'Authorization' : 'Bearer {}'.format(self.token),
            'cache-control': 'no-cache',
            'accept': 'application/json',
            'content-type': 'application/json'
        }
        
        
        payload = json.dumps(payload)
        response = self.__POST__(spark_jobs_endpoint,payloads=payload, headers=headers)
        return self.__jsonify__(json.dumps(response))
    
    def upload_and_submit_job(self, instance_display_name, app_volume_name, spark_job_filename, params_json={}, instance_id=None, target_directory=None):
        """
        @param string::instance_display_name: display name on the AE instance
        @param string::volume_name: volume display name
        @param string::source_file: source complete file path
        @param dict::params_json: spark job submit payload
        @param int::instance_id: Instance ID on the AE instance
        @param string::target_directory: path with directory structure, where file to be saved
        returns instance id for the AE instance 
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None")
        
        if app_volume_name == None:
            raise Exception("app volume name display name can't be None")
        
        if spark_job_filename == None:
            raise Exception("spark_job_filename can't be None")
        
        if target_directory == None:
            target_directory = "/my-spark-apps/"
        
        app_volume_def = {
            "volume_name": app_volume_name,
            "source_path": target_directory.lstrip('/').rstrip('/'),
            "mount_path": '/'+target_directory.lstrip('/').rstrip('/')
        }
        
        if "application" not in params_json or "application_jar" not in params_json:
            params_json["application"] = "/{}/{}".format(target_directory.lstrip('/').rstrip('/'),Path(spark_job_filename).name)
        
        if "engine" not in params_json:
            params_json["engine"] = {}
        
        if "volumes" not in params_json["engine"]:
            params_json["engine"]["volumes"] = []
        
        params_json["engine"]["volumes"].append(app_volume_def)
        
        
        self.start_volume(app_volume_name)
        target_file_name = Path(spark_job_filename).name
        response = self.add_file_to_volume(app_volume_name, spark_job_filename, target_file_name, target_directory)
        print(response)
        
        job_response = self.submit_job(instance_display_name, params_json=params_json)
        print(job_response)
    
    
    def get_all_jobs(self, instance_display_name=None, instance_id=None ):
        """
        @param string::instance_display_name: display name on the AE instance
        @param int::instance_id: Instance ID on the AE instance
        returns instance id for the AE instance 
        """
        
        if instance_display_name == None and instance_id ==None:
            raise Exception("Both instance_display_name and instance_id can't be None")
        spark_jobs_endpoint = self.get_spark_end_point(instance_display_name, instance_id)
        spark_jobs_endpoint= spark_jobs_endpoint["spark_jobs_endpoint"].replace(self.host, "")
#         spark_jobs_endpoint = spark_jobs_endpoint.lstrip("/")
        self.job_token = self.__get_jobs_auth_token__(self.token, instance_display_name)
        headers = {
            'jwt-auth-user-payload': self.job_token,
            'cache-control': 'no-cache',
            'accept': 'application/json',
            'content-type': 'application/json'
        }
        response = self.__GET__(spark_jobs_endpoint, headers=headers)
        return self.__jsonify__(json.dumps(response))
        
        
    def create_instance(self, instance_display_name,pre_existing_owner=False,transient_fields= {},service_instance_version = "3.0.1",create_arguments={} ):
        """
        @param string::instance_display_name: display name for the instance
        @param bool::pre_existing_owner: Set pre existing owner
        @param string::service_instance_version: set service instance version
        @param dict::transient_fields: dictionay to set transient fields, default is {}
        @param bool::pre_existing_owner: Set pre existing owner
        @param dict::create_arguments: set arguments for the volumes sample dictionary
            {
                "metadata":{
                   "volumeName":"volume name- must be created before",
                   "storageClass": "",
                   "storageSize": ""
                },
                "serviceInstanceDescription": "Description"
             }
        returns AE instance jobs history server
        """
        
        service_instance_type = "spark"
        
        if instance_display_name == None or len(instance_display_name) == 0:
            raise Exception("Instance display name can't be blank.")
        
        sample_creat_arguments = """
            {
                "metadata":{
                   "volumeName":"volume name- must be created before",
                   "storageClass": "",
                   "storageSize": ""
                },
                "serviceInstanceDescription": "Description"
             }"""
        
        if len(create_arguments) == 0:
            raise Exception("create_arguments dictionay can't be empty. Follow the sample: \n {}".format(sample_creat_arguments))
            
        if "metadata" not in create_arguments:
            raise Exception("create_arguments dictionay must have meta data. Follow the sample: \n {}".format(sample_creat_arguments))
        
        if "resources" not in create_arguments:
            create_arguments["resources"] = {}
        
        ##
        # dheerag, please fix problem here, volume_instance_display_name is not definied
        #
        # if "serviceInstanceDescription" not in create_arguments:
        #    create_arguments["serviceInstanceDescription"] = volume_instance_display_name
        
        payload = {
            "createArguments": create_arguments,
            "preExistingOwner": pre_existing_owner,
            "serviceInstanceDisplayName": instance_display_name,
            "serviceInstanceType": service_instance_type,
            "serviceInstanceVersion": service_instance_version,
            "transientFields": transient_fields
        }
        
        payload = json.dumps(payload)
        
        method = '/zen-data/v2/serviceInstance'
        response = self.__POST__(method, payload)
        return self.__jsonify__(json.dumps(response))
     
        
    
    def create_volume(self, volume_instance_display_name,pre_existing_owner=False,transient_fields= {},service_instance_version = "-",create_arguments={} ):
        """
        @param string::volume_instance_display_name: display name for the volume
        @param bool::pre_existing_owner: Set pre existing owner
        @param string::service_instance_version: set service instance version
        @param dict::transient_fields: dictionay to set transient fields, default is {}
        @param bool::pre_existing_owner: Set pre existing owner
        @param dict::create_arguments: set arguments for the volumes sample dictionary
            {
                "metadata": {
                    "storageClass": "ibmc-file-gold-gid",
                    "storageSize": "20Gi"
                },
                "resources": {},
                "serviceInstanceDescription": "volume 1"
            }
        returns AE instance jobs history server
        """
        
        service_instance_type = "volumes"
        
        if volume_instance_display_name == None or len(volume_instance_display_name) == 0:
            raise Exception("volume instance display name can't be blank.")
        
        sample_creat_arguments = """
            {
                "metadata": {
                    "storageClass": "ibmc-file-gold-gid",
                    "storageSize": "20Gi"
                },
                "resources": {},
                "serviceInstanceDescription": "volume 1"
            }"""
        
        if len(create_arguments) == 0:
            raise Exception("create_arguments dictionay can't be empty. Follow the sample: \n {}".format(sample_creat_arguments))
            
        if "metadata" not in create_arguments:
            raise Exception("create_arguments dictionay must have meta data. Follow the sample: \n {}".format(sample_creat_arguments))
        
        if "resources" not in create_arguments:
            create_arguments["resources"] = {}
        
        if "serviceInstanceDescription" not in create_arguments:
            create_arguments["serviceInstanceDescription"] = volume_instance_display_name
        
        payload = {
            "createArguments": create_arguments,
            "preExistingOwner": pre_existing_owner,
            "serviceInstanceDisplayName": volume_instance_display_name,
            "serviceInstanceType": service_instance_type,
            "serviceInstanceVersion": service_instance_version,
            "transientFields": transient_fields
        }
        
        payload = json.dumps(payload)
        
        method = '/zen-data/v2/serviceInstance'
        response = self.__POST__(method, payload)
        return self.__jsonify__(json.dumps(response))
    
    def get_spark_job_status(self, instance_display_name=None, instance_id=None, job_id=None):
        """
        @param string::instance_display_name: display name for the volume
        @param string::instance_id: Volume unique id
        @param string::job_id: spark job id
        """
        
        if instance_display_name == None and instance_id == None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        if job_id == None:
            raise Exception("job_id can't be None.")
        
        job_end_point = self.get_spark_end_point(instance_display_name, instance_id)['spark_jobs_endpoint']
        method = '{}/{}'.format(job_end_point.replace(self.host, ""), job_id)
        response = self.__GET__(method)
        return self.__jsonify__(json.dumps(response))
    
    def delete_spark_job(self, instance_display_name=None, instance_id=None, job_id=None):
        """
        @param string::instance_display_name: display name for the volume
        @param string::instance_id: Volume unique id
        @param string::job_id: spark job id
        """
        
        if instance_display_name == None and instance_id == None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        if job_id == None:
            raise Exception("job_id can't be None.")
        
        job_end_point = self.get_spark_end_point(instance_display_name, instance_id)['spark_jobs_endpoint']
        method = '{}/{}'.format(job_end_point.replace(self.host, ""), job_id)
        response = self.__DELETE__(method)
        return self.__jsonify__(json.dumps(response))
    
    def delete_all_finished_spark_job(self, instance_display_name=None, instance_id=None):
        """
        @param string::instance_display_name: display name for the volume
        @param string::instance_id: Volume unique id
        """
        
        if instance_display_name == None and instance_id == None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        
        job_list = json.loads(self.get_all_jobs(instance_display_name, instance_id))
        
        for job in job_list:
            if job["job_state"] == "FINISHED":
                self.delete_spark_job(instance_display_name, instance_id, job_id=job["id"])
            
    def delete_all_spark_job(self, instance_display_name=None, instance_id=None):
        """
        @param string::instance_display_name: display name for the volume
        @param string::instance_id: Volume unique id
        """
        
        if instance_display_name == None and instance_id == None:
            raise Exception("Both instance_display_name and instance_id can't be None, need atleast one.")
        
        job_list = json.loads(self.get_all_jobs(instance_display_name, instance_id))
        
        for job in job_list:
            self.delete_spark_job(instance_display_name, instance_id, job_id=job["id"])

    def delete_instance(self, instance_display_name=None, instance_id=None, service_instance_version = "-" ):
        """
        @param string::instance_display_name: display name for the volume
        @param string::instance_id: Volume unique id
        @param string::service_instance_version: set service instance version. Default : "-"  
        """
        
        service_instance_type = "spark"
        
        if instance_display_name == None and instance_id == None:
            raise Exception("Both instance_display_name and instance_id can't be blank")
        
        payload = {
            "serviceInstanceType": service_instance_type,
            "serviceInstanceVersion": service_instance_version,
        }
        
        if instance_id:
            payload["serviceInstanceId"] = instance_id
        if instance_display_name:
            payload["serviceInstanceDisplayName"] = instance_display_name
            
        payload = json.dumps(payload)
        
        method = '/zen-data/v2/serviceInstance'
        response = self.__DELETE__(method, payloads = payload)
        return self.__jsonify__(json.dumps(response))
    
    
    def delete_volume(self, volume_instance_display_name=None, volume_id=None, service_instance_version = "-" ):
        """
        @param string::volume_instance_display_name: display name for the volume
        @param string::volume_id: Volume unique id
        @param string::service_instance_version: set service instance version. Default : "-"  
        """
        
        service_instance_type = "volumes"
        
        if volume_instance_display_name == None and volume_id == None:
            raise Exception("Both volume_instance_display_name and volume_id can't be blank")
        
        payload = {
            "serviceInstanceType": service_instance_type,
            "serviceInstanceVersion": service_instance_version,
        }
        
        if volume_id:
            payload["serviceInstanceId"] = volume_id
        if volume_instance_display_name:
            payload["serviceInstanceDisplayName"] = volume_instance_display_name
            
        payload = json.dumps(payload)
        
        method = '/zen-data/v2/serviceInstance'
        response = self.__DELETE__(method, payloads = payload)
        return self.__jsonify__(json.dumps(response))

    def start_volume(self, volume_name):
        """
        @param string::volume_name: volume display name
        returns response from API
        """
        
        if volume_name == None:
            raise Exception("volume display name cannot be empty.")
        
        method= "/zen-data/v1/volumes/volume_services/{}".format(volume_name)
        payload = json.dumps({})
        response = self.__POST__(method, payloads=payload)
        return self.__jsonify__(json.dumps(response))
    
    def download_logs(self, instance_display_name, volume_name, job_id):
        """
        @param string::volume_name: volume display name
        @param string::source_file: source complete file path
        @param string::target_file_name: name of the file to be saved on colume
        @param string::target_directory: path with directory structure, where file to be saved
        returns response from API
        """
        
        if volume_name == None:
            raise Exception("volume display name cannot be empty.")
        
        if job_id == None:
            raise Exception("Job id can not be empty.")
        
        if instance_display_name == None:
            raise Exception("Instance display name can not be empty.")
            
        spark_jobs_endpoint = self.get_spark_end_point(instance_display_name)
        spark_jobs_endpoint= spark_jobs_endpoint["spark_jobs_endpoint"].replace(self.host, "")
        instance_id = spark_jobs_endpoint.split("/")[-3]
        method = "/zen-volumes/{}/v1/volumes/files/{}%2F{}%2Flogs%2Fspark-driver-{}-stdout".format(volume_name, instance_id, job_id, job_id)
        
        ###
        # comment out by kai, not used varaible
        #
        # conn = http.client.HTTPSConnection("cp4d-cpd-cp4d.pathfinder-royalbankofc-73aebe06726e634c608c4167edcc2aeb-0000.tor01.containers.appdomain.cloud")
        # payload = ''
        
        headers = {
          'Authorization': 'Bearer {}'.format(self.token),
        }
        response = self.__GET__(method, headers=headers)
        return self.__jsonify__(json.dumps(response))
    
    def get_file_from_volume(self, volume_name, source_file , target_file_name, target_directory= None):
        """
        @param string::volume_name: volume display name
        @param string::source_file: source complete file path
        @param string::target_file_name: name of the file to be saved on colume
        @param string::target_directory: path with directory structure, where file to be saved
        returns response from API
        """
        
        if volume_name == None:
            raise Exception("volume display name cannot be empty.")
        
        if source_file == None:
            raise Exception("source_file param cannot be empty.")
        
        if target_file_name == None:
            raise Exception("target_file_name param cannot be empty.")
        
        if target_directory != None:
            target_directory =  target_directory.lstrip("/")#.rstrip("/")
            target_directory =  target_directory.split("/")
            target_directory = "%2F".join(target_directory)
            method = "/zen-volumes/{}/v1/volumes/files/{}{}".format(volume_name, target_directory, target_file_name)
        else:
            method = "/zen-volumes/{}/v1/volumes/files/{}".format(volume_name, target_file_name)
        headers = {
          'Authorization': 'Bearer {}'.format(self.token),
        }
        response = self.__GET__(method, headers=headers)
        return self.__jsonify__(json.dumps(response))
    
    def add_file_to_volume(self, volume_name, source_file , target_file_name, target_directory= None):
        """
        @param string::volume_name: volume display name
        @param string::source_file: source complete file path
        @param string::target_file_name: name of the file to be saved on colume
        @param string::target_directory: path with directory structure, where file to be saved
        returns response from API
        """
        
        if volume_name == None:
            raise Exception("volume display name cannot be empty.")
        
        if source_file == None:
            raise Exception("source_file param cannot be empty.")
        
        if target_file_name == None:
            raise Exception("target_file_name param cannot be empty.")
            
        
        conn = http.client.HTTPSConnection(self.host,context = ssl._create_unverified_context())
        dataList = []
        boundary = 'wL36Yn8afVp8Ag7AmP8qZ0SA4n1v9T'
        dataList.append('--' + boundary)
        dataList.append('Content-Disposition: form-data; name=upFile; filename={0}'.format(target_file_name))

        fileType = mimetypes.guess_type(source_file)[0] or 'application/octet-stream'
        dataList.append('Content-Type: {}'.format(fileType))
        dataList.append('')

        with open(source_file) as f:
          dataList.append(f.read())
        dataList.append('--'+boundary+'--')
        dataList.append('')
        body = '\r\n'.join(dataList)
        payload = body
        headers = {
          'Authorization': 'Bearer {}'.format(self.token),
          'Content-type': 'multipart/form-data; boundary={}'.format(boundary)
        }
        
        if target_directory != None:
            target_directory =  target_directory.lstrip("/").rstrip("/")
            target_directory = target_directory+'/'
            target_directory =  target_directory.split("/")
            target_directory = "%2F".join(target_directory)
            method = "/zen-volumes/{}/v1/volumes/files/{}{}".format(volume_name, target_directory, target_file_name)
        else:
            method = "/zen-volumes/{}/v1/volumes/files/{}".format(volume_name, target_file_name)
        
        conn.request("PUT", method, payload, headers)
        res = conn.getresponse()
        result = res.read()
        
        try:
            result = json.loads(result)
        except:
            result = self.__jsonify__(json.dumps(result))
            
        if "_messageCode_" in result:
            if result["_messageCode_"] == "success":
                result["file_path"] = method#"/zen-volumes/{}/v1/volumes/files/{}/{}".format(volume_name, target_directory, target_file_name)
            else:
                self.__jsonify__(json.dumps(result))
                
        
        return self.__jsonify__(json.dumps(result))
    
    
    
    """
    "
    " authenicate user by username and password and get the authentication token 
    " docu: https://cloud.ibm.com/apidocs/watson-data-api#creating-an-iam-bearer-token
    "
    """
    def __get_auth_token__(self, uid, pwd, verbose=False):
        """
        @param::uid: username
        @param::pwd: password
        return authentication token
        """
        if uid == None or pwd == None:
            raise Exception('the username and password are both required.')
        
        
        conn = http.client.HTTPSConnection(
              self.host,
              context = ssl._create_unverified_context()
        )
        
        headers = {
            'content-type': "application/json",
            'cache-control': "no-cache",
            'password':pwd,
            'username':uid
        }
        method = '/v1/preauth/validateAuth'
        conn.request("GET", method, headers=headers)
        res = conn.getresponse()
        data = res.read().decode("utf-8")
        self.token = self.__jsonify__(data)['accessToken']
        return self.token
    
    def __get_jobs_auth_token__(self, token, display_name, verbose=False):
        """
        @param::token: token
        @param::AE instance display name: display_name
        return jobs authentication token
        """
        if token == None:
            raise Exception('Platform token is required.')
        
        
        conn = http.client.HTTPSConnection(
              self.host,
              context = ssl._create_unverified_context()
        )
        
        headers = {
            'content-type': "application/json",
            'cache-control': "no-cache",
            'Authorization': 'Bearer {}'.format(token)
        }
        payload = json.dumps({"serviceInstanceDisplayname": display_name})
        
        method = "/zen-data/v2/serviceInstance/token"
        conn.request("POST", method, headers=headers, body= payload)
        res = conn.getresponse()
        data = res.read().decode("utf-8")
        self.job_token = self.__jsonify__(data)['AccessToken']
        return self.job_token
            
        
    
    def __GET__(self, method, headers=None):
        """
        @param string:: method: the API method
        @param dict:: header: the http GET request header
        return the response data
        """
        if self.token == None:
            raise Exception('Authentication token is required.')
            
        if method == None:
            raise Exception('The API method is required.')
            
        conn = http.client.HTTPSConnection(
              self.host,
              context = ssl._create_unverified_context()
        )
        
        if headers == None:
            headers = {
                'authorization': 'Bearer %s'%(self.token),
                'cache-control': 'no-cache',
                'accept': 'application/json',
                'content-type': 'application/json'
            }
        
        conn.request("GET", method, headers=headers)
        res = conn.getresponse()
        return res.read().decode("utf-8")
    
    
    def __POST__(self, method, payloads=None, headers=None):
        """
        @param string:: method: the method API
        @param dict:: payloads: the payload of POST request
        @param dict:: headers: the header of POST request
        @return string:: the decoded response content
        """
        
        if self.token == None:
            raise Exception('Authentication token is required.')
            
        if method == None:
            raise Exception('The API method is required.')
            
        conn = http.client.HTTPSConnection(
              self.host,
              context = ssl._create_unverified_context()
        )
        
        if headers == None:
            headers = {
                'authorization': 'Bearer %s'%(self.token),
                'cache-control': "no-cache",
                'accept': 'application/json',
                'content-type': 'application/json'
                }
        conn.request("POST", method, payloads, headers)
        res = conn.getresponse()
        data = res.read().decode("utf-8")
        return data
    
    def __PUT__(self, method, payloads=None, headers=None):
        """
        @param string:: method: the method API
        @param dict:: payloads: the payload of POST request
        @param dict:: headers: the header of POST request
        @return string:: the decoded response content
        """
        
        if self.token == None:
            raise Exception('Authentication token is required.')
            
        if method == None:
            raise Exception('The API method is required.')
            
        conn = http.client.HTTPSConnection(
              self.host,
              context = ssl._create_unverified_context()
        )
        
        if headers == None:
            headers = {
                'authorization': 'Bearer %s'%(self.token),
                'cache-control': "no-cache",
                'accept': 'application/json',
                'content-type': 'application/json'
                }
            
        conn.request("PUT", method, payloads, headers)
        res = conn.getresponse()
        data = res.read().decode("utf-8")
        return data
    
    def __PATCH__(self, method, payloads=None, headers=None):
        """
        @param string:: method: the method API
        @param dict:: payloads: the payload of POST request
        @param dict:: headers: the header of POST request
        @return string:: the decoded response content
        """
        
        if self.token == None:
            raise Exception('Authentication token is required.')
            
        if method == None:
            raise Exception('The API method is required.')
            
        conn = http.client.HTTPSConnection(
              self.host,
              context = ssl._create_unverified_context()
        )
        
        if headers == None:
            headers = {
                'authorization': 'Bearer %s'%(self.token),
                'cache-control': "no-cache",
                'accept': 'application/json',
                'content-type': 'application/json'
                }
            
        conn.request("PATCH", method, payloads, headers)
        res = conn.getresponse()
        data = res.read().decode("utf-8")
        return data
    
    def __DELETE__(self, method, headers=None, payloads=None):
        """
        @param string:: method: the API method
        @param dict:: header: the http GET request header
        return the response data
        """
        if self.token == None:
            raise Exception('Authentication token is required.')
            
        if method == None:
            raise Exception('The API method is required.')
            
        conn = http.client.HTTPSConnection(
              self.host,
              context = ssl._create_unverified_context()
        )
        
        if headers == None:
            headers = {
                'authorization': 'Bearer %s'%(self.token),
                'cache-control': 'no-cache',
                'accept': 'application/json',
                'content-type': 'application/json'
            }
        
        
        conn.request("DELETE", method, payloads, headers=headers)
        res = conn.getresponse()
        return res.read().decode("utf-8")
        
    
    def __jsonify__(self, dumps):
        """
        @param::dumps: json dumps in string
        return json object
        """
        dumps_io = StringIO(dumps)
        return json.load(dumps_io)




#         ┌─┐       ┌─┐
#      ┌──┘ ┴───────┘ ┴──┐
#      │                 │
#      │       ───       │
#      │  ─┬┘       └┬─  │
#      │                 │
#      │       ─┴─       │
#      │                 │
#      └───┐         ┌───┘
#          │         │
#          │         │
#          │         │
#          │         └──────────────┐
#          │                        │
#          │                        ├─┐
#          │                        ┌─┘
#          │                        │
#          └─┐  ┐  ┌───────┬──┐  ┌──┘
#            │ ─┤ ─┤       │ ─┤ ─┤
#            └──┴──┘       └──┴──┘
#                 BLESSING FROM 
#           THE BUG-FREE MIGHTY BEAST
