# DEVELOPER NOTE:
#
#   TSRestV1 is a full implementation of the ThoughtSpot Cloud REST APIs with the goal
#   of making clear what is necessary for each API call.
#
#   It is intentionally written with less abstraction than possible so that each REST
#   API call can be viewed and understood as a 'reference implementation'.
#
#   Yes, we could do it better / more Pythonically.
#
#   We have chosen to make it as simple to understand as possible. There are comments
#   and notes written throughout to help the reader understand more.
#
from collections import OrderedDict
from typing import Optional, Dict, List, Union
import json

import requests


class MetadataTypes:
    """
    Value provided as the 'type' parameter in the
    /metadata/ endpoint calls.

    Enum style class to translate reference-guide names to friendlier
    values used in ThoughtSpot & documentation.
    """
    USER = 'USER'
    GROUP = 'USER_GROUP'
    PINBOARD = 'PINBOARD_ANSWER_BOOK'
    LIVEBOARD = 'PINBOARD_ANSWER_BOOK'
    WORKSHEET = 'LOGICAL_TABLE'
    CONNECTION = 'DATA_SOURCE'
    ANSWER = 'QUESTION_ANSWER_BOOK'
    TABLE = 'LOGICAL_TABLE'
    TAG = 'TAG'
    VIEW = 'LOGICAL_TABLE'
    COLUMN = 'LOGICAL_COLUMN'
    JOIN = 'LOGICAL_RELATIONSHIP'
    DATA_SOURCE = 'DATA_SOURCE'
    SQL_VIEW = 'LOGICAL_TABLE'


class MetadataSubtypes:
    WORKSHEET = 'WORKSHEET'
    TABLE = 'ONE_TO_ONE_LOGICAL'
    VIEW = 'AGGR_WORKSHEET'
    USER_UPLOAD = 'USER_DEFINED'
    PRIVATE_WORKSHEET = 'PRIVATE_WORKSHEET'
    SQL_VIEW = 'SQL_VIEW'


# ENUM combining together the UNIQUE identifiers (using the SubTypes for the LOGICAL_TABLE items)
class TSTypes:
    USER = 'USER'
    GROUP = 'USER_GROUP'
    PINBOARD = 'PINBOARD_ANSWER_BOOK'
    LIVEBOARD = 'PINBOARD_ANSWER_BOOK'
    TAG = 'TAG'
    COLUMN = 'LOGICAL_COLUMN'
    JOIN = 'LOGICAL_RELATIONSHIP'
    DATA_SOURCE = 'DATA_SOURCE'
    CONNECTION = 'DATA_SOURCE'
    ANSWER = 'QUESTION_ANSWER_BOOK'
    TABLE = 'ONE_TO_ONE_LOGICAL'
    WORKSHEET = 'WORKSHEET'
    SQL_VIEW = 'SQL_VIEW'
    VIEW = 'AGGR_WORKSHEET'
    PRIVATE_WORKSHEET = 'PRIVATE_WORKSHEET'
    USER_UPLOAD = 'USER_DEFINED'


class Sorts:
    DEFAULT = 'DEFAULT'
    NAME = 'NAME'
    DISPLAY_NAME = 'DISPLAY_NAME'
    AUTHOR = 'AUTHOR'
    CREATED = 'CREATED'
    MODIFIED = 'MODIFIED'


class Categories:
    ALL = 'ALL'
    MY = 'MY'
    FAVORITE = 'FAVORITE'
    REQUESTED = 'REQUESTED'


class ShareModes:
    """
    Value provided as the 'share_mode' parameter of the
    /security/share endpoint calls.

    Enum style class to translate reference-guide names to friendlier
    values used in ThoughtSpot & documentation.
    """
    READ_ONLY = 'READ_ONLY'
    NO_ACCESS = 'NO_ACCESS'
    MODIFY = 'MODIFY'
    EDIT = 'MODIFY'


class Privileges:
    """
    Value provided as the 'type' parameter of the
    /metadata/ endpoint calls.

    Enum style class to translate reference-guide names to friendlier
    values used in ThoughtSpot & documentation.
    """
    INNATE = 'AUTHORING'
    CAN_ADMINISTER_THOUGHTSPOT = 'ADMINISTRATION'
    CAN_UPLOAD_USER_DATA = 'USERDATAUPLOADING'
    CAN_DOWNLOAD_DATA = 'DATADOWNLOADING'
    CAN_MANAGE_DATA = 'DATAMANAGEMENT'
    CAN_SHARE_WITH_ALL_USERS = 'SHAREWITHALL'
    HAS_SPOTIQ_PRIVILEGE = 'A3ANALYSIS'
    CAN_USE_EXPERIMENTAL_FEATURES = 'EXPERIMENTALFEATUREPRIVILEG'
    CAN_ADMINISTER_AND_BYPASS_RLS = 'BYPASSRLS'
    CAN_INVOKE_CUSTOM_R_ANALYSIS = 'RANALYSIS'
    CANNOT_CREATE_OR_DELETE_PINBOARDS = 'DISABLE_PINBOARD_CREATION'


class PermissionTypes:
    EFFECTIVE = 'EFFECTIVE'
    DEFINED = 'DEFINED'


class GroupVisibility:
    DEFAULT = 'DEFAULT'
    SHARABLE = 'DEFAULT'
    NON_SHARABLE = 'NON_SHARABLE'


# Compatibility
MetadataNames = MetadataTypes
MetadataSorts = Sorts
MetadataCategories = Categories


#
# Method Naming Conventions: The methods are meant to be named after the endpoint naming. A '/' is replaced by an '_'
# such that 'user/list' becomes 'user_list()'.
# For endpoints with multiple HTTP verbs, the endpoint will be followed by "__" and the verb.
# This includes where the endpoint takes a /{guid} argument in the URL
# Thus: the user endpoint has "user__get()", "user__delete()", "user__put()"
#
class TSRestApiV1:
    """
    The main TSRestV1 class implements all of the baseline API methods
    as close to the documentation as possible within Pythonic expectations
    Other than sharing a requests.Session with the appropriate settings, each method
    is written to be relatively self-contained, for those wishing to re-implement in their own code
    """
    def __init__(self, server_url: str):
        # Protect from extra end slash on URL
        if server_url[-1] == '/':
            server_url = server_url[0:-1]

        self.server = server_url

        # REST API uses cookies to maintain the session, so you need to create an open Session
        self.requests_session = requests.Session()

        # X-Requested-By             is necessary for all calls.
        # Accept: application/json   isn't necessary with requests (default: Accept: */*) but might be in other frameworks
        #
        # This sets the header on any subsequent call
        self.api_headers = {'X-Requested-By': 'ThoughtSpot', 'Accept': 'application/json'}
        self.requests_session.headers.update(self.api_headers)

        # TS documentation shows the /tspublic/v1/ portion but it is always preceded by {server}/callosum/v1/
        self.base_url = '{server}/callosum/v1/tspublic/v1/'.format(server=self.server)
        self.non_public_base_url = '{server}/callosum/v1/'.format(server=self.server)
        # V2 REST API for basic implementation
        self.v2_base_url = '{server}/tspublic/rest/v2/'.format(server=self.server)

        # Flag for whether the version implements the export_fqn option of metadata/tml/export
        self.can_export_fqn = True

    #
    # Session management calls
    # - up here vs. in the SESSION section below (because these two are required)
    #
    def session_login(self, username: str, password: str, remember_me: bool = True) -> bool:
        endpoint = 'session/login'
        post_data = {'username': username, 'password': password, 'rememberme': str(remember_me).lower()}

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)

        # HTTP 204 - success, no content
        response.raise_for_status()
        return True

    def session_logout(self) -> bool:
        endpoint = 'session/logout'

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url)

        # HTTP 204 - success, no content
        response.raise_for_status()
        return True

    #
    # Root level API methods found below, divided into logical separations
    #

    #
    # DATA METHODS
    #
    def pinboarddata(
        self,
        pinboard_guid: str,
        vizids: List[str],
        format_type: str='COMPACT',
        batch_size: int=-1,
        page_number: int=-1,
        offset: int=-1
    ) -> Dict:
        endpoint = 'pinboarddata'

        url_params = {
            'id': pinboard_guid,
            'vizid': json.dumps(vizids),
            'batchsize': str(batch_size),
            'pagenumber': str(page_number),
            'offset': str(offset),
            'formattype': format_type
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, params=url_params)
        response.raise_for_status()

        return response.json()

    def searchdata(
        self,
        query_string: str,
        data_source_guid: str,
        format_type: str='COMPACT',
        batch_size: int=-1,
        page_number: int=-1,
        offset: int=-1
    ) -> Dict:
        endpoint = 'searchdata'

        url_params = {
            'query_string': query_string,
            'data_source_guid': data_source_guid,
            'batchsize': str(batch_size),
            'pagenumber': str(page_number),
            'offset': str(offset),
            'formattype': format_type
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    #
    # ADMIN Methods
    #

    #
    # CONNECTION methods
    #
    def connection_types(self) -> List:
        endpoint = 'connection/types'
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def connection_list(self, category: str = 'ALL',
                        sort: str = 'DEFAULT', sort_ascending: bool = True,
                        filter: Optional[str] = None, tagname: Optional[List[str]] = None,
                        batchsize: int = -1, offset: int = -1) -> Dict:
        endpoint = 'connection/list'

        url_params = {

            'category': category,
            'sort': sort.upper(),
            'sortascending': str(sort_ascending).lower(),
            'offset': offset

        }
        if filter is not None:
            url_params['pattern'] = filter
        if tagname is not None:
            url_params['tagname'] = json.dumps(tagname)
        if batchsize is not None:
            url_params['batchsize'] = batchsize

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def connection_create(self, connection_name: str, connection_type: str, metadata_json: str, description: str = "",
                          create_without_tables=True, use_internal_endpoint=False):
        endpoint = 'connection/create'

        post_data = {
            'name': connection_name,
            'description': description,
            'type': connection_type,
            'metadata': metadata_json,
        }
        if use_internal_endpoint is True:
            url = self.non_public_base_url + endpoint
        else:
            url = self.base_url + endpoint
            # Not available on 7.1.1 and before releases
            post_data['createEmpty'] = str(create_without_tables).lower()
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    def connection_update(self, connection_guid: str, connection_name: str, connection_type: str, metadata_json: str,
                          description: str = "", create_without_tables=True, use_internal_endpoint=False):
        endpoint = 'connection/update'

        post_data = {
            'id': connection_guid,
            'name': connection_name,
            'description': description,
            'type': connection_type,
            'metadata': metadata_json,

        }
        if use_internal_endpoint is True:
            url = self.non_public_base_url + endpoint
        else:
            url = self.base_url + endpoint
            # Not available on 7.1.1 and before releases
            post_data['createEmpty'] = str(create_without_tables).lower()
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    # Helper method for pulling the connection_configuration from metadata_details when type is DATA_SOURCE (connection)
    @staticmethod
    def get_connection_config_from_metadata_details(metadata_details_response):
        return metadata_details_response['storables'][0]['dataSourceContent']['configuration']

    # Helper method for pulling the connection_configuration from metadata_details when type is DATA_SOURCE (connection)
    @staticmethod
    def get_connection_name_from_metadata_details(metadata_details_response):
        return metadata_details_response['storables'][0]['header']['name']

    # Helper method for pulling the connection_configuration from metadata_details when type is DATA_SOURCE (connection)
    @staticmethod
    def get_connection_type_from_metadata_details(metadata_details_response):
        return metadata_details_response['storables'][0]['type']
    #
    # DATABASE methods - only applicable to Software using Falcon
    #

    #
    # DEPENDENCY methods
    #

    def dependency_listdependents(self, object_type: str, guids: List[str], batchsize: int = -1):
        endpoint = 'dependency/listdependents'

        post_data = {
            'type': object_type,
            'id': json.dumps(guids),
            'batchsize': str(batchsize),

        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    def dependency_listincomplete(self):
        endpoint = 'dependency/listincomplete'
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def dependency_logicalcolumn(self, logical_column_guids: List[str]):
        endpoint = 'dependency/logicalcolumn'

        url_params = {
            'id': json.dumps(logical_column_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def dependency_logicaltable(self, logical_table_guids: List[str]):
        endpoint = 'dependency/logicaltable'

        url_params = {
            'id': json.dumps(logical_table_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def dependency_logicalrelationship(self, logical_relationship_guids: List[str]):
        endpoint = 'dependency/logicalrelationship'

        url_params = {
            'id': json.dumps(logical_relationship_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def dependency_physicalcolumn(self, physical_column_guids: List[str]):
        endpoint = 'dependency/physicalcolumn'

        url_params = {
            'id': json.dumps(physical_column_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def dependency_physicaltable(self, physical_table_guids: List[str]):
        endpoint = 'dependency/physicaltable'

        url_params = {
            'id': json.dumps(physical_table_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def dependency_pinboard(self, pinboard_guids: List[str]):
        endpoint = 'dependency/pinboard'

        url_params = {
            'ids': json.dumps(pinboard_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    #
    # EXPORT METHODS
    #
    def export_pinboard_pdf(
        self,
        pinboard_id: str,
        one_visualization_per_page: bool=False,
        landscape_or_portrait: str='LANDSCAPE',
        cover_page: bool=True,
        logo: bool=True,
        page_numbers: bool=False,
        filter_page: bool=True,
        truncate_tables: bool=False,
        footer_text: str=None,
        visualization_ids: List[str]=None
    ) -> bytes:
        endpoint = 'export/pinboard/pdf'

        # NOTE: there is a 'transient_pinboard_content' option but it would only make sense within the browser
        # NOTE: it's unclear how to use visualization_ids, so not implemented yet

        layout_type = 'PINBOARD'

        if one_visualization_per_page is True:
            layout_type = 'VISUALIZATION'

        url_params = {
            'id': pinboard_id,
            'layout_type': layout_type,
            'orientation': landscape_or_portrait.upper(),
            'truncate_tables': str(truncate_tables).lower(),
            'include_cover_page': str(cover_page).lower(),
            'include_logo': str(logo).lower(),
            'include_page_number': str(page_numbers).lower(),
            'include_filter_page': str(filter_page).lower(),
        }

        if footer_text is not None:
            url_params['footer_text'] = footer_text

        url = self.base_url + endpoint

        # Override the Header for this specific call
        #   requires    Accept: application/octet-stream    (return the content as binary bytes)
        #
        response = self.requests_session.post(url=url, params=url_params, headers={'Accept': 'application/octet-stream'})

        # Return value is in Bytes format, so other methods can do what they want with it
        return response.content

    #
    # GROUP METHODS
    #

    # Requires multipart/form-data
    def group_removeprivilege(self, privilege: str, group_names: List[str]) -> Dict:
        endpoint = 'group/removeprivilege'

        files = {
            'privilege': privilege,
            'groupNames': json.dumps(group_names)
        }

        url = self.base_url + endpoint
        # Requires multipart/form-data
        response = self.requests_session.post(url=url, files=files)
        response.raise_for_status()
        return response.json()

    def group_get(self, group_guid: Optional[str] = None, name: Optional[str] = None) -> Union[Dict, List]:
        endpoint = 'group'
        url_params = {}
        if group_guid is not None:
            url_params['groupid'] = group_guid
        if name is not None:
            url_params['name'] = name

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def group_post(self, group_name: str, display_name: str, privileges: Optional[List[str]],
                   group_type: str = 'LOCAL_GROUP',
                   tenant_id: Optional[str] = None, visibility: str = 'DEFAULT'):
        endpoint = 'group'

        post_data = {
            'name': group_name,
            'display_name': display_name,
            'grouptype': group_type,
            'visibility': visibility
        }

        if privileges is not None:
            post_data['privileges'] = json.dumps(privileges)
        if tenant_id is not None:
            post_data['tenantid'] = tenant_id

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    def group_delete(self, group_guid: str):
        endpoint = 'group/{}'.format(group_guid)

        url = self.base_url + endpoint
        response = self.requests_session.delete(url=url)
        response.raise_for_status()
        return True

    def group_put(self, group_guid: str, content):
        endpoint = 'group/{}'.format(group_guid)

        post_data = {
        }
        if content is not None:
            post_data['content'] = content

        url = self.base_url + endpoint
        response = self.requests_session.put(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    # Add a User to a Group
    def group_user_post(self, group_guid: str, user_guid: str):
        endpoint = 'group/{}/user/{}'.format(group_guid, user_guid)

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url)
        response.raise_for_status()
        return response.json()

    # Remove user from a group
    def group_user_delete(self, group_guid: str, user_guid: str):
        endpoint = 'group/{}/user/{}'.format(group_guid, user_guid)

        url = self.base_url + endpoint
        response = self.requests_session.delete(url=url)
        response.raise_for_status()
        return True

    def group_users_get(self, group_guid: str):
        endpoint = 'group/{}/users'.format(group_guid)

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def group_users_post(self, group_guid: str, user_guids: List[str]):
        endpoint = 'group/{}/users'.format(group_guid)
        url_params = {
            'userids': json.dumps(user_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def group_users_delete(self, group_guid: str, user_guids: List[str]):
        endpoint = 'group/{}/users'.format(group_guid)
        url_params = {
            'userids': json.dumps(user_guids)
        }

        url = self.base_url + endpoint
        response = self.requests_session.delete(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # Requires multipart/form-data
    def group_addprivilege(self, privilege: str, group_names: str) -> Dict:
        endpoint = 'group/addprivilege'

        files = {
            'privilege': privilege,
            'groupNames': json.dumps(group_names)
        }

        url = self.base_url + endpoint
        # Requires multipart/form-data
        response = self.requests_session.post(url=url, files=files)
        response.raise_for_status()
        return response.json()

    # Starting August cloud release, this changed from session/group to group endpoint where it should be
    def group_listuser(self, group_guid: str) -> Dict:
        endpoint = 'group/listuser/{guid}'.format(guid=group_guid)
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    #
    # MATERIALIZATION Methods
    #

    def materialization_refreshview(self, guid: str) -> Dict:
        endpoint = 'materialization/refreshview/{guid}'.format(guid=guid)
        url = self.base_url + endpoint
        response = self.requests_session.post(url=url)
        response.raise_for_status()
        return response.json()

    #
    # METADATA Methods
    #

    def metadata_details(
        self,
        object_type: str,
        object_guids: List[str],
        show_hidden: bool=False,
        drop_question_details: bool=False,
        version: int=-1
    ) -> Dict:
        endpoint = 'metadata/details'

        url_params = {
            'type': object_type,
            'id': json.dumps(object_guids),
            'showhidden': str(show_hidden).lower(),
            'dropquestiondetails': str(drop_question_details).lower(),
            'version': str(version)
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # Helper method for pulling the connection_configuration from metadata_details when type is DATA_SOURCE (connection)
    @staticmethod
    def get_sheets_from_metadata_details(metadata_details_response):
        return metadata_details_response['storables'][0]['reportContent']['sheets']

    # Tag Methods

    # Format for assign tags is that the object_guids List can take any object type, but then you must
    # have a second List for object_type with an entry for each of the corresponding object_guids in the list
    # So really it's like a [{guid: , type: }, {guid:, type: }] structure but split into two separate JSON lists
    def metadata_assigntag(self, object_guids: List[str], object_type: List[str], tag_guids: Optional[List[str]] = None, tag_names: Optional[List[str]] = None) -> bool:
        endpoint = 'metadata/assigntag'
        if tag_guids is None and tag_names is None:
            raise Exception("Either one of tag_guids or tag_names are mandatory.")

        post_data = {
            'id': json.dumps(object_guids),
            'type': json.dumps(object_type)
        }
        if tag_guids is not None:
            post_data['tagid'] = json.dumps(tag_guids)
        if tag_names is not None:
            post_data['tagname'] = json.dumps(tag_names)

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        # Returns a 204 when it works right
        return True

    def metadata_listobjectheaders(self, object_type: str, subtypes: Optional[List[str]] = None,
                                   sort: str = 'DEFAULT', sort_ascending: bool = True,
                                   filter: Optional[str] = None, fetchids: Optional[List[str]] = None,
                                   skipids: Optional[List[str]] = None, tagname: Optional[List[str]] = None,
                                   category: Optional[str] = None, batchsize: int = -1, offset: int = -1,
                                   auto_created: Optional[bool] = None) -> Dict:
        endpoint = 'metadata/listobjectheaders'

        # Tables, Worksheets and Views all have the same object_type, with a sub-type to vary
        # This code allows sending the Sub-Type into Object_type and still run correctly
        if object_type in [MetadataSubtypes.TABLE, MetadataSubtypes.VIEW, MetadataSubtypes.WORKSHEET,
                           MetadataSubtypes.SQL_VIEW]:
            subtypes = [object_type]
            object_type = MetadataTypes.TABLE

        url_params = {
            'type': object_type,
            'sort': sort.upper(),
            'sortascending': str(sort_ascending).lower(),
            'offset': offset
        }
        if subtypes is not None:
            url_params['subtypes'] = json.dumps(subtypes)
        if filter is not None:
            url_params['pattern'] = filter
        if fetchids is not None:
            url_params['fetchids'] = json.dumps(fetchids)
        if skipids is not None:
            url_params['skipids'] = json.dumps(skipids)
        if tagname is not None:
            url_params['tagname'] = json.dumps(tagname)
        if batchsize is not None:
            url_params['batchsize'] = batchsize
        if category is not None:
            url_params['category'] = category
        if auto_created is not None:
            url_params['auto_created'] = str(auto_created).lower()

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def metadata_listvizheaders(self, guid: str) -> Dict:
        endpoint = 'metadata/listvizheaders'
        url_params = {'id': guid}
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # metadata/listas   used to return the set of objects a user or group can access
    def metadata_listas(self, user_or_group_guid: str, user_or_group: str, minimum_access_level: str = 'READ_ONLY',
                        filter: Optional[str] = None) -> Dict:
        endpoint = 'metadata/listas'

        url_params = {
            'type': user_or_group,
            'principalid': user_or_group_guid,
            'minimumaccesslevel': minimum_access_level,
        }

        if filter is not None:
            url_params['pattern'] = filter

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # /metadata/list gives the information available on the listing pages for each object type
    # can be used in the browser to generate menu systems / selector boxes for content scoped to the logged in user
    def metadata_list(self, object_type: str, subtypes: Optional[List[str]] = None,
                      owner_types: Optional[List[str]] = None, category: Optional[str] = None,
                      sort: str = 'DEFAULT', sort_ascending: bool = True, filter: Optional[str] = None,
                      fetchids: Optional[List[str]] = None, skipids: Optional[List[str]] = None,
                      tagname: Optional[List[str]] = None, batchsize: int = -1, offset: int =-1,
                      auto_created: Optional[bool] = None, show_hidden: Optional[bool] = False,
                      author_guid: Optional[str] = None):
        endpoint = 'metadata/list'

        # Tables, Worksheets and Views all have the same object_type, with a sub-type to vary
        # This code allows sending the Sub-Type into Object_type and still run correctly
        if object_type in [MetadataSubtypes.TABLE, MetadataSubtypes.VIEW, MetadataSubtypes.WORKSHEET,
                           MetadataSubtypes.SQL_VIEW]:
            subtypes = [object_type]
            object_type = MetadataTypes.TABLE

        url_params = {
            'type': object_type,
            'sort': sort.upper(),
            'sortascending': str(sort_ascending).lower(),
            'offset': offset
        }
        if subtypes is not None:
            url_params['subtypes'] = json.dumps(subtypes)
        if owner_types is not None:
            url_params['ownertypes'] = json.dumps(owner_types)
        if category is not None:
            url_params['category'] = category
        if filter is not None:
            url_params['pattern'] = filter
        if fetchids is not None:
            url_params['fetchids'] = json.dumps(fetchids)
        if skipids is not None:
            url_params['skipids'] = json.dumps(skipids)
        if tagname is not None:
            url_params['tagname'] = json.dumps(tagname)
        if batchsize is not None:
            url_params['batchsize'] = batchsize
        if auto_created is not None:
            url_params['auto_created'] = str(auto_created).lower()
        if show_hidden is not None:
            url_params['showhidden'] = str(show_hidden).lower()
        if author_guid is not None:
            url_params['author_guid'] = author_guid

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # Helper method to find a GUID from a name
    def metadata_list_find_guid(self, object_type: str, name: str):
        objects = self.metadata_list(object_type=object_type, filter=name)
        # Filter is case-insensitive and the equivalent of a wild-card, so need to look for exact match
        # on the response
        for o in objects['headers']:
            if o['name'] == name:
                return o['id']
        raise LookupError()

    # Favorite Methods
    def metadata_markasfavoritefor(self, user_guid: str, object_guids: List[str], object_type: str) -> Dict:
        endpoint = 'metadata/markunmarkfavoritefor'

        post_data = {
            'type': object_type,
            'ids': json.dumps(object_guids),
            'userid': user_guid
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    def metadata_unmarkasfavoritefor(self, user_guid: str, object_guids: List[str]) -> bool:
        endpoint = 'metadata/markunmarkfavoritefor'

        post_data = {
            'ids': json.dumps(object_guids),
            'userid': user_guid
        }

        url = self.base_url + endpoint
        response = self.requests_session.delete(url=url, data=post_data)
        response.raise_for_status()
        return True

    #
    # TML Methods (METADATA/TML)
    # TML import and export are distinguished by using POST with an {'Accept': 'text/plain'} header on the POST
    #

    # Some errors come through as part of a HTTP 200 response, just listed in the JSON
    @staticmethod
    def raise_tml_errors(response: requests.Response) -> Dict:
        if len(response.content) == 0:
            raise Exception('No response returned at all with status code {}'.format(response.status_code))
        else:
            j = response.json()
            # JSON error response checking

            # It is possible in a multiple file upload that some validated and others have errors

            if 'object' in j:
                for k in j['object']:
                    if 'info' in k:
                        # Older versions wrapped the errors in 'info'
                        if k['info']['status']['status_code'] == 'ERROR':
                            # print(k['info']['status']['error_message'])
                            raise SyntaxError(j['object'])
                    # Recent versions return as 'response'
                    elif 'response' in k:
                        if k['response']['status']['status_code'] == 'ERROR':
                            # print(k['info']['status']['error_message'])
                            raise SyntaxError(j['object'])
                # If no errors are raised, just return the regular response
                return response.json()
            else:
                return response.json()

    def metadata_tml_export(self, guid: str, export_associated=False, export_fqn=True) -> OrderedDict:
        # Always returns a Python Dict, converted from a request to the API to receive in JSON
        endpoint = 'metadata/tml/export'

        post_data = {
            'export_ids': json.dumps([guid]),
            'formattype': 'JSON',
            'export_associated': str(export_associated).lower()
        }
        # Added in version 8.9, can be set to skip for older releases
        if self.can_export_fqn is True:
            post_data['export_fqn'] = str(export_fqn).lower()

        url = self.base_url + endpoint
        # TML import is distinguished by having an {'Accept': 'text/plain'} header on the POST
        response = self.requests_session.post(url=url, data=post_data, headers={'Accept': 'text/plain'})
        response.raise_for_status()
        # Extra parsing of some 'error responses' that come through in JSON response on HTTP 200
        self.raise_tml_errors(response=response)

        # TML API returns a JSON response, with the TML document
        # object_pairs_hook forces an OrderedDict
        tml_json_response = response.json(object_pairs_hook=OrderedDict)
        objs = tml_json_response['object']

        if len(objs) == 1 and export_associated is False:
            # The TML is there in full under the 'edoc' section of the API JSON response
            tml_str = objs[0]['edoc']
            tml_obj = json.loads(tml_str)
        else:
            if export_associated is True:
                tml_obj = tml_json_response
            else:
                raise Exception()
        return tml_obj

    # Method to retrieve the all of the associate objects and retrieve original object and a dict of name:guid mapping
    def metadata_tml_export_with_associations_map(self, guid: str, export_fqn=True) -> (OrderedDict, Dict):
        # Always returns a Python Dict, converted from a request to the API to receive in JSON
        endpoint = 'metadata/tml/export'

        post_data = {
            'export_ids': json.dumps([guid]),
            'formattype': 'JSON',
            'export_associated': 'true'
        }
        # Added in version 8.9, can be set to skip for older releases
        if self.can_export_fqn is True:
            post_data['export_fqn'] = str(export_fqn).lower()

        url = self.base_url + endpoint
        # TML import is distinguished by having an {'Accept': 'text/plain'} header on the POST
        response = self.requests_session.post(url=url, data=post_data, headers={'Accept': 'text/plain'})
        response.raise_for_status()
        # Extra parsing of some 'error responses' that come through in JSON response on HTTP 200
        self.raise_tml_errors(response=response)

        # TML API returns a JSON response, with the TML document
        # object_pairs_hook forces an OrderedDict
        tml_json_response = response.json(object_pairs_hook=OrderedDict)
        objs = tml_json_response['object']

        # The first object will be the requested object
        tml_str = objs[0]['edoc']
        tml_obj = json.loads(tml_str)

        name_guid_map = {}

        for obj in objs:
            name_guid_map[obj['info']['name']] = obj['info']['id']

        return tml_obj, name_guid_map

    def metadata_tml_export_string(self, guid: str, formattype: str = 'YAML',
                                   export_associated=False, export_fqn=True) -> str:
        # Intended for a direct pull with no conversion
        endpoint = 'metadata/tml/export'
        # allow JSON or YAML in any casing
        formattype = formattype.upper()
        post_data = {
            'export_ids': json.dumps([guid]),
            'formattype': formattype,
            'export_associated': str(export_associated).lower()
        }
        # Added in version 8.9, can be set to skip for older releases
        if self.can_export_fqn is True:
            post_data['export_fqn'] = str(export_fqn).lower()
        url = self.base_url + endpoint

        # TML import is distinguished by having an {'Accept': 'text/plain'} header on the POST
        response = self.requests_session.post(url=url, data=post_data, headers={'Accept': 'text/plain'})
        response.raise_for_status()
        # Extra parsing of some 'error responses' that come through in JSON response on HTTP 200
        self.raise_tml_errors(response=response)

        # TML API returns a JSON response, with the TML document
        tml_json_response = response.json()
        objs = tml_json_response['object']

        if len(objs) == 1:
            # The TML is there in full under the 'edoc' section of the API JSON response
            response_str = objs[0]['edoc']
            return response_str

        # This would only happen if you did 'export_associated': 'true' or got no response (would probably
        # throw some sort of HTTP exception
        else:
            raise Exception()

    def metadata_tml_export_string_with_associations_map(self, guid: str, formattype: str = 'YAML',
                                                         export_fqn=True) -> (str, Dict):
        # Intended for a direct pull with no conversion
        endpoint = 'metadata/tml/export'
        # allow JSON or YAML in any casing
        formattype = formattype.upper()
        post_data = {
            'export_ids': json.dumps([guid]),
            'formattype': formattype,
            'export_associated': 'true'
        }
        # Added in version 8.9, can be set to skip for older releases
        if self.can_export_fqn is True:
            post_data['export_fqn'] = str(export_fqn).lower()
        url = self.base_url + endpoint

        # TML import is distinguished by having an {'Accept': 'text/plain'} header on the POST
        response = self.requests_session.post(url=url, data=post_data, headers={'Accept': 'text/plain'})
        response.raise_for_status()
        # Extra parsing of some 'error responses' that come through in JSON response on HTTP 200
        self.raise_tml_errors(response=response)

        # TML API returns a JSON response, with the TML document
        tml_json_response = response.json()
        objs = tml_json_response['object']

        # The TML is there in full under the 'edoc' section of the API JSON response
        response_str = objs[0]['edoc']

        name_guid_map = {}

        for obj in objs:
            name_guid_map[obj['info']['name']] = obj['info']['id']

        return response_str, name_guid_map

    # TML import is distinguished by having an {'Accept': 'text/plain'} header on the POST
    def metadata_tml_import(
        self,
        tml: Union[Dict, List[Dict]],
        create_new_on_server: bool = False,
        validate_only: bool = False,
        formattype: str = 'JSON',
        enable_block_tml_metadata_sync: Optional[bool] = None
    ) -> Dict:
        endpoint = 'metadata/tml/import'
        # allow JSON or YAML in any casing
        formattype = formattype.upper()

        # Adjust for single Dict
        if not isinstance(tml, list):
            tml_list = [tml]
        else:
            tml_list = tml

        if formattype == 'JSON':
            json_encoded_tml = json.dumps(tml_list)
        elif formattype == 'YAML':
            json_encoded_tml = json.dumps(tml_list)
        # Assume it's just a Python object which will dump to JSON matching the TML format
        else:
            json_encoded_tml = json.dumps(tml_list)

        import_policy = 'ALL_OR_NONE'

        if validate_only is True:
            import_policy = 'VALIDATE_ONLY'

        post_data = {
            'import_objects': json_encoded_tml,
            'import_policy': import_policy,
            'force_create': str(create_new_on_server).lower()
        }
        if enable_block_tml_metadata_sync is not None:
            post_data['enable_block_tml_metadata_sync'] = str(enable_block_tml_metadata_sync).lower()

        url = self.base_url + endpoint

        # TML import is distinguished by having an {'Accept': 'text/plain'} header on the POST
        response = self.requests_session.post(url=url, data=post_data, headers={'Accept': 'text/plain'})
        response.raise_for_status()
        # Extra parsing of some 'error responses' that come through in JSON response on HTTP 200
        self.raise_tml_errors(response=response)
        return response.json()

    # Parse the TML response from import to get the GUIDs
    def guids_from_imported_tml(self, tml_import_response) -> List[str]:
        # first level if a key called 'object', then array
        guids = []
        for a in tml_import_response['object']:
            guids.append(a['response']['header']['id_guid'])
        return guids

    #
    # PARTNER methods
    #

    def partner_snowflake_user(self, body: Dict) -> Dict:
        endpoint = 'partner/snowflake/user'
        post_data = {'body': json.dumps(body)}
        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    #
    # SECURITY methods
    #
    #   Content in ThoughtSpot belongs to its author/owner
    #   It can be shared to other Groups or Users
    #

    # There is a particular JSON object structure for giving sharing permissions
    # This method gives you a blank permissions Dict for that purpose
    @staticmethod
    def get_sharing_permissions_dict() -> Dict:
        sharing_dict = {'permissions': {}}
        return sharing_dict

    # This method takes in an existing permissions Dict and adds a new entry to it
    # It returns back the permissions Dict but there is never a copy, it acts upon the Dict passed in
    @staticmethod
    def add_permission_to_dict(permissions_dict: dict, guid: str, share_mode: str) -> Dict:
        for l1 in permissions_dict:
            permissions_dict[l1][guid] = {'shareMode': share_mode}
        return permissions_dict

    # Helper to create share permissions
    def create_share_permissions(self, read_only_users_or_groups_guids: Optional[List[str]] = (),
                                 edit_access_users_or_groups_guids: Optional[List[str]] = (),
                                 remove_access_users_or_groups_groups: Optional[List[str]] = ()) -> Dict:
        permissions_dict = self.get_sharing_permissions_dict()
        for a in read_only_users_or_groups_guids:
            self.add_permission_to_dict(permissions_dict=permissions_dict, guid=a, share_mode=ShareModes.READ_ONLY)
        for a in edit_access_users_or_groups_guids:
            self.add_permission_to_dict(permissions_dict=permissions_dict, guid=a, share_mode=ShareModes.EDIT)
        for a in remove_access_users_or_groups_groups:
            self.add_permission_to_dict(permissions_dict=permissions_dict, guid=a, share_mode=ShareModes.NO_ACCESS)
        return permissions_dict

    # Share any object type
    # Requires a Permissions Dict, which can be generated and modified with the two static methods above
    def security_share(
        self,
        shared_object_type: str,
        shared_object_guids: List[str],
        permissions: Dict,
        notify_users: Optional[bool] = False,
        message: Optional[str] = None,
        email_shares: List[str] = None,
        use_custom_embed_urls: bool = False
    ) -> bool:
        if email_shares is None:
            email_shares = []

        endpoint = 'security/share'
        if shared_object_type in [MetadataSubtypes.TABLE, MetadataSubtypes.VIEW, MetadataSubtypes.WORKSHEET,
                                  MetadataSubtypes.SQL_VIEW]:
            shared_object_type = MetadataTypes.TABLE
        post_data = {
            'type': shared_object_type,
            'id': json.dumps(shared_object_guids),
            'permission': json.dumps(permissions),
            'notify': str(notify_users).lower(),
            'emailshares': json.dumps(email_shares),
            'useCustomEmbedUrls': str(use_custom_embed_urls).lower()
        }

        if message is not None:
            post_data['message'] = message

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return True

    # Shares just a single viz within a Pinboard, without more complex sharing permissions of security/share
    def security_shareviz(
        self,
        shared_object_type: str,
        pinboard_guid: str,
        viz_guid: str,
        principal_ids: List[str],
        notify_users: Optional[bool]=False,
        message: Optional[str]=None,
        email_shares: List[str]=None,
        use_custom_embed_urls: bool=False
    ) -> bool:
        if email_shares is None:
            email_shares = []

        endpoint = 'security/shareviz'

        post_data = {
            'type': shared_object_type,
            'pinboardId': pinboard_guid,
            'principalids': json.dumps(principal_ids),
            'vizid': viz_guid,
            'notify': str(notify_users).lower(),
            'emailshares': json.dumps(email_shares),
            'useCustomEmbedUrls': str(use_custom_embed_urls).lower()
        }

        if message is not None:
            post_data['message'] = message

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return True

    def security_metadata_permissions(self, object_type: str, object_guids: List[str], dependent_share: bool = False,
                                      permission_type: str = 'EFFECTIVE'):
        endpoint = 'security/metadata/permissions'

        url_params = {
            'type': object_type,
            'id': json.dumps(object_guids),
            'dependentshare': str(dependent_share).lower(),
            'permissiontype': permission_type
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def security_metadata_permissions_by_id(self, object_type: str, object_guid: str, dependent_share: bool = False,
                                      permission_type: str = 'EFFECTIVE'):
        endpoint = 'security/metadata/{}/permissions'.format(object_guid)

        url_params = {
            'type': object_type,
            'dependentshare': str(dependent_share).lower(),
            'permissiontype': permission_type
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # ids_by_type is JSON in format { "{object_type_1} : ["{guid_1}, "{guid_2}"], "{object_type_2}" : ["{guid_3}"...] }
    def security_effectivepermissionbulk(self, ids_by_type: Dict, dependent_share: bool = False,):
        endpoint = 'security/effectivepermissionbulk'

        post_data = {
            'idsbytype': json.dumps(ids_by_type),
            'dependentshare': str(dependent_share).lower()
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    #
    # SESSION Methods
    #

    # Home Pinboard Methods
    def session_homepinboard_post(self, pinboard_guid: str, user_guid: str):
        endpoint = 'session/homepinboard'

        post_data = {
            'id': pinboard_guid,
            'userid': user_guid
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

    def session_homepinboard_get(self) -> Dict:
        endpoint = 'session/homepinboard'
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def session_homepinboard_delete(self) -> bool:
        endpoint = 'session/homepinboard'
        url = self.base_url + endpoint
        response = self.requests_session.delete(url=url)
        response.raise_for_status()
        return True

    def session_info(self) -> Dict:
        endpoint = 'session/info'
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def session_orgs_get(self, batchsize: int = -1, offset: int = -1) -> Dict:
        endpoint = 'session/orgs'

        url_params = {
            'batchsize': batchsize,
            'offset': offset
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def session_orgs_put(self, org_id: int):
        endpoint = 'session/orgs'

        post_data = {
            'orgid': org_id
        }

        url = self.base_url + endpoint
        response = self.requests_session.put(url=url, data=post_data)
        response.raise_for_status()

    def session_orgs_users(self, user_guid: str, org_scope: str = 'ALL'):
        endpoint = 'session/orgs/users/{}'.format(user_guid)

        url_params = {
            'orgScope': org_scope
        }

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # NOTE:
    #
    #   The below shows an implementation of session/auth/token but it should only be
    #   used from Authenticator Server with Secret Key retrieved in a secure manner only
    #   in memory
    #
    def session_auth_token(self, secret_key: str, username: str, access_level: str = 'FULL',
                           object_guid: Optional[str] = None, org_id: Optional[int] = None,
                           groups: Optional[List[str]] = None, auto_create_user: Optional[bool] = None):
        endpoint = 'session/auth/token'

        post_params = {
             'secret_key': secret_key,
             'username': username,
             'access_level': access_level,

        }
        if object_guid is not None:
            post_params['id'] = object_guid

        if org_id is not None:
            post_params['orgid'] = org_id

        if auto_create_user is not None:
            post_params['autocreate'] = str(auto_create_user).lower()

        if groups is not None:
            post_params['groups'] = json.dumps(groups)

        url = self.base_url + endpoint

        response = self.requests_session.post(url=url, data=post_params, headers={"Accept": "text/plain"})
        response.raise_for_status()
        return response.content.decode()

    # session/login/token is typically only used within the browser and handled by the Visual Embed SDK,
    # provided here for testing
    def session_login_token_post(self, username: str, auth_token: str, redirect_url: str):
        endpoint = 'session/login/token'

        post_params = {
             'username': username,
             'auth_token': auth_token,
             'redirect_url': redirect_url,  # need to url encode

        }

        url = self.base_url + endpoint

        response = self.requests_session.post(url=url, data=post_params, headers={'Accept': 'text/plain'})
        response.raise_for_status()
        return response
    #
    # USER Methods
    #

    def user_get(self, user_id: Optional[str] = None, name: Optional[str] = None) -> Union[Dict, List]:
        endpoint = 'user/'
        url_params = {}
        if user_id is not None:
            url_params['userid'] = user_id
        if name is not None:
            url_params['name'] = name

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def user_post(self, username: str, password: str, display_name: str, properties: Optional[Dict] = None,
                  groups: Optional[List[str]] = None, user_type: str = 'LOCAL_USER',
                  tenant_id: Optional[str] = None, visibility: str = 'DEFAULT'):
        endpoint = 'user'

        post_data = {
            'name': username,
            'password': password,
            'displayname': display_name,
            'usertype': user_type,
            'visibility': visibility
        }
        if properties is not None:
            post_data['properties'] = properties
        if groups is not None:
            post_data['groups'] = json.dumps(groups)
        if tenant_id is not None:
            post_data['tenantid'] = tenant_id

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    def user_delete(self, user_guid: str):
        endpoint = 'user/{}'.format(user_guid)

        url = self.base_url + endpoint
        response = self.requests_session.delete(url=url)
        response.raise_for_status()
        return True

    def user_put(self, user_guid: str, content, password: Optional[str]):
        endpoint = 'user/{}'.format(user_guid)

        post_data = {
            'userid': user_guid
        }
        if content is not None:
            post_data['content'] = content
        if password is not None:
            post_data['password'] = password

        url = self.base_url + endpoint
        response = self.requests_session.put(url=url, data=post_data)
        response.raise_for_status()

    def user_updatepassword(self, username: str, current_password: str, new_password: str):
        endpoint = 'user/updatepassword'

        post_data = {
            'name': username,
            'currentpassword': current_password,
            'newpassword': new_password
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

    # This exists but since it requires auth_token, would only be useful within a browser session
    # def user_resetpassword(self, user_guid: str, auth_token: str, new_password: str):
    #    endpoint = 'user/resetpassword'
    #
    #    post_data = {
    #        'userid': user_guid,
    #        'auth_token': auth_token,
    #        'password': new_password
    #    }
    #
    #    url = self.base_url + endpoint
    #    response = self.session.post(url=url, data=post_data)
    #    response.raise_for_status()

    # Implementation of the user/sync endpoint, which is fairly complex and runs a risk
    # with the remove_deleted option set to true
    #
    # Uses a multi-part POST, with the type of the principals parameter set to application/json
    def user_sync(
        self,
        principals_file: str,
        password: str,
        apply_changes: bool = False,
        remove_deleted: bool = False
    ) -> Dict:
        endpoint = 'user/sync'

        # You must set the type of principals to 'application/json' or 'text/json'
        files = {
            'principals': ('principals.json', principals_file, 'application/json'),
            'applyChanges': str(apply_changes).lower(),
            'removeDelete': str(remove_deleted).lower(),
            'password': password
        }

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=None, files=files)
        response.raise_for_status()
        return response.json()

    def user_transfer_ownership(self, current_owner_username: str, new_owner_username: str,
                                object_guids: Optional[List[str]] = None) -> bool:
        endpoint = 'user/transfer/ownership'

        url_params = {
            'fromUserName': current_owner_username,
            'toUserName': new_owner_username
        }

        if object_guids is not None:
            url_params['objectid'] = json.dumps(object_guids)

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, params=url_params)
        response.raise_for_status()
        return True

    # NOTE: preferences and preferencesProto are a big ?
    def user_updatepreference(self, user_guid: str, username: str, preferences: Dict,
                              preferences_proto: Optional[str] = None):
        endpoint = 'user/updatepreference'

        post_data = {
            'userid': user_guid,
            'username': username,
            'preferences': json.dumps(preferences),
        }
        if preferences_proto is not None:
            post_data['preferencesProto'] = preferences_proto

        url = self.base_url + endpoint
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

    def build_user_preferences(self, preferred_locale: Optional[str] = None, notify_on_share: Optional[bool] = None,
                               analyst_onboarding_complete: Optional[bool] = None, show_walk_me: Optional[bool] = None):
        preferences = {}
        if preferred_locale is not None:
            preferences['preferredLocale'] = preferred_locale
        if notify_on_share is not None:
            preferences['notifyOnShare'] = notify_on_share
        if analyst_onboarding_complete is not None:
            preferences['analystOnboardingComplete'] = analyst_onboarding_complete
        if show_walk_me is not None:
            preferences['showWalkMe'] = show_walk_me
        return preferences

    # Retrieves all USER and USER_GROUP objects
    def user_list(self) -> Dict:
        endpoint = 'user/list'
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def user_email(self, user_guid: str, user_email: str):
        endpoint = 'user/email'

        post_data = {
            'userid': user_guid,
            'emailid': user_email
        }

        url = self.base_url + endpoint
        response = self.requests_session.put(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    def user_groups_get(self, user_guid: str):
        endpoint = 'user/{}/groups'.format(user_guid)
        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    # Replaces all group membership?
    def user_groups_post(self, user_guid: str, group_guids: List[str]):
        endpoint = 'user/{}/groups'.format(user_guid)

        url = self.base_url + endpoint
        url_params = {
            'groupids': json.dumps(group_guids),
        }

        response = self.requests_session.post(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    # Adds to existing group membership?
    def user_groups_put(self, user_guid: str, group_guids: List[str]):
        endpoint = 'user/{}/groups'.format(user_guid)

        url = self.base_url + endpoint
        url_params = {
            'groupids': json.dumps(group_guids),
        }

        response = self.requests_session.put(url=url, params=url_params)
        response.raise_for_status()

    # Adds to existing group membership?
    def user_groups_delete(self, user_guid: str, group_guids: List[str]):
        endpoint = 'user/{}/groups'.format(user_guid)

        url = self.base_url + endpoint
        url_params = {
            'groupids': json.dumps(group_guids),
        }

        response = self.requests_session.delete(url=url, params=url_params)
        response.raise_for_status()

    def user_session_invalidate(self, usernames: Optional[List[str]] = None, user_guids: Optional[List[str]] = None):
        if usernames is None and user_guids is None:
            raise Exception()
        endpoint = 'user/session/invalidate'

        url = self.base_url + endpoint
        post_data = {}
        if usernames is not None:
            post_data['username'] = json.dumps(usernames)
        if user_guids is not None:
            post_data['userid'] = json.dumps(user_guids)
        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

    #
    # Log Streaming methods
    #
    def logs_topics(self, topic: str = 'security_logs', from_epoch: Optional[str] = None,
                    to_epoch: Optional[str] = None):
        endpoint = 'logs/topics/{}'.format(topic)

        url = self.base_url + endpoint
        url_params = {}
        if from_epoch is not None:
            url_params['fromEpoch'] = from_epoch
        if to_epoch is not None:
            url_params['toEpoch'] = to_epoch

        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    #
    # ADMIN methods, many concerning Custom Actions
    #
    def admin_configinfo(self):
        endpoint = 'admin/configinfo'

        url = self.base_url + endpoint

        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def admin_configinfo_overrides(self):
        endpoint = 'admin/configinfo/overrides'

        url = self.base_url + endpoint

        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def admin_configinfo_update(self, config_changes: Dict):
        endpoint = 'admin/configinfo/update'

        url = self.base_url + endpoint
        post_data = {
            'configchanges': json.dumps(config_changes)
        }

        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

    def admin_embed_actions(self, tags: Optional[List[str]] = None):
        endpoint = 'admin/embed/actions'
        url_params = {}
        if tags is not None:
            url_params['tags'] = json.dumps(tags)

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def admin_embed_action(self, action_guid: str):
        endpoint = 'admin/embed/actions/{}'.format(action_guid)

        url = self.base_url + endpoint
        response = self.requests_session.get(url=url)
        response.raise_for_status()
        return response.json()

    def admin_embed_action_post(self, embed_action_definition: Dict):
        endpoint = 'admin/embed/actions'

        url = self.base_url + endpoint
        post_data = {
            'embedaction': json.dumps(embed_action_definition)
        }

        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

        return response.json()

    def admin_embed_action_put(self, action_guid: str, embed_action_definition: Dict):
        endpoint = 'admin/embed/actions/{}'.format(action_guid)

        url = self.base_url + endpoint
        post_data = {
            'embedaction': json.dumps(embed_action_definition)
        }

        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

        return response.json()

    def admin_embed_action_delete(self, action_guid: str):
        endpoint = 'admin/embed/actions/{}'.format(action_guid)

        url = self.base_url + endpoint

        response = self.requests_session.delete(url=url)
        response.raise_for_status()

        return response.json()

    def admin_embed_action_associations_post(self, action_guid: str, action_association: Dict):
        endpoint = 'admin/embed/actions/{}/associations'.format(action_guid)

        url = self.base_url + endpoint
        post_data = {
            'actionassociation': json.dumps(action_association)
        }

        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()

        return response.json()

    def admin_embed_action_associations_get(self, action_guid: str):
        endpoint = 'admin/embed/actions/{}/associations'.format(action_guid)

        url = self.base_url + endpoint

        response = self.requests_session.get(url=url)
        response.raise_for_status()

        return response.json()

    def admin_embed_action_associations_delete(self, action_guid: str, action_association: Dict):
        endpoint = 'admin/embed/actions/{}/associations'.format(action_guid)

        url = self.base_url + endpoint
        post_data = {
            'actionassociation': json.dumps(action_association)
        }

        response = self.requests_session.delete(url=url, data=post_data)
        response.raise_for_status()

        return response.json()
    #
    # Non-public endpoints
    # No guarantees for these undocumented endpoints to stay consistent
    #

    def metadata_delete(self, object_type: str, guids=List[str], included_disabled=False):
        endpoint = 'metadata/delete'

        url = self.non_public_base_url + endpoint
        post_data = {
            'type': object_type,
            'id': json.dumps(guids),
            'includeddisabled': str(included_disabled).lower()
        }

        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        # Returns a 204 when complete

    # This is the metadata call that is used to display tables / worksheets etc. for a connection in the UI
    def connection_detail(self, connection_guid: str, sort: str = 'MODIFIED', sort_ascending: bool = True,
                          filter: Optional[str] = None, tagname: Optional[List[str]] = None, show_hidden: bool = False):
        endpoint = 'connection/detail/{}'.format(connection_guid)

        url = self.non_public_base_url + endpoint
        url_params = {
                     'sort': sort,
                     'sortascending': str(sort_ascending).lower(),
                     'showhidden': str(show_hidden).lower()
                     }
        if filter is not None:
            url_params['pattern'] = filter
        if tagname is not None:
            url_params['tagname'] = json.dumps(tagname)

        response = self.requests_session.get(url=url, params=url_params)
        response.raise_for_status()
        return response.json()

    def connection_fetch_connection(self, connection_guid: str, include_columns=False,
                                    authentication_type='SERVICE_ACCOUNT', config_json_string: Optional[str] = None,
                                    use_internal_endpoint=False):
        endpoint = 'connection/fetchConnection'

        if use_internal_endpoint is True:
            url = self.non_public_base_url + endpoint
            if config_json_string is None:
                raise Exception('The config_json_string (a JSON object converted to string using json.dumps() ) is required')
        else:
            url = self.base_url + endpoint

        # Example of a config_json, which may vary per connection
        #
        # config_json_string_example = '''
        #       {  "password": "",
        #          "role": "SE_ROLE",
        #          "warehouse": "SE_DEMO_WH",
        #          "accountName": "thoughtspot_partner",
        #          "user": "se_demo"
        #       }
        # '''
        #
        post_data = {'id': connection_guid,
                     'includeColumns': str(include_columns).lower(),
                     'authentication_type': authentication_type
                     }

        if config_json_string is not None:
            post_data['config'] = config_json_string

        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    def connection_fetch_live_columns(self, connection_guid, database_name: str,
                                      schema_name: str, table_name: str,
                                      authentication_type='SERVICE_ACCOUNT', config_json_string: Optional[str] = None,
                                      use_internal_endpoint=False):
        endpoint = 'connection/fetchLiveColumns'

        if use_internal_endpoint is True:
            url = self.non_public_base_url + endpoint
            if config_json_string is None:
                raise Exception('The config_json_string (a JSON object converted to string using json.dumps() ) is required')
        else:
            url = self.base_url + endpoint
        tables = [{"databaseName": database_name,
                  "schemaName": schema_name,
                  "tableName": table_name
                  }]
        # Example of a config_json, which may vary per connection
        #
        # config_json_string_example = '''
        #       {  "password": "",
        #          "role": "SE_ROLE",
        #          "warehouse": "SE_DEMO_WH",
        #          "accountName": "thoughtspot_partner",
        #          "user": "se_demo"
        #       }
        # '''
        #
        post_data = {'connection_id': connection_guid,
                     'tables': json.dumps(tables),
                     'authentication_type': authentication_type
                     }
        if config_json_string is not None:
            post_data['config'] = config_json_string

        response = self.requests_session.post(url=url, data=post_data)
        response.raise_for_status()
        return response.json()

    #
    # connection processing to generate create / update input
    #

    @staticmethod
    def get_databases_from_connection(external_databases_from_fetch_connection):
        dbs = []
        for db in external_databases_from_fetch_connection:
            dbs.append(db['name'])
        return dbs

    #
    #
    # tables_to_add_map format =  { 'database_name' : { 'schema_name' : ['table_name_1', 'table_name_2']}
    # tables_to_add_map format to bring in all tables = { 'database_name' : { 'schema_name' : [] }
    #

    # Output of this would bring in all tables of all schemas if input into tables_to_add_map
    # Useful to quickly build the tables_to_add_map object, just grab the pieces you want
    @staticmethod
    def get_databases_and_schemas_from_connection(external_databases_from_fetch_connection, schema_names_to_skip=[]):
        dbs = {}
        for db in external_databases_from_fetch_connection:
            dbs[db['name']] = {}
            for schema in db["schemas"]:
                if schema['name'] not in schema_names_to_skip:
                    dbs[db['name']][schema['name']] = []

        return dbs

    @staticmethod
    def get_selected_tables_from_connection(external_databases_from_fetch_connection, tables_to_add_map=None):
        selected_external_dbs = []
        for d in external_databases_from_fetch_connection:
            # Pull any database if it is part of the tables_to_add_map
            if tables_to_add_map is not None:
                if d['name'] in tables_to_add_map:
                    selected_external_dbs.append(d)
                    # No need to go any further if we know we'll need the database, so continue loop to next one
                    continue
            # If the database has any already selected / imported tables, we need to bring them as well
            # so that the table objects don't get deleted. Update must include all previously selected tables
            for s in d['schemas']:
                for t in s['tables']:
                    if t['selected'] is True:
                        selected_external_dbs.append(d)
                        break
        return selected_external_dbs

    # You only need to specify the columns when changing them, the 'selected' : true will maintain a table without changes
    # Use the selected_columns function above to get the external_databases object with all of the necessary databases
    def add_new_tables_to_connection(self, selected_external_databases, tables_to_add_map, connection_guid: str,
                                     config_json: str):
        external_databases = selected_external_databases
        # The external_databases object should have all database, schema, and table info for anything we'll bring in

        for db in external_databases:
            # if database already exists, we have the full structure and just need to add columns
            if db["name"] in tables_to_add_map.keys():
                for schema in db['schemas']:
                    # Don't bother if schema doesn't have any new tables to add
                    if schema['name'] in tables_to_add_map[db["name"]].keys():
                        for table in schema['tables']:
                            # Allow importing all tables from schema with an empty array
                            if (table['name'] in tables_to_add_map[db["name"]][schema["name"]]) or len(tables_to_add_map[db["name"]][schema["name"]]) == 0:
                                # Mark the table as selected
                                table['selected'] = True
                                table['linked'] = True

                                # Get the columns to add using connection_fetch_live_columns
                                table_columns = self.connection_fetch_live_columns(
                                    connection_guid=connection_guid,
                                    config_json_string=json.dumps(config_json),
                                    database_name=db["name"], schema_name=schema["name"],
                                    table_name=table["name"])

                                for t in table_columns:

                                    columns_list = []
                                    for c in table_columns[t]:
                                        c['selected'] = True  # Select every column
                                        c['isImported'] = False
                                        c['tableName'] = table["name"]
                                        c['schemaName'] = schema["name"]
                                        c['dbName'] = db["name"]
                                        columns_list.append(c)

                                    table['columns'] = columns_list

        final_response = {"configuration": config_json,
                          "externalDatabases": external_databases
                          }
        return final_response
