import glob
import json
import os
import re

from bs4 import BeautifulSoup
from distutils import dir_util

from .. import _common
from .. import _login

from seeq.sdk import *
from seeq.sdk.rest import ApiException


class Workbook:
    def __init__(self, definition=None):
        """
        :type definition: str, dict
        """
        if _common.is_guid(definition):
            self._pull(definition)
        elif isinstance(definition, dict):
            self.definition = definition
        else:
            self.definition = dict()

    def _pull(self, workbook_id):
        workbooks_api = WorkbooksApi(_login.client)
        workbook_output = workbooks_api.get_workbook(id=workbook_id)  # type: WorkbookOutputV1

        self.definition = Workbook._definition_from_struct(workbook_output)
        self.definition['Data'] = json.loads(workbook_output.data)
        self.definition['workbookState'] = json.loads(self.definition['workbookState'])

        self.worksheets = list()
        worksheet_output_list = workbooks_api.get_worksheets(workbook_id=workbook_id)  # type: WorksheetOutputListV1
        for worksheet_output in worksheet_output_list.worksheets:  # type: WorksheetOutputV1
            self.worksheets.append(Worksheet(self, worksheet_output))

        self._scrape_item_inventory()
        self._scrape_datasource_inventory()

    def push(self, prefix=None, owner_id=None, folder_id=None):
        workbook_item = Workbook.find_item_id(prefix, self.id)

        if owner_id is None:
            owner_id = Workbook.find_user(self.definition['Owner'])

        if owner_id is None:
            users_api = UsersApi(_login.client)
            user_output = users_api.get_me()  # type: UserOutputV1
            owner_id = user_output.id

        workbooks_api = WorkbooksApi(_login.client)
        items_api = ItemsApi(_login.client)
        props = list()
        if not workbook_item:
            workbook_input = WorkbookInputV1()
            workbook_input.name = self.definition['Name']
            workbook_input.description = self.definition['Description']
            workbook_input.folder_id = folder_id
            workbook_input.owner_id = owner_id
            workbook_input.data = json.dumps(self.definition['Data'])
            workbook_output = workbooks_api.create_workbook(body=workbook_input)  # type: WorkbookOutputV1

            data_id = Workbook.item_id_with_prefix(prefix, self.id)
            items_api.set_properties(id=workbook_output.id, body=[
                ScalarPropertyV1(name='Datasource Class', value=_common.DEFAULT_DATASOURCE_CLASS),
                ScalarPropertyV1(name='Datasource ID', value=_common.DEFAULT_DATASOURCE_ID),
                ScalarPropertyV1(name='Data ID', value=data_id)])

            workbook_id = workbook_output.id
        else:
            workbook_id = workbook_item.id
            props.append(ScalarPropertyV1(name='Name', value=self.definition['Name']))
            if _common.present(self.definition, 'Description'):
                props.append(ScalarPropertyV1(name='Description', value=self.definition['Description']))
            if _common.present(self.definition, 'Data'):
                props.append(ScalarPropertyV1(name='Data', value=json.dumps(self.definition['Data'])))

        if _common.present(self.definition, 'workbookState'):
            props.append(ScalarPropertyV1(name='workbookState', value=json.dumps(self.definition['workbookState'])))

        items_api.set_properties(id=workbook_id, body=props)

        for worksheet in self.worksheets:
            worksheet.push(workbook_id, prefix=prefix)

    @staticmethod
    def item_id_with_prefix(prefix, item_id):
        return item_id if prefix is None else ('%s-%s' % (prefix, item_id))

    @staticmethod
    def find_user(user_dict):
        auth_provider = None
        if user_dict['Directory']:
            auth_provider = Workbook.find_directory(user_dict['Directory'])

        users_api = UsersApi(_login.client)
        user_id = None
        if auth_provider:
            try:
                user_output = users_api.get_user_from_username(auth_datasource_class=auth_provider.datasource_class,
                                                               auth_datasource_id=auth_provider.datasource_id,
                                                               username=user_dict['Username'])  # type: UserOutputV1

                user_id = user_output.id
            except ApiException:
                # Fall through to lookup via autocomplete
                pass

        if user_id is None:
            identity_preview_list = users_api.autocomplete_users_and_groups(
                query=user_dict['Username'])  # type: IdentityPreviewListV1

            if len(identity_preview_list.items) == 1:
                identity_preview = identity_preview_list.items[0]  # type: IdentityPreviewV1
                user_id = identity_preview.id

        return user_id

    @staticmethod
    def find_directory(directory_name):
        auth_api = AuthApi(_login.client)
        auth_providers = auth_api.get_auth_providers()  # type: AuthProvidersOutputV1
        for auth_provider in auth_providers.auth_providers:  # type: DatasourceOutputV1
            if auth_provider.name == directory_name:
                return auth_provider

        return None

    @staticmethod
    def find_item_id(prefix, item_id):
        """
        :rtype: ItemSearchPreviewV1
        """
        items_api = ItemsApi(_login.client)

        if not prefix:
            try:
                item_output = items_api.get_item_and_all_properties(id=item_id)  # type: ItemOutputV1
                return item_output.id
            except ApiException:
                # Fall through to looking via Data ID
                pass

        data_id = Workbook.item_id_with_prefix(prefix, item_id)
        _filter = ['Datasource Class==%s && Datasource ID==%s && Data ID==%s' % (
            _common.DEFAULT_DATASOURCE_CLASS, _common.DEFAULT_DATASOURCE_ID, data_id)]

        search_results = items_api.search_items(
            filters=_filter,
            types=['Workbook', 'Worksheet', 'Workstep'],
            offset=0, limit=2)  # type: ItemSearchPreviewPaginatedListV1

        if len(search_results.items) == 0:
            return None

        if len(search_results.items) > 1:
            raise RuntimeError('Multiple workbook/worksheet/workstep items found with Data ID of "%s"', data_id)

        return search_results.items[0]

    @staticmethod
    def _definition_from_struct(workbook_output):
        """
        :type workbook_output: WorkbookOutputV1
        :rtype: dict
        """
        definition = Workbook.item_to_dict(workbook_output.id)

        definition['ID'] = workbook_output.id
        definition['Name'] = workbook_output.name
        definition['Description'] = workbook_output.description
        definition['Path'] = ' >> '.join([a.name for a in workbook_output.ancestors])
        definition['Workbook Type'] = Workbook._get_workbook_type(workbook_output)
        definition['Owner'] = dict()
        definition['Owner']['ID'] = workbook_output.owner.id
        definition['Owner']['Name'] = workbook_output.owner.name
        definition['Owner']['Username'] = workbook_output.owner.username

        # noinspection PyBroadException
        try:
            users_api = UsersApi(_login.client)
            user_output = users_api.get_user(id=workbook_output.owner.id)  # type: UserOutputV1
            definition['Owner']['Directory'] = user_output.datasource_name
        except BaseException:
            # If we can't get extra data on the user, that's OK
            pass

        return definition

    @property
    def id(self):
        return self.definition['ID'] if 'ID' in self.definition else None

    @property
    def name(self):
        return self.definition['Name'] if 'Name' in self.definition else None

    @property
    def referenced_items(self):
        referenced_items = list()
        for worksheet in self.worksheets:
            referenced_items.extend(worksheet.referenced_items)

        return referenced_items

    @staticmethod
    def _get_workbook_type(workbook_output):
        if not workbook_output.data:
            return 'Analysis'

        # noinspection PyBroadException
        try:
            data = json.loads(workbook_output.data)
        except BaseException:
            return 'Analysis'

        if 'isReportBinder' in data and data['isReportBinder']:
            return 'Topic'
        else:
            return 'Analysis'

    def _get_default_workbook_folder(self):
        return os.path.join(os.getcwd(), 'Workbook-%s' % self.id)

    @staticmethod
    def _get_workbook_json_file(workbook_folder):
        return os.path.join(workbook_folder, 'Workbook.json')

    @staticmethod
    def _get_items_json_file(workbook_folder):
        return os.path.join(workbook_folder, 'Items.json')

    @staticmethod
    def _get_datasources_json_file(workbook_folder):
        return os.path.join(workbook_folder, 'Datasources.json')

    def save(self, workbook_folder=None):
        if not workbook_folder:
            workbook_folder = self._get_default_workbook_folder()

        if os.path.exists(workbook_folder):
            dir_util.remove_tree(workbook_folder)

        os.makedirs(workbook_folder)

        workbook_json_file = Workbook._get_workbook_json_file(workbook_folder)

        definition_dict = dict(self.definition)
        definition_dict['Worksheets'] = list()
        for worksheet in self.worksheets:
            worksheet.save(workbook_folder)
            definition_dict['Worksheets'].append(worksheet.id)

        with open(workbook_json_file, 'w', encoding='utf-8') as f:
            json.dump(definition_dict, f, indent=4)

        items_json_file = Workbook._get_items_json_file(workbook_folder)
        with open(items_json_file, 'w', encoding='utf-8') as f:
            json.dump(self.item_inventory, f, indent=4, sort_keys=True)

        datasources_json_file = Workbook._get_datasources_json_file(workbook_folder)
        with open(datasources_json_file, 'w', encoding='utf-8') as f:
            json.dump(self.datasource_inventory, f, indent=4, sort_keys=True)

    def _load(self, workbook_folder):
        if not os.path.exists(workbook_folder):
            raise RuntimeError('Workbook folder "%s" does not exist' % workbook_folder)

        workbook_json_file = Workbook._get_workbook_json_file(workbook_folder)
        if not os.path.exists(workbook_json_file):
            raise RuntimeError('Workbook JSON file "%s" does not exist' % workbook_json_file)

        with open(workbook_json_file, 'r', encoding='utf-8') as f:
            self.definition = json.load(f)

        self.worksheets = list()
        for worksheet_id in self.definition['Worksheets']:
            self.worksheets.append(Worksheet.load(self, workbook_folder, worksheet_id))

        del self.definition['Worksheets']

        items_json_file = Workbook._get_items_json_file(workbook_folder)
        with open(items_json_file, 'r', encoding='utf-8') as f:
            self.item_inventory = json.load(f)

        datasources_json_file = Workbook._get_datasources_json_file(workbook_folder)
        with open(datasources_json_file, 'r', encoding='utf-8') as f:
            self.datasource_inventory = json.load(f)

    @staticmethod
    def load(workbook_folder):
        workbook = Workbook()
        workbook._load(workbook_folder)
        return workbook

    @staticmethod
    def _get_datasource_item_dict(datasource_class, datasource_id):
        _filters = ['Datasource Class==' + datasource_class,
                    'Datasource ID==' + datasource_id]

        filters_arg = [' && '.join(_filters), '@includeUnsearchable']

        items_api = ItemsApi(_login.client)
        item_search_list = items_api.search_items(
            types=['Datasource'],
            filters=filters_arg,
            limit=1)  # type: ItemSearchPreviewPaginatedListV1

        if len(item_search_list.items) != 1:
            raise RuntimeError(
                'Datasource Class "%s" and Datasource ID "%s" not found' % (datasource_class, datasource_id))

        return Workbook.item_to_dict(item_search_list.items[0].id)

    @staticmethod
    def item_to_dict(item_id):
        items_api = ItemsApi(_login.client)
        item_output = items_api.get_item_and_all_properties(id=item_id)  # type: ItemOutputV1
        item_dict = {prop.name: prop.value for prop in item_output.properties}
        item_dict['Name'] = item_output.name
        item_dict['Type'] = item_output.type
        if item_output.type in ['CalculatedScalar', 'CalculatedSignal', 'CalculatedCondition', 'Chart',
                                'ThresholdMetric']:
            calculated_item = Workbook._get_calculated_item(item_output)
            if calculated_item.type == 'ThresholdMetric':
                metric = calculated_item  # type: ThresholdMetricOutputV1
                item_dict['Metric Measured Item'] = metric.measured_item.id
                item_dict['Metric Bounding Condition'] = metric.bounding_condition.id

                if metric.thresholds:
                    item_dict['Thresholds'] = list()
                    for threshold in metric.thresholds:  # type: ThresholdOutputV1
                        if threshold.is_generated or threshold.value:
                            continue

                        threshold_dict = dict()
                        threshold_dict['Is Generated'] = threshold.is_generated
                        threshold_dict['Priority'] = threshold.priority
                        if threshold.item:
                            threshold_dict['Item ID'] = threshold.item.id
                        threshold_dict['Value'] = threshold.value

            else:
                item_dict['Formula Parameters'] = dict()
                for parameter in calculated_item.parameters:  # type: FormulaParameterOutputV1
                    if parameter.item:
                        item_dict['Formula Parameters'][parameter.name] = parameter.item.id
                    else:
                        item_dict['Formula Parameters'][parameter.name] = parameter.formula

        return item_dict

    def _scrape_datasource_inventory(self):
        referenced_datasources = set()
        for _id, item_dict in self.item_inventory.items():
            referenced_datasources.add((item_dict['Datasource Class'], item_dict['Datasource ID']))

        self.datasource_inventory = dict()
        for datasource_class, datasource_id in referenced_datasources:
            datasource_item_dict = Workbook._get_datasource_item_dict(datasource_class, datasource_id)
            self.datasource_inventory[datasource_item_dict['ID']] = dict()
            self.datasource_inventory[datasource_item_dict['ID']]['Datasource'] = datasource_item_dict

            self.datasource_inventory[datasource_item_dict['ID']]['Maps'] = list()

            _map = dict()
            _map['Old'] = dict()
            _map['Old']['Name'] = "(?<name>.*)"
            _map['Old']['Description'] = "(?<description>.*)"
            _map['Old']['Datasource Class'] = datasource_item_dict['Datasource Class']
            _map['Old']['Datasource ID'] = datasource_item_dict['Datasource ID']
            _map['Old']['Data ID'] = "(?<dataid>.*)"

            _map['New'] = dict()
            _map['New']['Name'] = "${name}"
            _map['New']['Description'] = "${description}"
            _map['New']['Datasource Class'] = datasource_item_dict['Datasource Class']
            _map['New']['Datasource ID'] = datasource_item_dict['Datasource ID']
            _map['New']['Data ID'] = "${dataid}"

            self.datasource_inventory[datasource_item_dict['ID']]['Maps'].append(_map)

        return self.datasource_inventory

    def _scrape_item_inventory(self):
        self._scrape_references_from_scope()

        self.item_inventory = dict()
        for reference in self.referenced_items:
            if reference.id in self.item_inventory:
                continue

            self.item_inventory[reference.id] = Workbook.item_to_dict(reference.id)

            dependencies = Workbook._scrape_references_from_dependencies(reference.id)

            for dependency in dependencies:
                if dependency.id in self.item_inventory:
                    continue

                self.item_inventory[dependency.id] = Workbook.item_to_dict(dependency.id)

    def _scrape_references_from_scope(self):
        items_api = ItemsApi(_login.client)

        self.scoped_items = list()
        offset = 0
        while True:
            search_results = items_api.search_items(
                filters=['', '@excludeGloballyScoped'],
                scope=self.id,
                offset=offset,
                limit=_common.DEFAULT_SEARCH_PAGE_SIZE,
            )  # type: ItemSearchPreviewPaginatedListV1

            self.scoped_items.extend([Reference(item.id, Reference.SCOPED) for item in search_results.items])

            if len(search_results.items) < search_results.limit:
                break

            offset += search_results.limit

        return self.scoped_items

    @staticmethod
    def _scrape_references_from_dependencies(item_id):
        items_api = ItemsApi(_login.client)
        referenced_items = list()

        try:
            dependencies = items_api.get_formula_dependencies(id=item_id)  # type: ItemDependencyOutputV1
        except ApiException as e:
            if e.status == 404:
                # For some reason, the item_id is unknown. We've seen this at Exxon, so just skip it.
                return referenced_items

            raise

        for dependency in dependencies.dependencies:  # type: ItemParameterOfOutputV1
            referenced_items.append(Reference(
                dependency.id,
                Reference.DEPENDENCY
            ))

        return referenced_items

    @staticmethod
    def _get_calculated_item(item):
        scalars_api = ScalarsApi(_login.client)
        signals_api = SignalsApi(_login.client)
        conditions_api = ConditionsApi(_login.client)
        formulas_api = FormulasApi(_login.client)
        metrics_api = MetricsApi(_login.client)

        if item.type == 'CalculatedScalar':
            item = scalars_api.get_scalar(id=item.id)  # type: CalculatedItemOutputV1
        elif item.type == 'CalculatedSignal':
            item = signals_api.get_signal(id=item.id)  # type: SignalOutputV1
        elif item.type == 'CalculatedCondition':
            item = conditions_api.get_condition(id=item.id)  # type: ConditionOutputV1
        elif item.type == 'Chart':
            item = formulas_api.get_function(id=item.id)  # type: CalculatedItemOutputV1
        elif item.type == 'ThresholdMetric':
            item = metrics_api.get_metric(id=item.id)  # type: ThresholdMetricOutputV1

        return item


class Worksheet:
    def __init__(self, workbook, definition=None):
        """
        :type workbook: Workbook
        :type definition: str, dict, WorksheetOutputV1
        """
        self.workbook = workbook

        if isinstance(definition, dict):
            self.definition = definition
        elif isinstance(definition, WorksheetOutputV1):
            self._pull(definition)
        else:
            self.definition = dict()

    def _pull(self, definition):
        worksheet_output = definition  # type: WorksheetOutputV1
        if _common.is_guid(definition):
            workbooks_api = WorkbooksApi(_login.client)
            worksheet_output = workbooks_api.get_worksheet(
                workbook_id=self.workbook.id, worksheet_id=definition)  # type: WorksheetOutputV1

        self.definition = Worksheet._definition_from_struct(worksheet_output)

        self.journal = Journal(self)

        current_workstep_id = Worksheet._get_current_workstep_id(worksheet_output.workstep)

        workstep_ids_to_retrieve = list()
        if current_workstep_id is not None:
            workstep_ids_to_retrieve.append(current_workstep_id)
        workstep_ids_to_retrieve.extend(self.journal.referenced_workstep_ids)

        self.worksteps = dict()
        for workstep_id in workstep_ids_to_retrieve:
            if workstep_id not in self.worksteps:
                self.worksteps[workstep_id] = Workstep(self, workstep_id)

        if current_workstep_id is not None:
            self.definition['Current Workstep ID'] = current_workstep_id

    def push(self, workbook_id, prefix=None):
        worksheet_item = Workbook.find_item_id(prefix, self.id)

        workbooks_api = WorkbooksApi(_login.client)
        items_api = ItemsApi(_login.client)
        props = list()
        if not worksheet_item:
            worksheet_input = WorksheetInputV1()
            worksheet_input.name = self.definition['Name']
            worksheet_output = workbooks_api.create_worksheet(
                workbook_id=workbook_id, body=worksheet_input)  # type: WorksheetOutputV1

            data_id = Workbook.item_id_with_prefix(prefix, self.id)
            items_api.set_properties(id=worksheet_output.id, body=[
                ScalarPropertyV1(name='Datasource Class', value=_common.DEFAULT_DATASOURCE_CLASS),
                ScalarPropertyV1(name='Datasource ID', value=_common.DEFAULT_DATASOURCE_ID),
                ScalarPropertyV1(name='Data ID', value=data_id)])

            worksheet_id = worksheet_output.id
        else:
            worksheet_id = worksheet_item.id
            props.append(ScalarPropertyV1(name='Name', value=self.definition['Name']))
            items_api.set_properties(id=worksheet_item.id, body=props)

        workstep_map = dict()
        for workstep_id, workstep in self.worksteps.items():
            pushed_workstep_id = workstep.push(workbook_id, worksheet_id)
            workstep_map[workstep_id] = pushed_workstep_id

            if workstep_id == self.definition['Current Workstep ID']:
                workbooks_api.set_current_workstep(workbook_id=workbook_id,
                                                   worksheet_id=worksheet_id,
                                                   workstep_id=pushed_workstep_id)

        self.journal.push(worksheet_id)

    @staticmethod
    def _definition_from_struct(worksheet_output):
        """
        :type worksheet_output: WorksheetOutputV1
        :rtype: dict
        """
        definition = Workbook.item_to_dict(worksheet_output.id)

        definition['ID'] = worksheet_output.id
        definition['Name'] = worksheet_output.name

        return definition

    @property
    def id(self):
        return self.definition['ID'] if 'ID' in self.definition else None

    @property
    def name(self):
        return self.definition['Name'] if 'Name' in self.definition else None

    @property
    def referenced_items(self):
        referenced_items = list()
        for workstep_id, workstep in self.worksteps.items():  # type: Workstep
            referenced_items.extend(workstep.referenced_items)

        referenced_items.extend(self.journal.referenced_items)

        return referenced_items

    @staticmethod
    def _get_current_workstep_id(workstep):
        if not workstep:
            return None

        return workstep.split('/')[-1]

    @staticmethod
    def _get_worksheet_json_file(workbook_folder, worksheet_id):
        return os.path.join(workbook_folder, 'Worksheet-%s.json' % worksheet_id)

    def save(self, workbook_folder):
        worksheet_json_file = Worksheet._get_worksheet_json_file(workbook_folder, self.id)

        with open(worksheet_json_file, 'w', encoding='utf-8') as f:
            json.dump(self.definition, f, indent=4)

        self.journal.save(workbook_folder)

        for workstep_id, workstep in self.worksteps.items():
            workstep.save(workbook_folder)

    def _load(self, workbook_folder, worksheet_id):
        worksheet_json_file = Worksheet._get_worksheet_json_file(workbook_folder, worksheet_id)

        with open(worksheet_json_file, 'r', encoding='utf-8') as f:
            self.definition = json.load(f)

        self.journal = Journal.load(self, workbook_folder)

        self.worksteps = dict()
        workstep_files = glob.glob(os.path.join(workbook_folder, 'Worksheet-%s-Workstep-*.json' % worksheet_id))
        for workstep_file in workstep_files:
            match = re.match(r'.*?Worksheet-.*?-Workstep-(.*?).json$', workstep_file)
            workstep_id = match.group(1)
            self.worksteps[workstep_id] = Workstep.load(self, workbook_folder, workstep_id)

    @staticmethod
    def load(workbook, workbook_folder, worksheet_id):
        worksheet = Worksheet(workbook)
        worksheet._load(workbook_folder, worksheet_id)
        return worksheet


class Journal:
    def __init__(self, worksheet):
        """
        :type worksheet: Worksheet
        """
        self.worksheet = worksheet
        self.html = None  # str

        if self.worksheet.id:
            self._pull()

        self.referenced_items = self._scrape_referenced_items()
        self.referenced_workstep_ids = self._scrape_referenced_workstep_ids()

    def _pull(self):
        annotations_api = AnnotationsApi(_login.client)
        annotations = annotations_api.get_annotations(
            annotates=[self.worksheet.id])  # type: AnnotationListOutputV1

        for annotation_item in annotations.items:  # type: AnnotationOutputV1
            annotation_output = annotations_api.get_annotation(id=annotation_item.id)  # AnnotationOutputV1
            self.html = annotation_output.document

    def push(self, worksheet_id):
        annotations_api = AnnotationsApi(_login.client)

        annotations = annotations_api.get_annotations(
            annotates=[worksheet_id])  # type: AnnotationListOutputV1

        for annotation in annotations.items:  # type: AnnotationOutputV1
            annotation = annotations_api.get_annotation(id=annotation.id)

            new_annotation = AnnotationInputV1()
            new_annotation.discoverable = annotation.discoverable
            new_annotation.document = self.html
            new_annotation.name = annotation.name
            new_annotation.description = annotation.description
            new_annotation.interests = []
            for interest in annotation.interests:  # type: AnnotationInterestOutputV1
                interest_item = interest.item  # type: ItemPreviewV1
                # At Chevron, we encountered a case where there were multiple interests returned with the same ID, which
                # caused Appserver to choke when updating the annotation. So filter those out.
                if any(interest_item.id == i.interest_id for i in new_annotation.interests):
                    continue
                new_interest = AnnotationInterestInputV1()
                new_interest.interest_id = interest_item.id
                new_interest.detail_id = interest.capsule
                new_annotation.interests.append(new_interest)
            new_annotation.type = annotation.type
            new_annotation.created_by_id = annotation.created_by.id

            annotations_api.update_annotation(id=annotation.id, body=new_annotation)

    @staticmethod
    def _get_journal_html_file(workbook_folder, worksheet_id):
        return os.path.join(workbook_folder, 'Journal-%s.html' % worksheet_id)

    def save(self, workbook_folder):
        journal_html_file = Journal._get_journal_html_file(workbook_folder, self.worksheet.id)
        with open(journal_html_file, 'w', encoding='utf-8') as f:
            if self.html:
                f.write(BeautifulSoup(self.html).prettify())

    def _load(self, workbook_folder):
        journal_html_file = Journal._get_journal_html_file(workbook_folder, self.worksheet.id)

        with open(journal_html_file, 'r', encoding='utf-8') as f:
            self.html = f.read()

    @staticmethod
    def load(worksheet, workbook_folder):
        journal = Journal(worksheet)
        journal._load(workbook_folder)
        return journal

    def _scrape_referenced_items(self):
        referenced_items = list()
        if self.html:
            matches = re.finditer(r'item&#61;(' + _common.GUID_REGEX + r')', self.html, re.IGNORECASE)
            for match in matches:
                referenced_items.append(Reference(match.group(1).upper(), Reference.JOURNAL, self.worksheet))

        return referenced_items

    def _scrape_referenced_workstep_ids(self):
        workstep_ids = list()
        if self.html:
            matches = re.finditer(r'workstep&#61;(' + _common.GUID_REGEX + r')', self.html, re.IGNORECASE)

            for match in matches:
                workstep_ids.append(match.group(1).upper())

        return workstep_ids


class Reference:
    JOURNAL = 'Journal'
    DETAILS = 'Details'
    SCOPED = 'Scoped'
    DEPENDENCY = 'Dependency'

    def __init__(self, _id, _provenance, worksheet=None):
        """
        :type _id: str
        :type _provenance: str
        :type worksheet: Worksheet
        """
        self.id = _id
        self.provenance = _provenance
        self.worksheet = worksheet


class Workstep:
    def __init__(self, worksheet, definition=None):
        """
        :type worksheet: Worksheet
        :type definition: str, dict
        """
        self.worksheet = worksheet

        if _common.is_guid(definition):
            self._pull(definition)
        elif isinstance(definition, dict):
            self.definition = definition

    def _pull(self, workstep_id):
        workbooks_api = WorkbooksApi(_login.client)
        try:
            workstep_output = workbooks_api.get_workstep(workbook_id=self.worksheet.workbook.id,
                                                         worksheet_id=self.worksheet.id,
                                                         workstep_id=workstep_id)  # type: WorkstepOutputV1

            self.definition = Workstep._definition_from_struct(workstep_output)
            self.definition['Data'] = json.loads(workstep_output.data)

        except ApiException as e:
            # Apparently it's possible for a workstep to be referenced and yet not exist
            if e.status != 404:
                raise

        self.referenced_items = self._scrape_referenced_items()

    def push(self, workbook_id, worksheet_id):
        workbooks_api = WorkbooksApi(_login.client)

        workstep_input = WorkstepInputV1()
        workstep_input.name = self.definition['Name']
        workstep_input.data = json.dumps(self.definition['Data'])
        workstep_output = workbooks_api.create_workstep(
            workbook_id=workbook_id, worksheet_id=worksheet_id, body=workstep_input)  # type: WorkstepOutputV1

        return workstep_output.id

    @staticmethod
    def _definition_from_struct(workstep_output):
        """
        :type workstep_output: WorkstepOutputV1
        :rtype: dict
        """
        return Workbook.item_to_dict(workstep_output.id)

    @property
    def id(self):
        return self.definition['ID'] if 'ID' in self.definition else None

    @property
    def data(self):
        return self.definition['Data'] if 'Data' in self.definition else dict()

    def _scrape_referenced_items(self):
        referenced_items = list()

        matches = re.finditer(_common.GUID_REGEX, json.dumps(self.data), re.IGNORECASE)

        for match in matches:
            referenced_items.append(Reference(match.group(0).upper(), Reference.DETAILS, self.worksheet))

        return referenced_items

    @staticmethod
    def _get_workstep_json_file(workbook_folder, worksheet_id, workstep_id):
        return os.path.join(workbook_folder, 'Worksheet-%s-Workstep-%s.json' % (worksheet_id, workstep_id))

    def save(self, workbook_folder):
        workstep_json_file = Workstep._get_workstep_json_file(workbook_folder, self.worksheet.id, self.id)
        with open(workstep_json_file, 'w', encoding='utf-8') as f:
            json.dump(self.definition, f, indent=4)

    def _load(self, workbook_folder, workstep_id):
        workstep_json_file = Workstep._get_workstep_json_file(workbook_folder, self.worksheet.id, workstep_id)

        with open(workstep_json_file, 'r', encoding='utf-8') as f:
            self.definition = json.load(f)

    @staticmethod
    def load(worksheet, workbook_folder, workstep_id):
        workstep = Workstep(worksheet)
        workstep._load(workbook_folder, workstep_id)
        return workstep
