"""Classes for running metamap (via subprocess) and parsing output (from json)
"""
from subprocess import Popen, PIPE, TimeoutExpired
from umls_tools.metamap import (
    objects as metamap_objects,
    processes
)
from umls_tools import objects as umls_objects, common
# from pyaddons import geometry
import sys
import re
import json
import time
import warnings
# import pprint as pp


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class MetamapServer(object):
    """Start and stop the metamap server and run metamap queries.

    Parameters
    ----------
    keep_alive : `bool`, optional, default: `False`
        The default behaviour is to start and stop the various metamap servers
        when entering/leaving the context manager (i.e. the ``with``
        statement). If this is ``True`` then the servers are started
        (if needed) not closed when the context manager exits.
    binary : `str`, optional, default: `metamap`
        The name of the metamap binary, change if needed.
    args : `list` of `str`, optional, default: `NoneType`
        Metamap arguments to use with each ``run`` command. Please note that
        these can be overridden when calling ``run``, however, if not then
        these will be used as default. If you do not supply them here you must
        supply them with ``run``
    """
    T_OPTION_REGEX = re.compile(
        r'({"AllDocuments":\s*\[)\s*\[.+\]\s*({"Document":)', re.IGNORECASE
    )
    """A regular expression to use when the -T option is used as this breaks
    the JSON and it can't be parsed (`re.Pattern`)
    """

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, keep_alive=False, binary="metamap", args=None):
        # These are indicators for the servers being available. We initialise
        # both the servers to False and then see if we can find as running
        # processes and/or run a test metamap call
        self.taggerserver = False
        self.wsdserver = False
        self.keep_alive = keep_alive
        self.binary = binary
        self._metamap_args = args

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __enter__(self):
        """Entry point for the context manager.
        """
        self.start()
        return self

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __exit__(self, exception_type, exception_value, traceback):
        """Exit point for the context manager
        """
        self.stop()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def start(self):
        """Explicit start server method. This will only attempt to start them
        if they can't be found as running processes or the metamap test run
        command fails.

        Returns
        -------
        wsdserver_proc : `umls_tools.metamap.processes.ProcessInfo`
            Information on the word sense disambiguation server process.
        taggerserver_proc : `umls_tools.metamap.processes.ProcessInfo`
            Information on the tagger server process.

        Notes
        -----
        Starting the WSD server is fiddly as it hangs in the terminal. It could
        be that it is supposed to? or maybe my JAVA version. Either way, you
        might see this warning when you first start it up and run a query.
        Everything still seems to work so I am not sure what it means!?

        .. code-block::

           WARNING: An illegal reflective access operation has occurred
           WARNING: Illegal reflective access by org.jdom.input.SAXBuilder (file:/scratch/metamap/public_mm/WSD_Server/lib/thirdparty.jar) to method com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl.getXMLReader()
           WARNING: Please consider reporting this to the maintainers of org.jdom.input.SAXBuilder
           WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
           WARNING: All illegal access operations will be denied in a future release
        """
        # First see if any processes are running, if there are that does not
        # mean that the WSD/tagger servers are running, we still need to check
        wsdserver_proc, taggerserver_proc = self.find_server_processes()
        if wsdserver_proc is not None and taggerserver_proc is not None:
            # Both the servers have processes so we run a test command to see
            # if it returns any errors
            ok = self.run_test_command()
            if ok is True:
                self.taggerserver = True
                self.wsdserver = True

        if self.taggerserver is False:
            self._start_tagger_server()

        if self.wsdserver is False:
            self._start_wsd_server()

        # First check if there are any processes, the presence of processes
        # does not mean they are running
        return self.find_server_processes()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def stop(self):
        """Explicit stop server method. Servers will only be stopped if they
        are found to be running in active processes.
        """
        # Then attempt to stop
        if self.keep_alive is False:
            self._stop_tagger_server()
            self._stop_wsd_server()

            # Make sure that all the server processes are killed
            self.kill_server_processes()

        # Switch off keep alive as this is an explicit call
        self.keep_alive = False

        # First check if there are any processes, the presence of processes
        # does not mean they are running
        wsdserver, taggerserver = self.find_server_processes()
        return wsdserver, taggerserver

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def run_test_command(self):
        """Run a test command to see if metamap is returning results from a
        known good query.

        Returns
        -------
        has_run : `bool`
            This will be ``True``, if metamap ran ok, ``False`` if not.

        Notes
        -----
        This test command runs with tagging and WSD enabled.
        """
        try:
            output, stderr_text = self._run("lung cancer", ['-yT'])
            return True
        except RuntimeError:
            return False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @staticmethod
    def find_server_processes():
        """Determine of the word sense disambiguation server (wsdserver) and
        the tagger server (taggerserver) have processes.

        Returns
        -------
        wsdserver : `umls_tools.metamap.ProcessInfo` or `NoneType`
            The process information for the word sense disambiguation server.
            This will be ``NoneType`` if no matching process is found.
        taggerserver : `umls_tools.metamap.ProcessInfo` or `NoneType`
            The process information for the tagger server. This will be
            ``NoneType`` if no matching process is found.

        Notes
        -----
        Note the processes may be running even if the servers have not been
        started. So this should not be used to determine if the servers are
        running.
        """
        wsdserver = None
        taggerserver = None

        # Loop through the running processes and attempt to locate signatures
        # of the running servers
        for i in processes.Processes.get():
            if "server.config.file=" in i.cmd:
                wsdserver = i
            elif "taggerserver.port=" in i.cmd:
                taggerserver = i
        return wsdserver, taggerserver

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def kill_server_processes(cls):
        """Kill any/all running tagger/WSD server processes.
        """
        wsd_server_proc, tagger_server_proc = cls.find_server_processes()

        while wsd_server_proc is not None or tagger_server_proc is not None:
            args = [i for i in [wsd_server_proc, tagger_server_proc]
                    if i is not None]
            processes.Processes.kill(*args)
            wsd_server_proc, tagger_server_proc = cls.find_server_processes()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _start_tagger_server(self):
        """Start the server.
        """
        # Attempt to start the tagger server
        server_output = self._run_server_command(
            "skrmedpostctl", start=True
        )
        if "started." in server_output:
            self.taggerserver = True
        # pp.pprint(server_output)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _start_wsd_server(self, timeout=10):
        """Start the word sense disambiguation server. This is much more fiddly
        than the tagger server as the WSD server hangs and blocks.

        Parameters
        ----------
        timeout : `int`
            The number of seconds we want to attempt to get the test command to
            run for. After that if we have no success then we will issue a
            warning.
        """
        # See if any of the servers are started, if not then we attempt to
        # start. If any of the servers are already started then we keep them
        # alive after the object has been destroyed
        sub_proc = Popen(
            ["wsdserverctl",  "start"], shell=False, stdout=PIPE, stdin=PIPE
        )

        try:
            # This is to stop the blocking process, we can test that the server
            # is running by issuing our test command
            sub_proc.communicate(input=b"\n", timeout=15)
        except TimeoutExpired:
            pass

        # Iterate until the command is run or we reach max iterations
        iters = 0
        while self.run_test_command() is False and iters <= timeout:
            time.sleep(1)
            iters += 1

        if iters == timeout and self.run_test_command() is False:
            warnings.warn("WSD server may not have started correctly")

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _stop_tagger_server(self):
        """Stop the tagger server.
        """
        # Attempt to start the tagger server
        server_output = self._run_server_command("skrmedpostctl", False)
        if any([re.match(r"Process \d+ stopped", i) for i in server_output]):
            self.taggerserver = False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _stop_wsd_server(self, max_iters=10):
        """Stop the WSD server.
        """
        server_output = self._run_server_command("wsdserverctl", False)
        if any([re.match(r"Process \d+ stopped", i) for i in server_output]):
            self.wsdserver = False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _run_server_command(self, server, start, wait=True):
        """Run a server start or stop command.

        Parameters
        ----------
        server : `str`
            The name of the server, either ``wsdserverctl`` or
            ``skrmedpostctl``.
        start : `bool`
            If start is ``True`` then this will start the server, if ``False``
            then it will stop the server.
        wait : `bool`, optional, default: `True`
            Should subprocesss wait after issueing the start command. This is
            mainly for the WSD server that hands when starting from cold
           (i.e. no process running).

        Returns
        -------
        output : `list` of `str`
            The STDOUT from the subprocess server start/stop command.
        """
        start_cmd = "start"
        if start is False:
            start_cmd = "stop"

        # Run a sub process to start the servers
        sub_proc = Popen([server, start_cmd], shell=False, stdout=PIPE)
        server_output = []
        for line in sub_proc.stdout:
            server_output.append(line.decode().strip())

        if wait is True:
            sub_proc.wait()
        return server_output

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def run(self, search_term, args=None):
        """Run metamap and return results objects.

        Parameters
        ----------
        search_term : `str`
            The text to parse using metamap.
        args : `list` of `str`, optional, default: `NoneType`
            Metamap arguments to use. Please note that override those given
            when initialising. If none are supplied here then the initialised
            commands are used. ``args`` must be supplied here or upon
            initialisation. The ``--JSONn`` argument is added internally.

        Returns
        -------
        results : `list` of `umls_tools.metamap.objects.Utterance`
            The utterance is the top level result of parsed text.
        """
        args = args or self._metamap_args

        if args is None:
            raise ValueError("no metamap arguments supplied")
        output, stderr_text = self._run(search_term, args)
        return MetamapJsonParser.parse(output)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def run_json(self, search_term, args=None):
        """Run metamap and return python-parsed JSON

        Parameters
        ----------
        search_term : `str`
            The text to parse using metamap.
        args : `list` of `str`, optional, default: `NoneType`
            Metamap arguments to use. Please note that override those given
            when initialising. If none are supplied here then the initialised
            commands are used. ``args`` must be supplied here or upon
            initialisation. The ``--JSONn`` argument is added internally.

        Returns
        -------
        results : `dict`
            The metamap raw results.
        """
        args = args or self._metamap_args

        if args is None:
            raise ValueError("no metamap arguments supplied")
        output, stderr_text = self._run(search_term, args)
        return output

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _run(self, search_term, args):
        """Actually run the metamap command.

        Parameters
        ----------
        search_term : `str`
            The text to parse using metamap.
        args : `list` or `str`
            The command line arguments to use when running metamap. The
            ``--JSONn`` argument is added internally.

        Returns
        -------
        results : `dict`
            The metamap raw results.
        """
        args = self._process_args(args)
        sub_proc = Popen(args,
                         stdin=PIPE,
                         stdout=PIPE,
                         stderr=PIPE)

        # Send the phrase to be parsed
        sub_proc.stdin.write("{0}\n".format(search_term).encode())
        sub_proc.stdin.close()

        # We read the Metamap results in line by line
        results = list()
        while True:
            r = sub_proc.stdout.readline()
            if not r:
                break

            results.append(r)

        # Capture STDERR
        stderr_text = sub_proc.stderr.read().decode()

        # The results string has carriage returns removed and is stored as
        # a string not bytes. We also remove the first line as this is
        # metamap repeating the input args.
        res_str = "".join([i.decode().strip() for i in results[1:]])
        try:
            # Decode the JSON into python data structures. If there is no
            # results then A JSON decode error is thrown, so we catch it
            # and raise
            return json.loads(res_str), stderr_text
        except json.decoder.JSONDecodeError as e:
            res_str, nsubs = self.T_OPTION_REGEX.subn('\\1\\2', res_str)
            if nsubs > 0:
                return json.loads(res_str), stderr_text

            raise RuntimeError(
                "results is length='{0}'".format(len(results))) from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _process_args(self, args):
        """Do some very basic argument processing, the search term is the first
        element, we also ensure that the JSON arguments are on

        Parameters
        ----------
        args : `list` or `str`
            The command line arguments to use when running metamap. The
            `--JSONn` argument is added internally.

        Returns
        -------
        args : `list` or `str`
            The processed command line arguments.
        """
        # Make sure we have JSON output
        if "--JSONf" not in args and "--JSONn" not in args:
            args = [self.binary, "--JSONn"] + args

        return args


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class MetamapJsonParser(object):
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def parse(cls, json):
        """Parse metamap parsed JSON decoded data into metamap objects.

        Parameters
        ----------
        json : `dict`
            Metamap JSON strings that have been parsed into Python objects.
            This should have an ``AllDocuments`` key with a `list` of the
            documents parsed by metamap.

        Returns
        -------
        results : `list` of `Utterances`
        """
        docs = []
        for i in json['AllDocuments']:
            docs.append(cls._parse_json_document(i))

        return docs

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _parse_json_document(cls, document):
        """Parse the utterances from the document section of a metamap JSON
        record.

        Parameters
        ----------
        document : `dict`
            A dictionary with a ``Document`` key that has a ``dict`` of
            document data fields.

        Returns
        -------
        all_utterances : `list` of `umls_tools.metamap.objects.Utterance`
            All the utterances that have been parsed out of a metamap document.
        """
        # Will hold all the parsed utterance objects.
        utt_obj = []

        # For the moment we only deal with the Utterances
        for i in document['Document']['Utterances']:
            utt_obj.append(cls._parse_utterance(i))

        return utt_obj

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _parse_utterance(cls, utt):
        """Parse an utterance section.

        Parameters
        ----------
        utt : `dict`
            A dictionary containing all the information for a single utterance.

        Returns
        -------
        utterance : `umls_tools.metamap.objects.Utterance`
            A parsed utterance.

        Notes
        -----
        An utterance is a collection of phrases that have been mapped to
        UMLS concepts.
        """
        # Loop through all the phrases and parse them
        phrase_obj = []
        for i in utt['Phrases']:
            phrase_obj.append(cls._parse_phrase(utt['UttText'], i))

        return metamap_objects.Utterance(utt['UttText'],
                                         utt['UttStartPos'],
                                         number=utt['UttNum'],
                                         section=utt['UttSection'],
                                         pmid=utt['PMID'],
                                         phrases=phrase_obj)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _parse_phrase(cls, utt_text, phrase):
        """Parse a phrase as given by an utterance.

        Parameters
        ----------
        utt_text : `str`
            The full text of the utterance from which the phrase has been
            extracted.
        phrase : `dict`
            A dictionary of the phrase data that needs to be parsed.

        Returns
        -------
        phrase : `umls_tools.metamap.objects.Phrase`
            The parsed phrase.

        Notes
        -----
        The utterance text is required for identifying the coordinates of
        UMLS CUI mappings within the phrase.
        """
        # First we parse the syntactic units, although these are loceted in the
        # phrase in the JSON output, these will be associated with mapping
        # objects in the parsed output
        syntax_units = cls._parse_syntax_units(
            phrase['PhraseText'],
            phrase['SyntaxUnits']
        )

        # Loop through all the mappings within the phrase.
        mapping_obj = []
        for i in phrase['Mappings']:
            mapping_obj.append(
                cls._parse_mapping(
                    utt_text,
                    phrase['PhraseText'],
                    phrase['PhraseStartPos'],
                    syntax_units,
                    i
                )
            )
        # Create a phrase object
        return metamap_objects.Phrase(
            phrase['PhraseText'],
            phrase['PhraseStartPos'],
            syntax_units=syntax_units,
            mappings=mapping_obj
        )

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _parse_mapping(cls, utt_text, phrase_text, phrase_start,
                       syntax_units, mapping):
        """Parse mappings into mapping objects.

        Parameters
        ----------
        utt_text : `str`
            The full text of the utterance from which the phrase has been
            extracted.
        phrase_text : `str`
            The full text of the phrase.
        phrase_start : `int`
            The index location of the start of the phrase within the utterance.
        syntax_units : `list` of `umls_tools.metamap.objects.SyntaxUnit`
            The parsed syntax units of the phrase.
        mapping : `dict`
            The mapping data, this ``dict`` should have the keys
            ``MappingCandidates`` (which contains a list of all the CUI
            mappings) and ``MappingScore`` (`int`).

        Returns
        -------
        parsed_mapping : `umls_tools.metamap.objects.Mapping`
            The mapping with the candidate CUIs
        """
        mapping_candidates = []
        for i in mapping['MappingCandidates']:
            mapping_candidates.append(
                cls._parse_mapping_candidate(utt_text, i))

        # Create the object
        # I am using the phrase_text for this
        return metamap_objects.Mapping(
            phrase_text, phrase_start,
            candidates=mapping_candidates,
            mapping_score=int(mapping['MappingScore'])
        )

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _parse_mapping_candidate(cls, utt_text, cui_mapping):
        """Parse a mapping candidate.

        Parameters
        ----------
        utt_text : `str`
            The full text of the utterance from which the phrase has been
            extracted.
        cui_mapping : `dict`
            The mapping candidate CUI information.

        Returns
        -------
        mapping_candidate : `umls_tools.metamap.objects.MappingCandidate`
            The mapping candidate CUIs. There can be > 1 CUI in a mapping
            candidate.

        Notes
        -----
        A mapping candidate is a sub component of a mapping that has been
        mapped to a UMLS CUI.
        """
        # We want to generate text for covering the extent of the mapping
        # not all the words in this region will map but all mapping words
        # will be in this region
        # First we initialise
        start_text = int(cui_mapping['ConceptPIs'][0]['StartPos'])
        end_text = (start_text +
                    int(cui_mapping['ConceptPIs'][0]['Length'])) - 1
        match_words = []
        for i in cui_mapping['ConceptPIs']:
            start = int(i['StartPos'])
            length = int(i['Length'])
            end = (start + length) - 1

            match_words.append((utt_text[start:end+1], start, end))
            start_text = min(start, start_text)
            end_text = max(end, end_text)

        # The part of the utterance that was used to map the CUIs
        cui_text = utt_text[start_text:end_text+1]

        # Now get the SABs
        sabs = []
        for i in cui_mapping['Sources']:
            sabs.append(umls_objects.Sab(i))

        # Now get the STY info
        stys = []
        for i in cui_mapping['SemTypes']:
            stys.append(umls_objects.get_sty_from_short(i))

        # Create the CUI object
        cui = umls_objects.Cui(
            cui_mapping['CandidateCUI'],
            term=cui_mapping['CandidateMatched'],
            pref_term=cui_mapping['CandidatePreferred'],
            sabs=sabs,
            stys=stys
        )
        return metamap_objects.MappingCandidate(
            cui_text, cui, start=start_text,
            mapping_score=int(cui_mapping['CandidateScore']),
            is_head=cui_mapping['IsHead'],
            is_overmatch=cui_mapping['IsOverMatch'],
            negated=cui_mapping['Negated'],
            status=cui_mapping['Status'],
            match_words=match_words,
            match_tokens=cui_mapping['MatchedWords']
        )

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _parse_syntax_units(cls, phrase_text, syntax_units):
        """Parse all the syntax units and workout their start position in their
        phrase.

        Parameters
        ----------
        phrase_text : `str`
            The full text of the phrase.
        syntax_units : `list` of `dict`
            Each `dict` is a single syntax unit with the possible keys:
            ``InputMatch``, ``LexCat``, ``LexMatch``, ``SyntaxType``,
            ``Tokens``.

        Returns
        -------
        syntax_units : `list` of `umls_tools.metamap.objects.SyntaxUnit`
            The parsed syntax units.
        """
        # Will hold all the parsed syntax unit objects
        syntax_unit_obj = []

        # The assumption is that the syntax units are in order which they
        # appear to by in the Metamap JSON output
        start_from = 0
        for i in syntax_units:
            try:
                idx = phrase_text.index(i['InputMatch'], start_from)
                start_from = idx + (len(i['InputMatch']) - 1)
            except ValueError:
                # I have noticed that you get errors like this:
                # ValueError: Parse error: 'delta - 6 desaturase'
                # is not in 'delta-6 desaturase measurement
                # So we will remove hyphenated flanked white space
                try:
                    i['InputMatch'] = re.sub(r'\s*([+/\-])\s*',
                                             r'\1', i['InputMatch'])
                    idx = phrase_text.index(i['InputMatch'], start_from)
                    start_from = idx + (len(i['InputMatch']) - 1)
                except ValueError as e:
                    raise ValueError(
                        "Parse error: '{0}' is"
                        " not in '{1}".format(i['InputMatch'], phrase_text)
                    ) from e

            kwargs = {}
            # Loop through the key word args, not all will be present
            for kw in [('LexMatch', 'lex_match'), ('SyntaxType', 'syn_type'),
                       ('LexCat', 'lex_cat'), ('Tokens', 'tokens')]:
                try:
                    kwargs[kw[1]] = i[kw[0]]
                except KeyError:
                    # Syntax unit might not have all the possible keys
                    pass

            syntax_unit_obj.append(
                metamap_objects.SyntaxUnit(i['InputMatch'], idx, **kwargs)
            )

        return syntax_unit_obj


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def summarise(results):
    """Extract the pertinent info from the results.

    Parameters
    ----------
    results : `list` of `umls_tools.metamap.objects.Utterance`
        The result utterances.

    Returns
    -------
    summary : `list` of `dict`
        The summarised data.
    """
    flattened = []
    for pc, p in enumerate(results):
        row = {'doc_no': pc}
        for uc, u in enumerate(p):
            row['utt_no'] = uc
            row['utt_text'] = u.text
            row['utt_coords'] = (u.start, u.end)
            for cph, ph in enumerate(u.phrases):
                row['ph_no'] = cph
                row['ph_text'] = ph.text
                row['ph_coords'] = (ph.start, ph.end)
                for mc, m in enumerate(ph.mappings):
                    row['map_no'] = mc
                    row['map_text'] = m.text
                    row['map_coords'] = (m.start, m.end)
                    for cc, c in enumerate(m.candidates):
                        row['cand_no'] = cc
                        row['cand_text'] = c.text
                        row['cand_coords'] = (c.start, c.end)
                        row['cand_neg'] = c.negated
                        row['cuis'] = (c.cui, c.cui.stys)
                        flattened.append(row.copy())
    return flattened


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class TagTerm(object):
    """Overlay matamap results onto a term string and also add your own
    annotations.

    Parameters
    ----------
    term : `str`
        The term that will be overlayed with metamap results.
    remap : `dict` or `NoneType`, optional, default: `NoneType`
        Do you want to map semantic types to simpler groups, if so provide
        the mappings here ``{<SEMANTIC TYPE>: GROUP}``.
    include : `dict` or `NoneType`, optional, default: `NoneType`
        Do not add candidate terms unless they have semantic types that
        occur in this list.
    exclude : `list` or `NoneType`, optional, default: `NoneType`
        Do not add candidate terms if they have semantic types that
        occur in this list. Include has priority over exclude.
    tag_delimiter : `tuple` or `NoneType`, optional, default: `('<', '>)`
        The delimiter for a tag name when output as a string. Should be of
        length two with element [0] being the left delimiter and element [1]
        being the right delimiter.
    """
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, term, remap={}, include=[], exclude=[],
                 tag_delimiter=('<', '>')):
        if not isinstance(term, str):
            raise ValueError("term must be a string")

        self.proc_term = []
        self.candidates = []
        self.remap = remap or dict()
        self.include = include or list()
        self.exclude = exclude or list()
        self.tag_delimiter = tag_delimiter

        if len(self.tag_delimiter) != 2:
            raise ValueError("tag_delimiter must of length 2")

        # We are not collapsing excessive space at the moment
        curr_coord = 0
        for i in term.split(' '):
            if i == '':
                i = ' '
            start = curr_coord
            end = start + len(i) - 1

            # Account for the space and 0-based
            curr_coord = end + 2
            self.proc_term.append(BaseTag(start, end, i, None))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __repr__(self):
        """Pretty Printing
        """
        return self.get_string(remove=True)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def add_tag_equals_str(self, str_text, tag):
        """Adds a tag based on a term text == a string

        Parameters
        ----------
        str_text :`str`
            The string to search for
        tag :`str`
            The tag to assign to matching strings
        """
        for i in self.proc_term:
            if i.text == str_text:
                i.tag = tag

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_string(self, remove=False):
        """Get the term as a string with the words represented by the tags.

        Parameters
        ----------
        remove : `bool`, optional, default: `False`
            Remove strings from the output that do not have any assigned tags.

        Returns
        -------
        tag_str : `str`
            The string represented as tags, with tags separated by spaces.
        """
        str_tags = []

        for i in self.get_tag_list():
            # Is the TAG in remap
            for t in i.all_tags:
                try:
                    str_tags.append(self.remap[t])
                except KeyError:
                    if t is not None:
                        str_tags.append(t)
                    elif remove is False:
                        str_tags.append(i.text)

        return " ".join(str_tags)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_tag_list(self):
        """Get a list of the tags from the original term and the overlayed metamap
        tags.

        Returns
        -------
        merged_tags : `list`
            The merged tags.

        Notes
        -----
        TODO: I need to update this docstring.
        """
        # Will hold the overlayed tag list
        tag_list = []

        # Make sure the metamap candidates are sorted and copied into a new
        # list (as this will be emptied as they are overlayed
        self.candidates.sort(key=lambda x: x.start)
        ordered_candidates = self.candidates

        try:
            # Grad the first candidate, if the candidates are emptry then this
            # will raise an error, in which case we return all the term tags
            cand = ordered_candidates.pop(0)
        except IndexError:
            return list(self.proc_term)

        tagno = 0
        overlap_sequence = []
        while True:
            try:
                i = self.proc_term[tagno]
            except IndexError:
                break

            # Does the current candidate overlay the current term
            if common.overlap_1d((i.start, i.end),
                                 (cand.start, cand.end)) > 0:
                # if so does the current term have a manual tag assigned to it
                # if it does then we keep it and also keep the candidate. If it
                # is None then we bin ther term and keep the candidate
                if i.tag is not None:
                    tag_list.append(i)

                overlap_sequence.append(i)
                tagno += 1
                # So now we move on as we do not want to add this term
                continue
            else:
                # Here there is no overlap, what we do depends on if we have
                # had previous matches with this candidate. If not then then we
                # do nothing. However, if we have then this is the end of a
                # sequence of matches so we want to move onto the next
                # candidate
                if len(overlap_sequence) > 0:
                    tag_list.append(cand)
                    try:
                        cand = ordered_candidates.pop(0)
                    except IndexError:
                        # No more candidates so we break out and just add the
                        # rest of the tags
                        break

                    # Make sure any overlap sequences are cleared and that the
                    # tagno is set back one as we want to test the current term
                    # again against the new candidate
                    overlap_sequence = []
                    continue
            tag_list.append(i)
            tagno += 1

        if len(overlap_sequence) > 0:
            tag_list.append(cand)
            overlap_sequence = []

        # Make sure any remaining terms are added
        tag_list.extend(self.proc_term[tagno:])
        return tag_list

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def add_utterance(self, utterance):
        """Adds all the MappingCandidates in an Utterance.

        Parameters
        ----------
        utterance : `umls_tools.metamap.objects.Utterance`
            An utterance object to overlay.

        Notes
        -----
        The mapping candidates are stored in ``CandidateTag`` objects that can
        store one or more ``MappingCandidates``.
        """
        # The candidates, this extracts all the candidates that should be in
        # the utterance
        cand = []

        # The results should be in order, so store all the mapping candidates
        # The candidates and MappingCandidate objects that inturn will contain
        # CUIs andthese will contain Stys
        for ph in utterance.phrases:
            for m in ph.mappings:
                cand.extend(m.candidates)

        # Make sure the candidates are sorted on the start position
        self.candidates.sort(key=lambda x: x.start)

        # Loop through all the candidates
        while True:
            try:
                # get the current candidate, when we have exhausted all of them
                # then we break out of the loopx
                cur_cand = cand.pop(0)
            except IndexError:
                break

            # Now we check if we want to keep this candidate based on
            # include/exclude
            stys = [s.term for s in cur_cand.cui.stys]
            if len(self.include) > 0:
                # Get the semantic types
                matches = [s for s in stys if s in self.include]

                # If we have not matched any of the includes then we continue
                if len(matches) == 0:
                    continue
            elif len(self.exclude) > 0:
                # If we have defined some excludes, that is we will not use
                # MappingCandidates that have an STY in excludes
                # Get the semantic types that are in the excludes
                matches = [s for s in stys if s in self.exclude]
                # if we have an sty that matches an exclude and we have another
                # sty that does not match an exclude, then we will only remove
                # the offending STY
                if len(matches) >= 1 and len(stys) > 1:
                    # remove the sty from the cui
                    for c in cur_cand.cui:
                        idx = 0
                        while True:
                            try:
                                s = c.stys[idx]
                            except IndexError:
                                break

                            if s.term in matches:
                                c.stys.pop(idx)
                elif len(matches) >= 1:
                    continue

            # Now loop through all the CandidateTag objects that we have, the
            # CandidateTag objects can contain more than one candidate (i.e.
            # for overlapping candidates). found will indicate if we have found
            # a MappingCandidate that overlaps a CandidateTag, if so it is
            # added to the CandidateTag
            found = False
            for i in self.candidates:
                # if we overlap
                if common.overlap_1d((i.start, i.end),
                                     (cur_cand.start, cur_cand.end)) > 0:
                    # Add to the CandidateTag and do not check any more
                    # CandidateTags
                    i.add(cur_cand)
                    found = True
                    break

            # If we have not found any overlaps then we generate a new
            # CandidateTag and store it
            if found is False:
                self.candidates.append(CandidateTag(cur_cand))


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class BaseTag(object):
    """A representation of a simple tagged term.

    Parameters
    ----------
    start : `int`
        The start location of the term in the parent text.
    end : `int`
        The end location of the term in the parent text.
    text : `str`
        The text of the term.
    tag : `str` or `NoneType`, optional, default: `NoneType`
        The tag of the term.
    """
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, start, end, text, tag=None):
        self.start = start
        self.end = end
        self.text = text
        self.tag = tag

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __repr__(self):
        return "<{0}|{1}>".format(self.text, self.tag)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def all_tags(self):
        """Return all the tags associated with the term
        """
        return [self.tag]


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class CandidateTag(object):
    """A representation of a UMLS mapped tag.

    Parameters
    ----------
    candidate : `umls_tools.metamap.objects.MappingCandidate`
        The mapping candidate to add.
    """
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, candidate):
        self.candidates = [candidate]
        self._sorted = True

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __repr__(self):
        return "<{0}|{1}>".format(self.text, self.tag)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def add(self, candidate):
        """Add a mapping candidate.

        Parameters
        ----------
        candidate : `umls_tools.metamap.objects.MappingCandidate`
            The mapping candidate to add.
        """
        self.candidates.append(candidate)
        self._sorted = False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def start(self):
        """The tag start position.
        """
        self._do_sort()
        return self.candidates[0].start

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def end(self):
        """The tag end position.
        """
        self._do_sort()
        return self.candidates[-1].end

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def text(self):
        """The tag text position.
        """
        self._do_sort()
        return " ".join([i.cui.pref_term for i in self.candidates])

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def tag(self):
        """Get concatenated terms and semantic types.
        """
        self._do_sort()
        return "|".join(
            ["|".join([s.term for s in i.cui.stys]) for i in self.candidates]
        )

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def all_tags(self):
        self._do_sort()
        return [s.term for i in self.candidates for s in i.cui.stys]

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _do_sort(self):
        """Sort the mapping candidates if required.
        """
        if self._sorted is False:
            self.candidates.sort(key=lambda x: x.start)
            self._sorted = True
