# todo: detect loops generated by forwardMatch
# debugging parser (side by side)
# unit conversion
# caching and section triggers

from collections import OrderedDict
import types
import re
import sys
import os
import logging
import inspect
import io
from urllib.parse import urlparse, urlunparse
from builtins import str
from builtins import range
from builtins import object

from nomadcore.json_support import jsonIndentF
from nomadcore.unit_conversion import unit_conversion
from nomadcore.caching_backend import CachingLevel, ActiveBackend
from nomadcore.annotator import Annotator


logger = logging.getLogger("nomadcore.simple_parser")
logger.debug = types.MethodType(lambda *args, **kwargs: None, logger)

annotate = False


class SkipFileException(Exception):
    pass


class PushbackLineFile(object):
    """a file interface where it is possible to put back read lines"""

    def __init__(self, fIn, lines=tuple()):
        self.fIn = fIn
        self.fInLine = None
        self.lines = list(lines)
        self.lineNr = 0
        self.name = fIn.name

    def readline(self):
        if self.lines:
            self.lineNr += 1
            return self.lines.pop()
        else:
            res = self.fIn.readline()
            if res:
                self.fInLine = res
                self.lineNr += 1
            return res

    def peekline(self):
        pos = self.fIn.tell()
        line = self.fIn.readline()
        self.fIn.seek(pos)
        return line

    def pushbackLine(self, line):
        self.fInLine = line
        self.lines.append(line)
        self.lineNr -= 1


def uriFromRelPath(baseUri, basePath, newPath):
    """
    Returns an uri corresponding to newPath assuming that base path has uri baseUri.

    This will never change the net location (archive).
    """
    p1 = os.path.normpath(os.path.abspath(basePath))
    p2 = os.path.normpath(os.path.abspath(newPath))
    rPath = os.path.relpath(p2, p1)
    bUri = urlparse(baseUri)
    nUri = bUri._replace(path=os.path.normpath(os.path.join(bUri.path, rPath)))
    return urlunparse(nUri)


class SimpleMatcher(object):
    """
    A Something that matches either a single line, or multiple lines grouped together.

    This is the base of a declarative parser.
    """
    class SubFlags(object):
        Sequenced = 0    # the subMatchers should be executed in sequence
        Unordered = 1    # the subMatchers can be in any order

        # if the matcher has weak == true, this matcher should nevel steal
        # the matcher can/should repeat immediately

    def __init__(self, startReStr, endReStr=None, subMatchers=tuple(), sections=tuple(),
                 subFlags=SubFlags.Sequenced,
                 weak=False,     # this matcher should not "steal" the position
                 repeats=False,  # this matcher is expected to repeat
                 required=False,  # this value is required to have been matched on section close
                 # this section goes not steal the context, useful for example for low level debugging/error messages.
                 floating=False,
                 # valid from the point they are read until the exit from the enclosing section
                 # if start match should not eat input, but be forwarded to adHoc and subMatchers
                 forwardMatch=False,
                 name="",
                 adHoc=None,
                 otherMetaInfo=tuple(),  # The metainfos that are later manually added ot the backend
                 fixedStartValues=None,
                 fixedEndValues=None,
                 dependencies=None,
                 defLine=0,
                 defFile='',
                 coverageIgnore=False,  # mark line as ignored in coverage analysis
                 onClose=None,   # A dictionary of onClose callbacks that are specific to this SimpleMatcher
                 onOpen=None,   # A dictionary of onOpen callbacks that are specific to this SimpleMatcher
                 # A callback function that is called when the starting regex is matcher. If the regex has any capturing groups, they will be provided as well to this function as parameter called "groups".
                 startReAction=None,
                 ):
        self.index = -1
        self.startReStr = startReStr
        self.endReStr = endReStr
        self.subMatchers = list(subMatchers)
        self.sections = list(sections)
        self.subFlags = subFlags
        self.weak = weak
        self.repeats = repeats
        self.required = required
        self.floating = floating
        self.forwardMatch = forwardMatch
        self.superMatcher = None
        self.adHoc = adHoc
        self.otherMetaInfo = list(otherMetaInfo)
        if dependencies is None:
            dependencies = {}
        self.dependencies = dependencies
        self.name = name
        self.fixedStartValues = fixedStartValues
        self.fixedEndValues = fixedEndValues
        self.onClose = onClose
        self.onOpen = onOpen
        self.startReAction = startReAction
        # Boolean flag used by the ParserOptimizer to determine which SimpleMatchers to keep
        self.keep = False
        # boolean flag to signal that this SimpleMatcher does not have any
        # effect (besides progressing input file):
        #   - no data extracted
        #   - no submatchers
        #   - no adHoc
        #   - no sections
        #   - no startReActions
        self.does_nothing = \
            len(subMatchers) == 0 and \
            len(sections) == 0 and \
            fixedStartValues is None and \
            fixedEndValues is None and \
            adHoc is None and \
            startReAction is None

        if self.does_nothing:
            if startReStr is not None and len(extractGroupNames(startReStr)) > 0:
                self.does_nothing = False
            elif endReStr is not None and len(extractGroupNames(endReStr)) > 0:
                self.does_nothing = False

        self.coverageIgnore = coverageIgnore
        if coverageIgnore and not self.does_nothing:
            logger.debug(
                "SimpleMatcher '%s' has an effect and is marked as coverageIgnore", name)

        self.coverageIgnore = coverageIgnore
        caller = inspect.currentframe()
        if (defFile == '') and (defLine == 0):
            if (caller is not None) and (caller.f_back is not None):
                callerInfo = inspect.getframeinfo(caller.f_back)
                self.defFile = callerInfo.filename
                self.defLine = callerInfo.lineno
                # logger.debug('caller frame info %s' % str(callerInfo))
            else:
                logger.info('could not get caller info')
        else:
            self.defFile = defFile
            self.defLine = defLine
        for g in subMatchers:
            if g.superMatcher is None:
                g.superMatcher = self
            elif g.superMatcher != self:
                raise Exception("added %s as submatcher of %s, but is already a submatcher of %s" %
                                (g.desc(), self.desc(), g.superMatcher.desc()))

    def copy(self):
        """creates a deep copy (but with no index, and in the top level copy no superMatcher)"""
        simpleMatcher = SimpleMatcher(
            startReStr=self.startReStr,
            endReStr=self.endReStr,
            subMatchers=[x.copy() for x in self.subMatchers],
            sections=self.sections,
            subFlags=self.subFlags,
            weak=self.weak,
            repeats=self.repeats,
            required=self.required,
            floating=self.floating,
            forwardMatch=self.forwardMatch,
            adHoc=self.adHoc,
            otherMetaInfo=self.otherMetaInfo,
            name=self.name,
            fixedStartValues=self.fixedStartValues,
            fixedEndValues=self.fixedEndValues,
            dependencies=self.dependencies,
            defLine=self.defLine,
            defFile=self.defFile,
            onClose=self.onClose,
            onOpen=self.onOpen,
            startReAction=self.startReAction)
        simpleMatcher.keep = self.keep
        return simpleMatcher

    def toDict(self):
        res = {
            "index": self.index,
            "startReStr": self.startReStr,
            "endReStr": self.endReStr,
            "subMatchersIndexes": [x.index for x in self.subMatchers],
            "sections": self.sections,
            "subFlags": self.subFlags,
            "defFile": self.defFile,
            "defLine": self.defLine,
            "superMatcher": self.superMatcher.index if self.superMatcher else None
        }
        if self.weak:
            res["weak"] = True
        if self.required:
            res["required"] = True
        if self.repeats:
            res["repeats"] = True
        if self.name:
            res["name"] = self.name
        if self.forwardMatch:
            res["forwardMatch"] = True
        if self.adHoc:
            res["adHoc"] = True
        return res

    def desc(self):
        """description of the matcher"""
        if len(self.name) > 0:
            return ("matcher #%d %s (%s)" % (self.index, self.name, self.startReStr))
        else:
            return ("matcher #%d (%s)" % (self.index, self.startReStr))

    def calcIndexes(self, flatIndex):
        """calculates the flat index of this and each sub matcher"""
        flatIndex.append(self)
        if self.index != -1:
            raise Exception("matcher already had index %d, now inserted at %d" % (
                self.index, len(flatIndex) - 1))
        self.index = len(flatIndex) - 1
        for m in self.subMatchers:
            m.calcIndexes(flatIndex)

    def clearIndexes(self):
        """ clears indexes of this matcher (and all submatchers)"""
        self.index = -1
        for g in self.subMatchers:
            g.clearIndexes()

    def directMetaNames(self):
        """The names of the meta infos the could be matched by this matcher"""
        groups = self.groupNames()
        groups.extend(self.otherMetaInfo)
        return groups

    def groupNames(self):
        """Return the groupnames that are in startReStr or in endReStr.
        """
        groups = [x[0] for x in extractGroupNames(self.startReStr)]
        if self.endReStr:
            groups.extend([x[0] for x in extractGroupNames(self.endReStr)])
        return groups

    def allMetaNames(self, metaNames=None):
        """The names of the meta infos the could be matched by this matcher or its sub matchers.
        groups can be a """
        if metaNames is None:
            metaNames = []

        metaNames.extend(self.directMetaNames())
        for m in self.subMatchers:
            m.allMetaNames(metaNames)
        return metaNames

    def allSections(self, sections=None):
        """all the names of the sections that might be opened by this matcher"""
        if sections is None:
            sections = set()
        sections.extends(self.sections)
        for m in self.subMatchers:
            m.allSections(sections)
        return sections

    def printParsingTree(self, level=0):
        """Print out a textual representation of the parsing tree for debugging
        purposes. Shows the different sections and metanames in a tree-like
        hierarchy.
        """
        metanames = self.directMetaNames()
        sections = self.sections

        # First print sections
        if sections:
            logger.info((level) * "  " + "SECTIONS: " + str(sections))

        # Then print metanames. Indentation increases if a section has also been opened.
        metaname_level = level
        if sections and metanames:
            metaname_level += 1
        if metanames:
            logger.info(metaname_level * "  " + "METAINFOS: " + str(metanames))

        # Recursion
        new_level = level
        if sections or metanames:
            new_level += 1
        for child in self.subMatchers:
            child.printParsingTree(new_level)


def closingParen(regex, i, nparen):
    """returns the position after the closing parethesis"""
    j = i
    length = len(regex)
    while j < length:
        c = regex[j]
        j += 1
        if c == '\\':
            j += 1
        elif c == '(':
            nparen += 1
        elif c == ')':
            nparen -= 1
            if nparen <= 0:
                return j
    return length


def disableGroups(regex):
    """change all groups to non capturing"""
    i = 0
    j = 0
    res = ""
    flags = ""
    length = len(regex)
    try:
        while j < length:
            c = regex[j]
            j += 1
            if c == '\\':
                c = regex[j]
                if c in "0123456789":
                    raise Exception('backreferneces not supported')
                j += 1
            elif c == '(':
                if j >= length:
                    break
                c = regex[j]
                j += 1
                if c == '?':
                    if j >= length:
                        break
                    c = regex[j]
                    j += 1
                    if c == ':':
                        pass
                    elif c == 'P':
                        c = regex[j]
                        j += 1
                        if c == '<':
                            res += regex[i: j - 2] + ":"
                            k = regex.find('>', j)
                            if k == -1:
                                raise Exception("expected (P<xx>)")
                            i = k + 1
                            j = k + 1
                        elif c == '=':
                            raise Exception('backreferneces not supported')
                        else:
                            raise Exception(
                                'unexpected character sequence "(P%s"' % c)
                    elif c in 'im':
                        if c == 'i':
                            flags += 'i'
                        res += regex[i: j - 3]
                        while j < length:
                            c = regex[j]
                            j += 1
                            if c == ')':
                                break
                            if c == 'i':
                                flags += 'i'
                            elif not c == 'm':
                                raise Exception(
                                    'Regexp flag %s is not supported by disableGroups only i (ignore case) is supported, and m (multiline) is the default' % c)
                        i = j
                    elif c in 'Lsux':
                        raise Exception(
                            'Regexp flag %s is not supported by disableGroups' % c)
                    elif c == '#':
                        res += regex[i]
                        j = closingParen(regex, j, 1)
                        i = j
                    elif c in '=!':
                        pass
                    elif c == '<':
                        if j < length:
                            c = regex[j]
                            j += 1
                            if c in '=!':
                                raise Exception(
                                    "Regexp sequence (?<%s...) unknwon" % c)
                    elif c == '(':
                        raise Exception(
                            "boolean matching not supported by disableGroups")
                    else:
                        raise Exception("unsupported character " + c + " after (?")
                else:
                    res += regex[i: j - 1] + "?:"
                    i = j - 1
    except Exception as e:
        raise Exception("disableGroups failure on '%s' at %d" %
                        (regex, j - 1), e)
    res += regex[i:]
    return res


def decodeUnits(units):
    """decodes units: _ between letters is a *, _ before a number is a -, and ^ is inserted before numbers
    for example "m_s_1" becomes m*s^-1"""
    i = 0
    j = 0
    res = []
    chars = 1
    number = 2
    underscoreAfterChar = 3
    underscoreAfterNumber = 4
    state = chars
    length = len(units)
    while j < length:
        if state == chars:
            while j < length:
                c = units[j]
                j += 1
                if c.isalpha():
                    pass
                elif c.isdigit():
                    res.append(units[i:j - 1])
                    i = j - 1
                    res.append('^')
                    state = number
                    break
                elif c == '_':
                    res.append(units[i:j - 1])
                    i = j
                    state = underscoreAfterChar
                    break
                else:
                    raise Exception(
                        "invalid char %r at %d while decoding units from %r" % (c, j - 1, units))
        elif state == number:
            while j < length:
                c = units[j]
                j += 1
                if c.isalpha():
                    res.append(units[i:j - 1])
                    res.append('*')
                    i = j - 1
                    state = chars
                    break
                elif c.isdigit():
                    pass
                elif c == '_':
                    res.append(units[i:j - 1])
                    i = j
                    state = underscoreAfterNumber
                    break
                else:
                    raise Exception(
                        "invalid char %r at %d while decoding units from %r" % (c, j - 1, units))
        elif state == underscoreAfterChar:
            while j < length:
                c = units[j]
                j += 1
                if c.isalpha():
                    res.append('*')
                    i = j - 1
                    state = chars
                    break
                elif c.isdigit():
                    res.append('^-')
                    i = j - 1
                    state = number
                    break
                else:
                    raise Exception(
                        "invalid char %r at %d while decoding units from %r" % (c, j - 1, units))
        elif state == underscoreAfterNumber:
            while j < length:
                c = units[j]
                j += 1
                if c.isalpha():
                    res.append('*')
                    i = j - 1
                    state = chars
                    break
                else:
                    raise Exception(
                        "invalid char %r at %d while decoding units from %r" % (c, j - 1, units))
        else:
            raise Exception(
                "invalid state %s at %d while decoding units from %r" % (state, j - 1, units))
    if state == chars:
        res.append(units[i:])
    elif state == number:
        res.append(units[i:])
    else:
        raise Exception(
            "invalid state %s at end of decodeUnits(%r)" % (state, units))
    return "".join(res)


def extractGroupNames(regex):
    """extracts all group names from a regex"""
    j = 0
    groupNames = []
    length = len(regex)
    try:
        while j < length:
            c = regex[j]
            j += 1
            if c == '\\':
                c = regex[j]
                if c in "0123456789":
                    raise Exception('backreferences not supported')
                j += 1
            elif c == '(':
                if j >= length:
                    break
                c = regex[j]
                j += 1
                if c == '?':
                    if j >= length:
                        break
                    c = regex[j]
                    j += 1
                    if c == ':':
                        pass
                    elif c == 'P':
                        c = regex[j]
                        j += 1
                        if c == '<':
                            k = regex.find('>', j)
                            if k == -1:
                                raise Exception("expected (P<xx>)")
                            gs = regex[j: k].split("__")
                            if len(gs) > 2:
                                raise Exception(
                                    "only a __ is accepted in a group name (to add units), got %s" % regex[j: k])
                            if len(gs) == 2:
                                unit = decodeUnits(gs[1])
                            else:
                                unit = None
                            gAndUnit = (gs[0], unit)
                            groupNames.append(gAndUnit)
                            j = k + 1
    except Exception as e:
        raise Exception("extractGroupNames failure on '%s' at %d" %
                        (regex, j - 1), e)
    return groupNames


class CompiledMatcher(object):

    def matchersToRe(self, possibleNexts):
        "builds a unique regular expression for the given possible matchers"
        res = []
        for startEnd, nextMatcher in possibleNexts:
            baseRe = disableGroups(
                nextMatcher.startReStr if startEnd == 0 else nextMatcher.endReStr)
            newRe = "(?P<i%d>%s)" % (nextMatcher.index * 2 + startEnd, baseRe)
            res.append(newRe)
        return re.compile("|".join(res))

    def __init__(self, matcher, possibleNexts, possibleNextsEnd, parserBuilder):
        self.matcher = matcher
        self.groupname = [[], []]
        self.startRe = re.compile(matcher.startReStr)
        self.groupname[0].extend(str(num)
                                 for num in range(1, self.startRe.groups + 1))
        for name, num in self.startRe.groupindex.items():
            self.groupname[0][num - 1] = name
        self.endRe = None
        if matcher.endReStr:
            self.endRe = re.compile(matcher.endReStr)
            self.groupname[1].extend(str(num)
                                     for num in range(1, self.endRe.groups + 1))
            for name, num in self.endRe.groupindex.items():
                self.groupname[1][num - 1] = name
        self.floating = matcher.floating
        currentM = matcher.superMatcher
        while currentM:
            if currentM.floating:
                if self.floating:
                    raise Exception("nesting of floating in floating not supported (%s in %s)" % (
                        currentM.desc(), matcher.desc()))
                self.floating = True
            currentM = currentM.superMatcher
        self.possibleNexts = possibleNexts
        self.possibleNextsEnd = possibleNextsEnd
        self.possibleNextsRe = self.matchersToRe(possibleNexts)
        self.possibleNextsEndRe = self.matchersToRe(possibleNextsEnd)
        strValueTransform = parserBuilder.strValueTransform
        if strValueTransform is None:
            strValueTransform = {}
        converters = {}
        transformers = {}
        groups = extractGroupNames(matcher.startReStr)
        if matcher.endReStr:
            groups.extend(extractGroupNames(matcher.endReStr))
        for gName, units in groups:
            metaInfo = parserBuilder.metaInfoEnv.infoKinds[gName]
            if units in strValueTransform:
                # override units by output units of string transform function
                transformers[gName] = strValueTransform[units][0]
                units = strValueTransform[units][1]
            if units:
                # By default use the unit given in metainfo
                target_unit = metaInfo.units

                # If there is a metainfo-specific unit conversion, use it
                metainfo_unit = None
                if parserBuilder.metainfo_units is not None:
                    metainfo_unit = parserBuilder.metainfo_units.get(
                        metaInfo.name)
                    if metainfo_unit is not None:
                        target_unit = metainfo_unit
                # If there is a dimension-specific unit conversion, use it
                if metainfo_unit is None and parserBuilder.default_units is not None:
                    source_dim = unit_conversion.ureg(units).units.dimensionality
                    map_unit = parserBuilder.default_units.get(str(source_dim))
                    if map_unit:
                        target_unit = map_unit

                converters[gName] = unit_conversion.convert_unit_function(
                    units, target_unit)
        self.converters = converters
        self.transformers = transformers

    def addStrValue(self, backend, metaNameWithUnits, strValue):
        """adds a string value with unit conversions (only for the groups in start and endRe)"""
        metaName = metaNameWithUnits.split("__")[0]
        transformer = self.transformers.get(metaName, None)
        if transformer:
            value = transformer(strValue)
        else:
            value = backend.convertScalarStringValue(metaName, strValue)
        return metaName, self.addValue(backend, metaName, value)

    def addValue(self, backend, metaName, value):
        """Adds a value with unit conversions (only for the groups in start and
        endRe)
        """
        converter = self.converters.get(metaName, None)
        if converter:
            value = converter(value)
        backend.addValue(metaName, value)
        return value

    def toDict(self):
        res = {
            "matcher": self.matcher.toDict()
        }
        res["startRe"] = self.startRe.pattern if self.startRe else ""
        res["endRe"] = self.endRe.pattern if self.endRe else ""
        res["floating"] = self.floating
        res["possibleNextsRe"] = self.possibleNextsRe.pattern if self.possibleNextsRe else ""
        res["possibleNextsEndRe"] = self.possibleNextsEndRe.pattern if self.possibleNextsEndRe else ""
        return res

    def handleStartMatch(self, parser):
        line = parser.fIn.readline()
        logger.debug("handleStartMatch of %s on (%s)",
                     self.matcher.desc(), line)
        m = self.startRe.match(line)  # , re.MULTILINE)
        if not m:
            raise Exception("Expected to match %s on %r" %
                            (self.startRe.pattern, line))
        self.handleMatchTelemetry(parser, m, line, 0)
        result_dict = {}

        # If a separate transform function was defined in the matcher, call it
        if self.matcher.startReAction is not None:
            self.matcher.startReAction(parser.backend, m.groups())

        for k, v in sorted(m.groupdict().items()):
            if v is None:
                # a group may be optional (subexpression of ? or | in regex)
                continue
            k_converted, v_converted = self.addStrValue(parser.backend, k, v)
            result_dict[k_converted] = v_converted

        # Emit fixed start values
        if self.matcher.fixedStartValues:
            for k, v in sorted(self.matcher.fixedStartValues.items()):
                if k not in result_dict:
                    v_converted = self.addValue(parser.backend, k, v)
                    result_dict[k] = v_converted

        # If the match needs to be forwarded, push the read line back into the
        # file stream
        if self.matcher.forwardMatch:
            logger.debug(
                "handleStartMatch of %s on (%s) pushing back line", self.matcher.desc(), line)
            parser.fIn.pushbackLine(line)

        return result_dict

    def handleEndMatch(self, parser):
        line = parser.fIn.readline()
        m = self.endRe.match(line)  # , re.MULTILINE)
        self.handleMatchTelemetry(parser, m, line, 1)
        result_dict = {}
        for k, v in sorted(m.groupdict().items()):
            if v is None:
                # a group may be optional (subexpression of ? or | in regex)
                continue
            k_converted, v_converted = self.addStrValue(parser.backend, k, v)
            result_dict[k_converted] = v_converted
        if self.matcher.fixedEndValues:
            for k, v in sorted(self.matcher.fixedEndValues.items()):
                if k not in result_dict:
                    v_converted = self.addValue(parser.backend, k, v)
                    result_dict[k] = v_converted
        return result_dict

    def findNextWithRe(self, regex, possible, parser):
        # searching for match would be more efficient than doing line by line as here...
        # catch empty regular expression
        if regex.pattern == '':
            return -3
        while True:
            line = parser.fIn.readline()
            if not line:
                return -1
            m = regex.match(line)  # , re.MULTILINE)
            if m:
                # logger.debug("line %d has a match for %d: %s", parser.fIn.lineNr, self.matcher.index, m.groupdict())
                groups = m.groups()
                for i in range(len(groups)):
                    if groups[i] is not None:
                        parser.fIn.pushbackLine(line)
                        targetStartEnd, targetMatcher = possible[i]
                        nextI = 2 * targetMatcher.index + targetStartEnd
                        if m.group("i%s" % nextI) is None:
                            raise Exception(
                                "inconsistent match (group name vs group pos)")
                        # The call to self.annotate needs to be in
                        # handleStartMatch/handleEndMatch, where the
                        # original RE is applied, including all capturing groups
                        # self.annotate(m,line,parser,targetMatcher,targetStartEnd)
                        return nextI
                return -2
            self.handleMatchTelemetry(parser, None, line, 0)
            # logger.debug("line %d %r has no next match for matcher #%d (%s)", parser.fIn.lineNr, line, self.matcher.index, self.possibleNextsRe.pattern)
            parser.skipped += 1

    def findNextMatch(self, parser):
        nextI = None
        if self.matcher.adHoc:
            nextI = self.matcher.adHoc(parser)

        if nextI is not None:
            return nextI
        return self.findNextWithRe(self.possibleNextsRe, self.possibleNexts, parser)

    def findNextMatchEnd(self, parser):
        return self.findNextWithRe(self.possibleNextsEndRe, self.possibleNextsEnd, parser)

    def getMatchTelemetry(self, parser, match, fInLine, targetStartEnd):
        result = {
            # raw line
            'fInLine': fInLine,
            'fInLineNr': parser.fIn.lineNr,
            # information about SimpleMatcher
            'matcherName': self.matcher.name,
            'defFile': self.matcher.defFile,
            'defLine': self.matcher.defLine,
            'matcher_does_nothing': self.matcher.does_nothing,  # matcher without effect
            'which_re': 'end' if targetStartEnd else 'start',
            # classification of match
            'matchFlags': 0,
            'match': 0,  # 0 - no, 1 - partial, 3 - full
            'coverageIgnore': 0,  # 0 - no, 1 - local, 3 - global
            # overall span of match, and spans of group captures
            'span': [],
            # capture group names
            'matcherGroup': [],
        }
        if match:
            span = match.span()
            if span[0] == 0 and span[1] == len(fInLine):
                result['match'] = 3  # full match
            else:
                result['match'] = 1  # partial match
            if self.matcher.coverageIgnore:  # matcher is local coverageIgnore
                result['coverageIgnore'] = 1
            result['matcherGroup'] = self.groupname[targetStartEnd]
            # Forward compatibility with 'regex' or 're2', which support
            # multiple captures per group:
            #   span: list of lists of tuples: [groups, captures, 2]
            result['span'] = [[span]] + list([]
                                             for i in range(match.re.groups))
            # capture groups
            for groupi in range(1, match.re.groups + 1):
                if match.group(groupi) is not None:
                    result['span'][groupi].append(match.span(groupi))
        else:
            m_ci = parser.coverageIgnore.match(
                fInLine)  # check global coverageIgnore
            if m_ci:
                result['coverageIgnore'] = 3
                span = m_ci.span()
                result['span'] = [[span]]
                if span[0] == 0 and span[1] == len(fInLine):
                    result['match'] = 3  # full match
                else:
                    result['match'] = 1  # partial match
                result['matcher_does_nothing'] = True
                result['matcherName'] = 'coverageIgnore'
        # flatten span:
        #   original:
        #     -- whole_re -- group1 -- group2 ------ group3 --------
        #     [ [ (0,25) ], [ (4,5) ],     [],    [ (1,3), (10,15) ], ... ]
        #   flattened:
        #     [ 1, 0, 25,    1, 4, 5,       0,    2, 1, 3, 10, 15, ... ]
        #   per group (first one 'global'/whole RE span):
        #     number_of_captures, start1, end1, ...
        #   this encoding scheme supports "optional" groups (r"(?P<blah>\d+)?")
        #   and is future-compatible with multiple-captures-per-group
        #   supported by modules regex / re2 (r"(?:(?P<blah>\d+)\s+)+")
        span_flat = []
        for g in result['span']:
            span_flat.append(len(g))
            for s in g:
                span_flat.extend(s)
        result['matchSpansFlat'] = span_flat
        result['matchFlags'] = \
            result['match'] | \
            (result['coverageIgnore'] << 2) | \
            targetStartEnd << 5 | \
            int(result['matcher_does_nothing'] and result['match']) << 6
        return result

    def handleMatchTelemetry(self, parser, match, line, targetStartEnd):
        match_telemetry = self.getMatchTelemetry(
            parser, match, line, targetStartEnd)
        parser.updateParsingStats(match_telemetry)
        parser.backend.addMatchTelemetry(match_telemetry)
        parser.annotator.annotate(match_telemetry)


class ParsingContext(object):  # use slots?
    Start = 0
    End = 1

    def __init__(self, index, sections, compiledMatcher, startEnd):
        self.index = index
        self.startEnd = startEnd
        self.sections = sections
        self.compiledMatcher = compiledMatcher


def extractOnCloseTriggers(obj):
    """extracts all triggers from the object obj

    triggers should start with onClose_ and then have a valid section name.
    They will be called with backend, gIndex and the section that is being closed
    (of type CachingSection)"""
    triggers = {}
    for attr in dir(obj):
        if attr.startswith("onClose_"):
            name = attr[8:]
            triggers[name] = getattr(obj, attr)
    return triggers


def extractOnOpenTriggers(obj):
    """extracts all triggers from the object obj

    triggers should start with onOpen_ and then have a valid section name.
    They will be called with backend, gIndex and the section that is being closed
    (of type CachingSection)"""
    triggers = {}
    for attr in dir(obj):
        if attr.startswith("onOpen_"):
            name = attr[7:]
            triggers[name] = getattr(obj, attr)
    return triggers


def extractStrValueTransforms(obj):
    """extracts all string-to-value transformations from obj

    name of transformation method starts with 'strValueTransform_'
    and then have a 'pseudo-unit-name' starting with 'str', which can then be
    referenced as a 'unit' in regex group names.
    Output units of transform method can be specified as function attribute
    'units'
        def strValueTransform_strDate(self.string)
            ....
        strValueTransform_strDate.units = 's'
    would than be used, if you have
        (?P<time_run_date_start__strDate>...)
    in your regex"""
    transform = {}
    for attr in dir(obj):
        match = re.match(r"strValueTransform_(str\S+)", attr)
        if match:
            name = match.group(1)
            callback = getattr(obj, attr)
            units = getattr(callback, 'units', None)
            transform[name] = [callback, units]
    return transform


class SimpleParserBuilder(object):
    def __init__(self, rootMatcher, metaInfoEnv, metaInfoToKeep=None, default_units=None, metainfo_units=None, strValueTransform=None):
        """
        Args:
            metaInfoToKeep: A list of metanames. Any SimpleMatchers (and their
                parents) that handle these metanames are kept in the parsing.
            metainfo_units: An optional dictionary that maps metanames into unit
                definitions. Used to override the default SI conversion.
            default_units: An optional dictionary that maps dimensions (e.g.
                "[energy]") into unit definitions. Used to override the default SI
                conversion.
        """
        self.rootMatcher = rootMatcher
        self.flatIndex = []
        self.compiledMatchers = {}
        self.rootMatcher.clearIndexes()
        self.rootMatcher.calcIndexes(self.flatIndex)
        self.current = None
        self.metaInfoEnv = metaInfoEnv
        self.metaInfoToKeep = metaInfoToKeep
        self.default_units = default_units
        self.metainfo_units = metainfo_units
        self.strValueTransform = strValueTransform

    def verifyMetaInfo(self, outF):
        missingMeta = {}
        missingSections = {}
        for matcher in self.flatIndex:
            for metaName in matcher.directMetaNames():
                if metaName not in self.metaInfoEnv:
                    ms = missingMeta.get(metaName, None)
                    if ms:
                        ms.append(matcher)
                    else:
                        missingMeta[metaName] = [matcher]
            for sectionName in matcher.sections:
                if sectionName not in self.metaInfoEnv:
                    ms = missingSections.get(sectionName, None)
                    if ms:
                        ms.append(matcher)
                    else:
                        missingSections[sectionName] = [matcher]
        for metaName, matchers in missingMeta.items():
            outF.write(
                "Meta info %s is not in the meta info, but is used in the following matchers:\n" % metaName)
            for m in matchers:
                outF.write("  ")
                outF.write(m.desc())
                outF.write("\n")
            outF.write("""If it was not a typo, you might want to add a meta info similar to the following:
  {
    "name": "%s",
    "superNames": [],
    "dtypeStr": "f",
    "shape": []
  }
""" % (metaName))
        for sectionName, matchers in missingSections.items():
            outF.write(
                "A section named %s is not defined in the meta info, but is used in the following matchers:\n" % sectionName)
            for m in matchers:
                outF.write("  ")
                outF.write(m.desc())
                outF.write("\n")
            outF.write("""If it was not a typo, you might want to add a meta info similar to the following:
  {
    "name": "%s",
    "kindStr": "type_section",
    "superNames": []
  }
""" % (sectionName))
        outF.flush()
        return (not (missingSections or missingMeta))

    def handleMatcher(self, matcher, inheritedStealingMatchers, inheritedFloatingMatchers):
        superMatcher = matcher.superMatcher
        firstMatchers = []
        floatingMatchers = []
        firstMatchersEnd = []
        floatingMatchersEnd = []
        first = True
        for m in matcher.subMatchers:
            if first:
                first = False
                if m.floating:
                    floatingMatchers.append((ParsingContext.Start, m))
                else:
                    firstMatchers.append((ParsingContext.Start, m))
            else:
                if m.floating:
                    floatingMatchers.append((ParsingContext.Start, m))
                elif not m.weak:
                    firstMatchers.append((ParsingContext.Start, m))
        if matcher.endReStr:  # move before the sub matchers?
            firstMatchers.append((ParsingContext.End, matcher))
        if matcher.repeats:
            firstMatchers.append((ParsingContext.Start, matcher))
            firstMatchersEnd.append((ParsingContext.Start, matcher))
        if superMatcher is not None:
            subFlags = superMatcher.subFlags
            if subFlags == SimpleMatcher.SubFlags.Sequenced:
                found = 0
                for m in superMatcher.subMatchers:
                    if m == matcher:
                        found = 1
                    elif m.floating:
                        floatingMatchers.append((ParsingContext.Start, m))
                        floatingMatchersEnd.append((ParsingContext.Start, m))
                        if found == 1:
                            found += 1
                    elif found == 1:
                        if not m.weak or not matcher.subMatchers:
                            firstMatchers.append((ParsingContext.Start, m))
                        firstMatchersEnd.append((ParsingContext.Start, m))
                        found += 1
                    elif found > 0:
                        if not m.weak:
                            firstMatchers.append((ParsingContext.Start, m))
                            firstMatchersEnd.append((ParsingContext.Start, m))
            elif subFlags == SimpleMatcher.SubFlags.Unordered:
                for m in superMatcher.subMatchers:
                    if m == matcher:
                        continue
                    if m.weak:
                        raise Exception(
                            "Weak not supported in unordered superMatcher for " + m.desc())
                    if m.floating:
                        floatingMatchers.append((ParsingContext.Start, m))
                        floatingMatchersEnd.append((ParsingContext.Start, m))
                    else:
                        firstMatchers.append((ParsingContext.Start, m))
                        firstMatchersEnd.append((ParsingContext.Start, m))
            else:
                raise Exception("unknown subFlags %d" % subFlags)
        logger.debug("firstMatchers: %s", [
                     (x[0], x[1].index) for x in firstMatchers])
        logger.debug("firstMatchersEnd %s", [
                     (x[0], x[1].index) for x in firstMatchersEnd])
        logger.debug("inheritedStealingMatchers\n %s", [
                     (x[0], x[1].index) for x in inheritedStealingMatchers])
        logger.debug("floatingMatchers %s", [
                     (x[0], x[1].index) for x in floatingMatchers])
        logger.debug("floatingMatchersEnd %s", [
                     (x[0], x[1].index) for x in floatingMatchersEnd])
        logger.debug("inheritedFloatingMatchers %s", [
                     (x[0], x[1].index) for x in inheritedFloatingMatchers])
        self.compiledMatchers[matcher.index] = CompiledMatcher(
            matcher=matcher,
            possibleNexts=firstMatchers + inheritedStealingMatchers + floatingMatchers + inheritedFloatingMatchers,
            possibleNextsEnd=firstMatchersEnd + inheritedStealingMatchers + floatingMatchersEnd + inheritedFloatingMatchers,
            parserBuilder=self)

    def scanMatcher(self, matcher, inheritedStealingMatchers, inheritedFloatingMatchers):
        self.handleMatcher(matcher, inheritedStealingMatchers,
                           inheritedFloatingMatchers)
        superMatcher = matcher.superMatcher
        firstMatchers = []
        floatingMatchers = []
        if matcher.endReStr:
            firstMatchers.append((ParsingContext.End, matcher))
        if matcher.repeats:
            firstMatchers.append((ParsingContext.Start, matcher))
        if superMatcher and matcher.subMatchers:
            subFlags = superMatcher.subFlags
            if subFlags == SimpleMatcher.SubFlags.Sequenced:
                found = False
                for m in superMatcher.subMatchers:
                    if m == matcher:
                        found = True
                    elif m.floating and not m.weak:
                        floatingMatchers.append((ParsingContext.Start, m))
                    elif found and not m.weak:
                        firstMatchers.append((ParsingContext.Start, m))
            elif subFlags == SimpleMatcher.SubFlags.Unordered:
                for m in superMatcher.subMatchers:
                    if m == matcher:
                        continue
                    if m.weak:
                        raise Exception(
                            "Weak not supported in unordered superMatcher for " + m.desc())
                    if m.floating:
                        floatingMatchers.append((ParsingContext.Start, m))
                    else:
                        firstMatchers.append((ParsingContext.Start, m))
            firstMatchers.extend(inheritedStealingMatchers)
            floatingMatchers.extend(inheritedFloatingMatchers)
        for m in matcher.subMatchers:
            self.scanMatcher(m, firstMatchers, floatingMatchers)

    def compile(self):
        # Try to optimize the parsing process
        self.optimizeParsingTree()

        # Compile the remaining tree
        self.scanMatcher(self.rootMatcher, [], [])

    def optimizeParsingTree(self):
        """If the ParserBuilder has been given additional information regarding
        the parsing (metaInfoToKeep, metaInfoToSkip), this function will prune
        the parsing tree so that only the relevant information is parsed.
        """
        # Check if only certain metainfo is wanted. If so, remove unnecessary
        # items from the parsing.
        if self.metaInfoToKeep:
            logger.info("Optimizing parsing tree...")
            optimizer = ParserOptimizer()
            optimizer.optimizeParsingTree(
                self.rootMatcher, self.metaInfoToKeep)
            # self.rootMatcher.pruneParsingTree(self.metaInfoToKeep)

        # If logging is activated, print out the pruned tree for debugging
        # purposes
        if logger.isEnabledFor(logging.DEBUG):
            logger.info("Parsing tree after optimization:")
            self.rootMatcher.printParsingTree()

    def buildParser(self, fIn, backend, superContext, baseUri, basePath):
        return SimpleParser(self, fIn, backend, superContext, baseUri, basePath)

    def writeMatchers(self, outF, extraIndent=0):
        outF.write("[")
        for i in range(len(self.flatIndex)):
            if i != 0:
                outF.write(", ")
            jsonIndentF(self.flatIndex[i].toDict(),
                        outF, extraIndent=extraIndent + 2)
        outF.write("\n")
        outF.write(" " * extraIndent)
        outF.write("]")

    def writeCompiledMatchers(self, outF, extraIndent=0):
        outF.write("{")
        keys = list(self.compiledMatchers.keys())
        keys.sort()
        first = True
        for i in keys:
            if first:
                first = False
            else:
                outF.write(", ")
            jsonIndentF(str(i), outF)
            outF.write(":")
            jsonIndentF(self.compiledMatchers[i].toDict(
            ), outF, extraIndent=extraIndent + 2)
        outF.write("\n")
        outF.write(" " * extraIndent)
        outF.write("}")


class SimpleParser(object):
    def __init__(self, parserBuilder, fIn, backend, superContext=None, baseUri=None, basePath=None):
        self.parserBuilder = parserBuilder
        self.fIn = fIn
        self.backend = backend
        self.context = []
        self.skipped = 0
        self.superContext = superContext
        self.lastMatch = {}
        self.baseUri = baseUri
        self.basePath = basePath
        if backend.sectionManagers is not None:
            r = backend.sectionManagers.get("section_run")
            if r:
                r.onOpen.append(self.emitBaseUri)

        annofilename = None
        # by default, ignore empty lines in coverage analysis
        self.coverageIgnore = getattr(superContext, 'coverageIgnore',
                                      re.compile(r"^\s*$"))
        self.parsingStats = {}
        for counter in ['total', 'ignore', 'match', 'partial', 'unmatched']:
            self.parsingStats[counter] = 0
        if annotate:
            annofilename = fIn.fIn.name + ".annotate"
        else:
            logger.info("no annotations requested for " + fIn.fIn.name)
        self.annotator = Annotator(annotateFilename=annofilename)
        self.enterInState(0)
        compiledRootMatcher = self.parserBuilder.compiledMatchers[0]
        if compiledRootMatcher.matcher.fixedStartValues:
            for k, v in sorted(compiledRootMatcher.matcher.fixedStartValues.items()):
                compiledRootMatcher.addValue(backend, k, v)

    def uriForPath(self, path):
        """return the uri corresponding to the given path"""
        if self.baseUri and self.basePath and path:
            return uriFromRelPath(self.baseUri, self.basePath, path)
        return path

    def emitBaseUri(self, backend, gIndex, section):
        """writes out the uri to the sources"""
        if self.baseUri:
            backend.addValue("calculation_file_uri", self.baseUri)

    def enterInState(self, stateIndex):
        compiledMatcher = self.parserBuilder.compiledMatchers[stateIndex]
        sects = OrderedDict()

        for s in compiledMatcher.matcher.sections:
            gIndex = self.backend.openSection(s)
            sects[s] = gIndex

            # Call the matcher specific onOpen callbacks
            if compiledMatcher.matcher.onOpen:
                callback = compiledMatcher.matcher.onOpen.get(s)
                if callback:
                    section_manager = self.backend.sectionManagers[s]
                    section = section_manager.openSections[gIndex]
                    callback(self.backend, gIndex, section)

        # call the not section-bound onOpen function of the specific matcher
        if compiledMatcher.matcher.onOpen:
            callback = compiledMatcher.matcher.onOpen.get(None)
            if callback:
                callback(self.backend, None, None)

        self.context.append(ParsingContext(
            stateIndex, sects, compiledMatcher, ParsingContext.Start))

    def contextClose(self, cNow):

        if cNow.startEnd == ParsingContext.Start:
            # call the not section-bound onClose function of the specific matcher
            if cNow.compiledMatcher.matcher.onClose:
                callback = cNow.compiledMatcher.matcher.onClose.get(None)
                if callback:
                    callback(self.backend, None, None)

            for s, gIndex in reversed(list(cNow.sections.items())):

                # Call the matcher specific onClose callbacks
                if cNow.compiledMatcher.matcher.onClose:
                    callback = cNow.compiledMatcher.matcher.onClose.get(s)
                    if callback:
                        section_manager = self.backend.sectionManagers[s]
                        section = section_manager.openSections[gIndex]
                        callback(self.backend, gIndex, section)

                # Call the global onClose callbacks
                self.backend.closeSection(s, gIndex)

            cNow.startEnd = ParsingContext.End
            return True
        else:
            return False

    def contextPop(self):
        cNow = self.context.pop()
        self.contextClose(cNow)

    def contextDesc(self):
        return str(["#%d %s" % (c.index, ["start", "end"][c.startEnd]) for c in self.context])

    def goToMatcher(self, targetMatcher, startEnd):
        logger.debug("goToMatcher(#%d, %s)", targetMatcher.index,
                     ["start", "end"][startEnd])
        states = set(c.index for c in self.context)
        targetStack = []
        mNow = targetMatcher
        while (mNow and mNow.index not in states):
            targetStack.append(mNow)
            mNow = mNow.superMatcher
        if not mNow:
            raise Exception("No common matcher between the target %s and the current context %s" % (
                targetMatcher.desc(), self.contextDesc()))
        states = list(c.index for c in self.context)
        commonIndex = mNow.index
        idx = states.index(commonIndex)
        if mNow == targetMatcher:
            if startEnd == ParsingContext.Start:
                targetStack.append(mNow)
                idx -= 1
        elif self.context[len(self.context) - 1].compiledMatcher.floating and self.parserBuilder.compiledMatchers[targetMatcher.index].floating:
            # we are in a floating match and we go to another one, rewind only the floating part
            while not self.context[idx].compiledMatcher.floating:
                idx += 1
            idx -= 1
        # close
        while len(self.context) > idx + 1:
            self.contextPop()
        # does not close yet targetMatcher is startEnd == Close, should be done outside after
        # and reopen
        while targetStack:
            tMatcher = targetStack.pop()
            self.enterInState(tMatcher.index)

    def parseStep(self):
        if not self.context:
            return False
        currentContext = self.context[len(self.context) - 1]
        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("lineNr: %d context: %s",
                         self.fIn.lineNr, self.contextDesc())
        if currentContext.startEnd == ParsingContext.Start:
            nextI = currentContext.compiledMatcher.findNextMatch(self)
        else:
            nextI = currentContext.compiledMatcher.findNextMatchEnd(self)
        logger.debug("lineNr: %d nextI: %d", self.fIn.lineNr, nextI)

        if nextI < 0:
            cNames = self.contextDesc()
            while self.context:
                self.contextPop()
            if nextI != -1 and nextI != -3:
                raise Exception(
                    "finished with error with parsing context %s" % (cNames))
        else:
            index = nextI // 2
            startEnd = nextI % 2
            matcherNew = self.parserBuilder.flatIndex[index]
            self.goToMatcher(matcherNew, startEnd)
            logger.debug("new context: %s\n", self.contextDesc())
            currentCtx = self.context[len(self.context) - 1]
            if startEnd == ParsingContext.End:
                self.lastMatch = currentCtx.compiledMatcher.handleEndMatch(
                    self)
                self.contextClose(currentCtx)
            else:
                self.lastMatch = currentCtx.compiledMatcher.handleStartMatch(
                    self)
        return len(self.context) > 0

    def parse(self):
        while self.parseStep():
            pass

    def updateParsingStats(self, matchTelemetry):
        # update counters
        self.parsingStats['total'] += 1
        if matchTelemetry['coverageIgnore']:
            self.parsingStats['ignore'] += 1
        elif matchTelemetry['match'] > 1:
            self.parsingStats['match'] += 1
        elif matchTelemetry['match']:
            self.parsingStats['partial'] += 1
        else:
            self.parsingStats['unmatched'] += 1


# This cache only works if the simpleParser (root of the SM tree) is not recreated
# on each parse
_compile_parser_cache = {}


def compileParser(simpleParser, metaInfo, metaInfoToKeep, default_units=None, metainfo_units=None, strValueTransform=None):
    """compiles the given simple parser"""
    if simpleParser not in _compile_parser_cache:
        parserBuilder = SimpleParserBuilder(
            simpleParser, metaInfo, metaInfoToKeep, default_units, metainfo_units, strValueTransform)
        if logger.isEnabledFor(logging.DEBUG):
            s = io.StringIO()
            s.write("matchers:")
            parserBuilder.writeMatchers(s, 2)
            logger.debug(s.getvalue())
        if not parserBuilder.verifyMetaInfo(sys.stderr):
            sys.exit(1)
        parserBuilder.compile()
        if logger.isEnabledFor(logging.DEBUG):
            s = io.StringIO()
            s.write("compiledMatchers:")
            parserBuilder.writeCompiledMatchers(s, 2)
            logger.debug(s.getvalue())

        _compile_parser_cache[simpleParser] = parserBuilder

    return _compile_parser_cache[simpleParser]


def runParser(compiledParser, backend, superContext, fIn, uri, path):
    """parses the open file fIn with the given compiledParser into the backend using superContext as parser SuperContext"""
    parser = compiledParser.buildParser(PushbackLineFile(
        fIn), backend, superContext=superContext, baseUri=uri, basePath=path)
    try:
        superContext.startedParsing(fIn.name, parser)
    except AttributeError:
        logging.exception("problem calling superContext.startedParsing")
        pass
    parser.parse()
    return parser.parsingStats


def defaultParseFile(parserInfo):
    def parseF(parserBuilder, uri, path, backend, superContext):
        with open(path, "r") as fIn:
            backend.startedParsingSession(uri, parserInfo)
            try:
                parsingStats = runParser(
                    parserBuilder, backend, superContext, fIn, uri, path)
                backend.finishedParsingSession(
                    "ParseSuccess", None,
                    parsingStats=parsingStats)
            except SkipFileException:
                # FIXME: close all open sections here instead of relying on the caller doing it
                backend.finishedParsingSession(
                    "ParseSkipped", None,
                    parsingStats={})
    return parseF


def mainFunction(mainFileDescription,
                 metaInfoEnv,
                 parserInfo,
                 parseFile=None,
                 outF=None,
                 cachingLevelForMetaName=None,
                 defaultDataCachingLevel=CachingLevel.ForwardAndCache,
                 defaultSectionCachingLevel=CachingLevel.Forward,
                 superContext=None,
                 onClose=None,
                 onOpen=None,
                 default_units=None,
                 metainfo_units=None,
                 superBackend=None,
                 metaInfoToKeep=None,
                 mainFile=None,
                 strValueTransform=None):
    """
    Args:
        default_units: A list of default unit definitions.
            E.g. ["eV", "Bohr"]. Used to override the default conversion to SI
            units. Leave as 'None' if no overriding necessary.
        metainfo_units: A mapping between metainfo names and unit definitions.
            E.g. {"energy_total": "eV"}. Used to override the default conversion to
            SI units. Leave as 'None' if no overriding necessary.
        super_backend: Backend to use for the parsing. This backend will be
            called by the ActiveBackend that handles the caching. If not
            specified, the default JsonParseEventsWriterBackend is used.
        mainFile: The path to the file that is parsed. If not specified,
            reads the filename from the command line arguments.
    """
    usage = """{exeName} [--annotate] [--matchTelemetry] [--meta-info] [--help] [--specialize] [--stream] [--uri uri] [--verbose] [path/toFile]

    --annotate writes annotated debug info of parsing process to file path/toFile.annotate
    --help prints this message
    --verbose writes metainfo to stderr and detailed debug info of parsing process to file detailed.log

    If a path to a file is given this is parsed
""".format(exeName=os.path.basename(sys.argv[0] if len(sys.argv) > 0 else "simple_parser"))
    if parseFile is None:
        parseFile = defaultParseFile(parserInfo)

    uri = None
    verbose = False
    fileToParse = None

    if cachingLevelForMetaName is None:
        cachingLevelForMetaName = {}
    if onClose is None:
        onClose = {}
    if onOpen is None:
        onOpen = {}
    if strValueTransform is None:
        strValueTransform = {}

    ii = 1
    while ii < len(sys.argv):
        arg = sys.argv[ii]
        ii += 1
        if arg == "--help":
            sys.stderr.write(usage)
            sys.exit(0)
        elif arg == "--verbose":
            verbose = True
        elif arg == "--annotate":
            global annotate
            annotate = True
        elif arg == "--uri":
            if ii >= len(sys.argv):
                raise Exception("missing uri after --uri")
            uri = sys.argv[ii]
            ii += 1
        elif fileToParse is None:
            fileToParse = arg
        else:
            sys.stderr.write("unexpected argument " + arg + "\n")
            sys.stderr.write(usage)
            sys.exit(1)

    if metaInfoEnv is None:
        metaInfoEnv = superBackend.metaInfoEnv()

    # If a main file has been specified, override the one given as a command
    # line argument
    if mainFile is not None:
        fileToParse = mainFile

    if verbose:
        import nomadcore.ActivateLogging
        nomadcore.ActivateLogging.debugToFile()

    if uri is None and fileToParse:
        uri = "file://" + fileToParse

    assert superBackend is not None
    assert fileToParse is not None

    if superContext:
        strValueTransform = dict(strValueTransform)
        for attr, callback in extractStrValueTransforms(superContext).items():
            oldCallbacks = strValueTransform.get(attr, None)
            if attr not in strValueTransform:
                strValueTransform[attr] = callback

    parserBuilder = compileParser(
        mainFileDescription, metaInfoEnv, metaInfoToKeep, default_units, metainfo_units,
        strValueTransform)

    if superContext:
        onClose = dict(onClose)
        for attr, callback in extractOnCloseTriggers(superContext).items():
            oldCallbacks = onClose.get(attr, None)
            if oldCallbacks:
                oldCallbacks.append(callback)
            else:
                onClose[attr] = [callback]
        onOpen = dict(onOpen)
        for attr, callback in extractOnOpenTriggers(superContext).items():
            oldCallbacks = onOpen.get(attr, None)
            if oldCallbacks:
                oldCallbacks.append(callback)
            else:
                onOpen[attr] = [callback]

    backend = ActiveBackend.activeBackend(
        metaInfoEnv=metaInfoEnv,
        cachingLevelForMetaName=cachingLevelForMetaName,
        defaultDataCachingLevel=defaultDataCachingLevel,
        defaultSectionCachingLevel=defaultSectionCachingLevel,
        onClose=onClose,
        onOpen=onOpen,
        superBackend=superBackend,
        default_units=default_units,
        metainfo_units=metainfo_units)

    parseFile(parserBuilder, uri, fileToParse, backend, superContext)


class AncillaryParser(object):
    """This is an ancillary parser that can be used inside a running parser to parse external files.

    It should be instantiated in the startedParsing function of the ParserContext class
    where the main parser is received.
    The same backend is used as for the main parser, therefore, the output of the ancillary
    parser is included in the output of the the main parser.
    Be careful when you open sections with the ancillary parser. Be sure not to write the opening events for sections
    that are already opened by the main parser. You can use cachingLevelForMetaName to set a section to Ignore or Cache.
    Thus, you can still use sections and their onClose triggers, but no opening events are written.
    """

    def __init__(self, fileDescription, parser, cachingLevelForMetaName, superContext):
        """Compiles parser and set up backend.

        Args:
            fileDescription: SimpleMatcher that will be used to parse.
            cachingLevelForMetaName: Dictinonary to set the caching level for the metadata.
            parser: The mains parser that is already running. It is used to get the current metadata and backend.
            superContext: Context for the ancillary parser.
        """
        self.mainParser = parser
        default_units = parser.parserBuilder.default_units
        metainfo_units = parser.parserBuilder.metainfo_units
        # compile parser
        self.compiledParser = compileParser(fileDescription, parser.parserBuilder.metaInfoEnv,
                                            parser.parserBuilder.metaInfoToKeep, default_units, metainfo_units)
        # save superContext
        self.superContext = superContext
        # extract onClose functions from superContext
        onClose = {}
        for attr, callback in extractOnCloseTriggers(superContext).items():
            onClose[attr] = [callback]
        onOpen = {}
        for attr, callback in extractOnOpenTriggers(superContext).items():
            onOpen[attr] = [callback]
        # create backend for parser
        self.backend = ActiveBackend.activeBackend(
            metaInfoEnv=parser.parserBuilder.metaInfoEnv,
            cachingLevelForMetaName=cachingLevelForMetaName,
            onClose=onClose,
            onOpen=onOpen,
            superBackend=parser.backend.superBackend,
            # write no parser info upon start and end of parsing
            propagateStartFinishParsing=False,
            default_units=default_units,
            metainfo_units=metainfo_units)

    def parseFile(self, fIn):
        """Parses file.

        Args:
            fIn: File that is parsed.
        """
        currentUri = self.mainParser.uriForPath(fIn.name)
        if currentUri:
            self.backend.superBackend.addValue(
                "calculation_file_uri", currentUri)
        runParser(self.compiledParser, self.backend, self.superContext, PushbackLineFile(
            fIn), self.mainParser.baseUri, self.mainParser.basePath)


class ParserOptimizer(object):
    """For optimizing a hierarchy of SimpleMatchers based on a list of metainfo
    names that should be included in the parsing.
    """

    def optimizeParsingTree(self, rootMatcher, metaInfoToKeep=None):
        """This function will remove any parsing unnecessary parsing actions
        from the parsing tree based on the given list of metainfo names to keep
        and metainfo names to skip.

        Args:
            rootMatcher: A SimpleMatcher object that is the root of a parsing
                hierarchy.
            metaInfoToKeep: List of metainfonames to keep in parsing
        """
        # First identify the parts that should be kept
        if metaInfoToKeep is None:
            metaInfoToKeep = []

        if metaInfoToKeep:

            # Single values are put inside a list for convenience
            if not isinstance(metaInfoToKeep, list):
                metaInfoToKeep = [metaInfoToKeep]

            # Search the matchers that are in the keep list
            parseList = []
            leaves = []
            matchDict = {}
            self.searchMatchers(rootMatcher, metaInfoToKeep,
                                parseList, leaves, matchDict)

            # If something was not matched, prompt the user
            for name, found in matchDict.items():
                if not found:
                    logger.warning("Could not find any matches for '{}'. Check that it exists in the SimpleMatcher tree. Remember to declare items that are manually placed into backend with the SimpleMatcher attribute 'otherMetaInfo'".format(name))

            # Follow the parents of the parsed matchers towards the root and mark
            # them parsed
            for matcher in parseList:
                self.keepParents(matcher)

            # Cut the empty branches
            for leaf in leaves:
                logger.debug("LEAF: {}".format(leaf.startReStr))
                self.tryToRemoveFromParsingTree(leaf)

    def searchMatchers(self, simpleMatcher, metaNames, parseList, unParsedleaves, matchDict):
        """Walk through the parsing tree and mark the matchers that are in the
        given list of metanames.

        Args:
            simpleMatcher: The simplematcher that is the target of the operation
            metanames: List of metanames to search for
            parseList: List where the found matchers will be placed
            leaves: List where the leaves, i.e. the submatchers with no
                children in the parsing tree will be placed.
        """
        # Initialize a dictionary that keeps track of the found matches
        if len(matchDict) == 0:
            for item in metaNames:
                matchDict[item] = False

        # Collect all the defined sections, otherMetanames and metanames inside
        # regexps into one list
        myMetanames = list(simpleMatcher.sections)
        myMetanames.extend(simpleMatcher.directMetaNames())

        # See if this matcher has metainfos of interest and add any possible
        # dependencies to the list of metainfos to keep
        found = False
        for _, name in enumerate(metaNames):
            if name in myMetanames:
                found = True
                matchDict[name] = True
                dependencies = simpleMatcher.dependencies.get(name)
                logger.debug("Found matcher: {}".format(
                    simpleMatcher.startReStr))
                simpleMatcher.keep = True
                parseList.append(simpleMatcher)

                # If dependencies stated, include them into the search
                if dependencies:
                    metaNames.extend(dependencies)
                # If no dependencies stated, include the whole subtree
                else:
                    self.keepChildren(simpleMatcher)
                    return
                break

        # Check the children
        for submatcher in simpleMatcher.subMatchers:
            self.searchMatchers(submatcher, metaNames,
                                parseList, unParsedleaves, matchDict)

        # Save non-parsed leaves
        if not simpleMatcher.subMatchers and not found:
            unParsedleaves.append(simpleMatcher)

    def keepParents(self, simpleMatcher):
        """Called on a matcher that should be parsed. Starts to recursively
        go through the tree marking the parent matchers as parsed until root is
        reached.

        Args:
            simpleMatcher: The simplematcher that is the target of the operation
        """
        parent = simpleMatcher.superMatcher
        if not parent:
            return
        keepParent = parent.keep
        if keepParent:
            logger.debug("PARENT KEPT:" + simpleMatcher.superMatcher.startReStr)
            return
        else:
            logger.debug("KEEPING PARENT:" + simpleMatcher.superMatcher.startReStr)
            simpleMatcher.superMatcher.keep = True
            self.keepParents(simpleMatcher.superMatcher)

    def keepChildren(self, simpleMatcher):
        """
        Args:
            simpleMatcher: The simplematcher that is the target of the operation
        """
        for child in simpleMatcher.subMatchers:
            child.keep = True
            self.keepChildren(child)

    def tryToRemoveFromParsingTree(self, simpleMatcher):
        """Tries to remove the given simpleMatcher from the parsing tree. First
        checks if the parent should be kept in parsing. If yes, removes this
        object. If not, climbs up the root towards the root by calling this
        function recursively.

        Args:
            simpleMatcher: The simplematcher that is the target of the operation
        """
        if simpleMatcher.keep:
            raise Exception(
                "Trying to remove a matcher that should not be removed from tree.")

        parent = simpleMatcher.superMatcher
        if not parent:
            return

        keepParent = parent.keep
        if keepParent:
            logger.debug("PARENT PARSED: {}".format(parent.startReStr))
            parent = simpleMatcher.superMatcher
            children = parent.subMatchers
            for i_child, child in enumerate(children):
                if child == simpleMatcher:
                    logger.debug("REMOVED: {}".format(
                        simpleMatcher.startReStr))
                    del children[i_child]
                    return
        else:
            logger.debug("PARENT NOT PARSED: {}".format(parent.startReStr))
            self.tryToRemoveFromParsingTree(parent)
