"""
Module defines a few concepts.

1. `TransformationSpec`: this is a specification that executes on a
    working directory. It defines what text and files the solver
    has access to and how it is allowed to manipulate the working directory.
    A spec is made up of several "statements" (StsStatement). Each statement contains
    some selectors which narrow down the statement to parts of code,
    and then a verb which is applied on each of those parts of code.
2. "marking up": A marked up version of of a file is an xml-like version
    of a file. It is the result of applying the `TransformationSpec`
    to the file. This represented as a tree of `StsMarkupNode`s. For example a
    simplified example might be
    ```xml
    <StsNormalText>
    def foo(x):
    <StsInsert/>
    </StsNormalText>
    ```
    representing a method completion problem.
3. `markup_path()`: This function will execute a `TransformationSpec` on
    a file and return the marked up version of the file.

The design decouples the contents of a working directory from the specification
from the actual content of the files. An alternative design might not have
had the concept of a `TransformationSpec` and instead just stored a working
directory filled with already marked up files (so basically just make it all
a bunch of xml-like files). I chose the decoupled design for two reasons:

1. We might guess that most files in a project will be untouched and possibly
   reused in multiple problems. Treating the spec as program that executes on a
   working directory allows the code problem to hopefully store minimally unique
   information and multiple code problems can share the same working directory.
2. Might allow more complex specifications (like allowing the solver to generate
   new files, or make queries to some execution or analysis system). These
   might be hard to specify in the "working directory of xml-like files" design.

However, it definitely does add extra complexity. It is not clear how much
point (1) really matters, as expect we will end up eventually caching a lot
of the markedup files in the future to avoid reexecuting the spec each time.
Depending on what we find, we might simplify the design future. However,
this design hopefully gives us a lot of expressivity as needed.
"""

import dataclasses
import json
import re
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import Generator, Iterator, Sequence
from dataclasses import dataclass, field
from functools import cached_property
from re import Pattern
from typing import (
    Any,
    Generic,
    Optional,
    TypeVar,
    Union,
)
from xml.sax import saxutils

import numpy as np
import xxhash
from typeguard import typechecked

from synthegrator.environments import ProjectWorkingDirectory
from synthegrator.lang_specs.lang_spec import LangSpec, lang_spec_for_path
from synthegrator.lang_specs.lang_spec_python import PythonLangSpec
from synthegrator.memory_fs import ProjectDir
from synthegrator.util import hash_string_base62, normalize_probs_dict

MARK_KEY_TYPE = str


@typechecked
def markup_path(
    wd: ProjectWorkingDirectory,
    path: str,
    spec: "TransformationSpec",
    encoding: str = "utf-8",
) -> "MarkElement":
    """
    Executes a `TransformationSpec` on a file in a working directory.
    Any selectors that apply to the file will be performed on it
    """
    content = wd.files[path].content
    content = content.decode(encoding)
    node = MarkElement(
        verb=StsNormalText(),
        children=[MarkText(content, 0, len(content))],
        start_char=0,
        end_char=len(content),
    )
    if spec is None:
        return node
    for stmt in spec.statements:
        path_selector = stmt.path_selector
        if (
            path_selector is not None  # None path selectors match everything
            and not path_selector.does_match_path(path)
        ):
            continue
        text_selector = stmt.text_selector
        if text_selector is None:
            spans = [(0, len(content))]
        else:
            spans = text_selector.get_matching_spans(content, path)
        for start, end in spans:
            node.insert_node(stmt.verb, start, end)
    return node.freeze()


@typechecked
def get_mark_element(
    spec: "TransformationSpec",
    path: str,
    wd: ProjectWorkingDirectory,
    mark_id: MARK_KEY_TYPE,
) -> Union["MarkElement", None]:
    marked_up = markup_path(
        wd=wd,
        path=path,
        spec=spec,
    )
    if marked_up is None:
        return None
    for node in marked_up.depth_first_iterate():
        if isinstance(node, MarkElement) and node.mark_id == mark_id:
            return node
    return None


regex_all = re.compile(r".+")


def visible_lines_is_editable(
    markedup_text: "MarkElement",
) -> np.ndarray:
    is_editable_str_parts = []
    # Our approach will be to construct a new line where all
    # the non-new-line spots that are editable are marked with a '1' and
    # all the spots that are not editable are marked with a '0'.
    # We will then split the string on newlines and convert the strings
    # to ints and threshold them to get the final array.
    # This is just an approach to handle some wackyness where
    #   there is a mix of editable and non-editable on a line
    for state in iterate_parser_states_for_node(markedup_text):
        node = state.node
        if not state.is_visible:
            continue
        if isinstance(node, MarkText):
            lines = node.text.splitlines(keepends=True)
            val = "1" if state.is_editable else "0"
            is_editable_str_parts.append(val)
            if len(lines) > 1:
                for _l in lines[1:]:
                    is_editable_str_parts.append("\n")
                    is_editable_str_parts.append(val)
    joined = "".join(is_editable_str_parts)
    return np.array([int(x) > 0 for x in joined.splitlines(keepends=False)])


def get_verbs_per_path(
    wd: ProjectWorkingDirectory,
    spec: "TransformationSpec",
) -> dict[str, list["StsVerb"]]:
    """
    A helper method that returns a mapping from paths to all the verbs
    used in the path
    """
    paths: dict[str, list[StsVerb]] = defaultdict(list)
    files = wd.files
    for stmt in spec:
        if stmt.path_selector is None:
            for path, _node in files.walk(include_dirs=False):
                paths[path].append(stmt.verb)
        else:
            for path in stmt.path_selector.yield_matching_paths(wd.files):
                paths[path].append(stmt.verb)
    return paths


def map_paths_to_path_ids(
    wd: ProjectWorkingDirectory,
    spec: "TransformationSpec",
) -> dict[str, str]:
    all_paths = list(get_verbs_per_path(wd, spec).keys())
    all_paths.sort()
    return {path: hex(i)[2:] for i, path in enumerate(all_paths)}


"""
### TransformationSpec
"""


class TransformationSpec:
    """
    A transformation spec acts like a little program that specifies ways that
    code can be transformed by a solver in order solve a problem.
    """

    def __init__(self, statements: tuple["StsSpecStatement", ...]):
        self._statements = statements
        self._verify_all_static_marks_unique()

    def _verify_all_static_marks_unique(self):
        static_marks = {}
        for stmt in self._statements:
            if stmt.operation_is_static():
                mark_id = stmt.static_mark_id
                if mark_id in static_marks:
                    msg = "Static verbs must currently all be unique"
                    raise ValueError(msg)
                static_marks[mark_id] = True

    def __iter__(self):
        return iter(self.statements)

    @property
    def statements(self):
        return self._statements

    def editing_statements(self) -> Generator["StsSpecStatement", Any, None]:
        for stmt in self.statements:
            if isinstance(stmt.verb, StsPlaceTransforming):
                yield stmt

    def count_editing_statements(self) -> int:
        return len(list(self.editing_statements()))

    def question_statements(self) -> Generator["StsSpecStatement", Any, None]:
        for statement in self.statements:
            if isinstance(statement.verb, StsValueQuery):
                yield statement

    def classification_question_statements(
        self,
    ) -> Generator["StsSpecStatement", Any, None]:
        for statement in self.question_statements():
            if isinstance(statement.verb, StsCodeClassificationQuestion):
                yield statement

    def __eq__(self, other):
        if not isinstance(other, TransformationSpec):
            return False
        return self.statements == other.statements

    def __repr__(self):
        return f"TransformationSpec(statements={self.statements})"


@dataclass(frozen=True)
class StsSpecStatement:
    """One statement in a `TransformationSpec` program"""

    path_selector: Union["StsPathSelector", None, str]
    """Selects paths to apply to the statement too.
    If None, applies to all files (not directories). If it is a string,
    then it will automatically be converted to a `StsPath` selector."""
    text_selector: Union["StsTextSelector", None]
    """Selects text in the file to apply to the statement too. If None, then
    the entire file will be selected."""
    verb: "StsVerb"
    """The verb that gets applied to the selected text"""

    def __post_init__(self):
        if isinstance(self.path_selector, str):
            object.__setattr__(
                self,
                "path_selector",
                StsPath(self.path_selector),
            )

    def attempt_to_get_path_as_exact_path(self):
        if isinstance(self.path_selector, StsPath):
            if self.path_selector.is_exact_path():
                return self.path_selector.path_glob
            msg = "Path selector is not an exact path"
            raise ValueError(msg)
        msg = "unknown handled path"
        raise ValueError(msg)

    def operation_is_static(self) -> bool:
        return self.verb.operation_is_static()

    @property
    def static_mark_id(self) -> str:
        return self.verb.static_mark_id


@dataclass(frozen=True)
class StsSelector:
    pass


@dataclass(frozen=True)
class StsPathSelector(StsSelector, ABC):
    @abstractmethod
    def does_match_path(self, path: str) -> bool:
        raise NotImplementedError

    def yield_matching_paths(self, project_dir: ProjectDir) -> Iterator[str]:
        for path, _node in project_dir.walk():
            if self.does_match_path(path):
                yield path


@dataclass(frozen=True)
class StsPath(StsPathSelector):
    """A path selector that matches on a given glob (recursive blobs not fully implemented)"""

    path_glob: str

    @cached_property
    def regex(self) -> Pattern:
        val = self.path_glob.replace("**", ".*")
        val = val.replace("*", "[^/]*")
        val = val.replace("?", "[^/]")
        val = f"^{val}$"
        return re.compile(val)

    def does_match_path(self, path: str) -> bool:
        """Checks to see if the path matches performing glob expansion"""
        return self.regex.fullmatch(path) is not None

    def is_exact_path(self) -> bool:
        return "*" not in self.path_glob and "?" not in self.path_glob

    def yield_matching_paths(self, project_dir: ProjectDir) -> Iterator[str]:
        yield from project_dir.expand_glob(self.path_glob)


@dataclass(frozen=True)
class StsTextSelector(StsSelector):
    def get_matching_spans(
        self,
        text: str,
        path: str,
    ) -> Sequence[tuple[int, int]]:
        raise NotImplementedError


@dataclass(frozen=True)
class StsSelectorCharSpan(StsTextSelector):
    """Selects a fixed span of characters in the text"""

    start_char: int
    end_char: int

    def get_matching_spans(
        self,
        text: str,
        path: str,
    ) -> Sequence[tuple[int, int]]:
        return [(self.start_char, self.end_char)]


@dataclass(frozen=True)
class StsPrefixCommentSelector(StsTextSelector):
    """Selects the lines corresponding to prefix comments"""

    def get_matching_spans(
        self,
        text: str,
        path: str,
    ) -> Sequence[tuple[int, int]]:
        lang_spec = lang_spec_for_path(path)
        if lang_spec is None:
            return []
        out = []
        for comment in lang_spec.get_prefix_comments(text):
            out.extend(
                StsSelectorLineSpan(
                    start_line_idx=comment.doc_string_node.start_point[0],
                    end_line_idx=comment.doc_string_node.end_point[0] + 1,
                ).get_matching_spans(text, path),
            )
        return out


@dataclass(frozen=True)
class StsSelectorFullFile(StsTextSelector):
    """Selects the entire file"""

    def get_matching_spans(
        self,
        text: str,
        path: str,
    ) -> Sequence[tuple[int, int]]:
        return [(0, len(text))]


@dataclass(frozen=True)
class StsSelectorLineSpan(StsTextSelector):
    """
    Selects a fixed span of lines in the text. Note that it is line
    indicies which are 0 indexed, not 1-indexed line numbers
    """

    start_line_idx: int
    end_line_idx: int
    include_trailing_newline: bool = True

    def get_matching_spans(
        self,
        text: str,
        path: str,
    ) -> Sequence[tuple[int, int]]:
        """Returns the char span corresponding the the line span"""
        lines = text.splitlines(keepends=True)
        if self.start_line_idx > len(lines) or self.end_line_idx > len(lines):
            msg = "Invalid line span selection"
            raise IndexError(msg)
        if self.start_line_idx > self.end_line_idx:
            msg = "Start line index must be less than end line index"
            raise ValueError(msg)
        start_char = sum(len(line) for line in lines[: self.start_line_idx])
        end_char = start_char + sum(
            len(line) for line in lines[self.start_line_idx : self.end_line_idx]
        )
        # Remove trailing '\n' from count if it exists
        if (
            lines[self.end_line_idx - 1].endswith("\n")
            and not self.include_trailing_newline
        ):
            end_char -= 1
        return [(start_char, end_char)]


@dataclass(frozen=True)
class StsVerb:
    """
    A verb applies something at the given selection. This can
    be thought of like 'adding a node of xml'
    """

    @classmethod
    def get_allows_children(cls) -> bool:
        return True

    def operation_is_static(self) -> bool:
        """
        We define the verb operation as "static" if it applies
        once for anything that matches selectors. All the nodes
        in this selection will have the same mark_id.

        The intended usecase here is something like a code question,
        where we are asking about one or more regions of code,
        but really only expect one answer.

        However, if we wanted to say mark all function bodies as
        editable, we would not a static operation. Instead we want
        static to be False, so that we can assign a different value
        to each of the function bodies.
        """
        return False

    @cached_property
    def static_mark_id(self) -> str:
        if not self.operation_is_static():
            msg = "Operation is not static"
            raise ValueError(msg)
        return "s_" + hash_string_base62(
            f"{dataclasses.asdict(self)}",
            digest_size_bytes=4,
        )


@dataclass(frozen=True)
class StsNormalText(StsVerb):
    """
    Normal text is visible to solvers but cannot be by default edited.
    It is the default for any text in the working directory.
    """


@dataclass(frozen=True)
class StsInvisible(StsVerb):
    """Marks text as invisible to a solver, but still present when executing"""

    # TODO alt display text


@dataclass(frozen=True)
class StsPathInvisible(StsVerb):
    """Marks an entire path (and descendants) as invisible to a solver"""


@dataclass(frozen=True)
class StsPlaceTransforming(StsVerb):
    """A verb that specifies an edit to code"""

    max_length_chars: int = None
    max_length_lines: int = None
    lang_spec_name: str = field(
        default_factory=PythonLangSpec().get_lang_md_name,
    )
    """TODO think about if this is actually needed"""
    set_base_space_indent: int = None
    """If set, the solver should assume the value is a at a line-start. The value
    should start with this whitespace."""
    set_ending_text: str = None
    """The solution must end with this string. Solver can make sure
    to either append this or truncate to this point."""
    stop_at_block_end: bool = False
    """Whether the insertion should stop after the end of the current block
    (such as at the end of the current function)"""
    # TODO?
    # stop_text_sequences: list[str] = None
    # """If set, the solver should not insert any text that
    # contains any of these strings. Some solvers might have restrictions
    # on how many stop sequences they support (for example OpenAI only supports
    # up to 4). Note that parameters that `max_line_length` might interact with
    # this as if only limited to one line, then that might use up one of
    # the stop sequences from the solver model."""


@dataclass(frozen=True)
class StsInsert(StsPlaceTransforming):
    """Inserting new text at this point"""

    @classmethod
    def get_allows_children(cls) -> bool:
        return False


@dataclass(frozen=True)
class StsAllowFileWrite(StsPlaceTransforming):
    """
    NOT SUPPORTED YET. Intended to represent allowing the solver to
    write a new file to the working directory
    """

    forced_fn: str = None
    """Whether the solver must use a certain file name. If None, then the solver
    may choose any file name."""
    must_write: bool = False
    """Whether a solution is required to write this file"""
    lang_specs: tuple[LangSpec, ...] = None

    def __post_init__(self):
        raise NotImplementedError


@dataclass(frozen=True)
class StsEditable(StsPlaceTransforming):
    current_text_visible: bool = True


class StsValueQuery(StsVerb):
    """
    An StsVerb that doesn't transform any code in the working directory
    but instead the value itself is part of the answer.
    """

    required: bool

    def operation_is_static(self) -> bool:
        # Default to True for these kinds of queries
        return True


@dataclass(frozen=True)
class StsCodeQuestion(StsValueQuery):
    """
    Requests a freeform answer to question about code. Typically,
    this is not about creating or editing code (those can likely
    be expressed some other way like with StsEditable and instructions),
    but some more natural language-esque question.
    """

    question_text: str
    """The text of the question we want answered"""


@dataclass(frozen=True)
class StsCodeClassificationQuestion(StsCodeQuestion):
    """
    A code question with predefined set of answers.

    A few different values are valid:
    - A json serialized mapping string key answers to a probability.
    - A single string corresponding to selected answer. Assumed to have
        probability 1.0.
    """

    valid_answers: tuple[str, ...]

    def value_to_prob_dict(
        self,
        answer: str,
    ) -> dict[str, float]:
        if "{" in answer and "}" in answer:
            return json.loads(answer)
        return normalize_probs_dict({answer: 1.0}, self.valid_answers)


@dataclass(frozen=True)
class StsDocTestInjectionPoint(StsVerb):
    """
    NOT SUPPORTED YET. A point where a doc test can be inserted
    in cases where we want to support things like toggling on and off
    the test visibility or vary the displayed test cases accross several solves
    """

    def __post_init__(self):
        raise NotImplementedError


@dataclass(frozen=True)
class StsFewShotInjectionPoint(StsVerb):
    """
    TODO some way of representing where to insert few shot examples
    (but need to think through how this fits in with dialog few shots. It
    might be a bit complex than a single point)
    """

    def __post_init__(self):
        raise NotImplementedError


"""
#### Mark Nodes ####
"""


class StsMarkNode(ABC):
    """
    A xml-like (DOM-like) node that represents the result after applying
    a `TransformationSpec` to a file.
    """

    _original_start_char: int
    _original_end_char: int
    _frozen: bool = False

    def __init__(
        self,
        start_char: int,
        end_char: int,
    ):
        self._original_end_char = end_char
        self._original_start_char = start_char

    @property
    def mark_start_original_char(self):
        return self._original_start_char

    @mark_start_original_char.setter
    def mark_start_original_char(self, val: int):
        if self._frozen:
            msg = "Cannot modify frozen mark"
            raise RuntimeError(msg)
        self._original_start_char = val

    @property
    def mark_end_original_char(self):
        return self._original_end_char

    @mark_end_original_char.setter
    def mark_end_original_char(self, val: int):
        if self._frozen:
            msg = "Cannot modify frozen mark"
            raise RuntimeError(msg)
        self._original_end_char = val

    def freeze(self):
        self._frozen = True

    def as_xml(self, have_attributes: bool = True) -> str:
        raise NotImplementedError

    def pretty_print_xml(self) -> None:
        from xml.dom.minidom import parseString

        dom = parseString(self.as_xml())
        pretty_xml_as_string = dom.toprettyxml()
        print(pretty_xml_as_string)

    def depth_first_iterate(self) -> Generator["StsMarkNode", bool, None]:
        """
        A coroutine that yields all nodes starting with self. If
        the coroutine is sent False, then children of the current node are not
        visited.
        """
        visit_children = yield self
        if visit_children is None or visit_children:
            for child in self.children:
                yield from child.depth_first_iterate()

    def iterate_parser_states(
        self,
    ) -> Generator["StandardParserStates", bool, None]:
        """
        A similar coroutine to `depth_first_iterate`, but yields parser states
        that contains information about the current state (such as visible or invisible)
        """
        yield from iterate_parser_states_for_node(self)

    @property
    def children(self):
        raise NotImplementedError

    @children.setter
    def children(self, val: Sequence["StsMarkNode"]):
        raise NotImplementedError


class MarkText(StsMarkNode):
    """
    A DOM text node. Contains the text. For example <a>hello</a> would
    would be an `MarkElement` with a single `MarkText` child with text 'hello'.
    MarkText nodes are always leaves in the DOM tree.
    """

    def __init__(self, text: str, start_char: int, end_char: int):
        super().__init__(start_char, end_char)
        if len(text) != (end_char - start_char):
            msg = (
                f"Text length {len(text)} does not match char span"
                f" {start_char} to {end_char}"
            )
            raise Exception(msg)

        self._text = text

    def split(self, orig_char_point) -> tuple["MarkText", "MarkText"]:
        """Splits this text node into two at the given character index"""
        if (
            orig_char_point < self._original_start_char
            or orig_char_point > self._original_end_char
        ):
            msg = "Split point must be within text node"
            raise ValueError(msg)
        if orig_char_point == self._original_start_char:
            return (None, self)
        if orig_char_point == self._original_end_char:
            return (self, None)
        first = MarkText(
            self._text[: orig_char_point - self._original_start_char],
            self._original_start_char,
            orig_char_point,
        )
        second = MarkText(
            self._text[orig_char_point - self._original_start_char :],
            orig_char_point,
            self._original_end_char,
        )
        return (first, second)

    @property
    def children(self):
        return ()

    @children.setter
    def children(self, val: Sequence["StsMarkNode"]):
        msg = "Cannot set the children of a text node"
        raise RuntimeError(msg)

    def split_span(
        self,
        start,
        end,
    ) -> tuple[Optional["MarkText"], Optional["MarkText"], Optional["MarkText"]]:
        """Returns (before, inner, after) a selection of given span"""
        if end < start:
            msg = "End cannot be less than start."
            raise Exception(msg)

        inner_start_idx = start - self._original_start_char
        inner_end_idx = end - self._original_start_char
        inner_start_idx = max(0, min(inner_start_idx, len(self._text)))
        inner_end_idx = max(0, min(inner_end_idx, len(self._text)))
        if inner_start_idx == 0:
            before = None
        else:
            before = MarkText(
                self._text[:inner_start_idx],
                self._original_start_char,
                self._original_start_char + inner_start_idx,
            )
        if inner_start_idx == inner_end_idx or inner_start_idx >= len(
            self._text,
        ):
            inner = None
        else:
            inner = MarkText(
                self._text[
                    max(0, inner_start_idx) : min(
                        inner_end_idx,
                        len(self._text),
                    )
                ],
                self._original_start_char + max(0, inner_start_idx),
                self._original_start_char + min(inner_end_idx, len(self._text)),
            )
        if inner_end_idx >= len(self._text):
            after = None
        else:
            after = MarkText(
                self._text[min(inner_end_idx, len(self._text)) :],
                self._original_start_char + max(0, inner_end_idx),
                self._original_end_char,
            )
        return before, inner, after

    @property
    def text(self):
        return self._text

    @text.setter
    def text(self, val: str):
        if self._frozen:
            msg = "Cannot modify frozen mark"
            raise RuntimeError(msg)
        self._text = val

    def _tuple(self):
        return (self._text, self._original_start_char, self._original_end_char)

    def __eq__(self, other):
        if isinstance(other, MarkText):
            return self._tuple() == other._tuple()
        return NotImplemented

    def as_xml(self, have_attributes: bool = True) -> str:
        return saxutils.escape(self._text)

    def __repr__(self):
        return (
            f"MarkText({self._text!r}, {self._original_start_char},"
            f" {self._original_end_char})"
        )


T = TypeVar("T", bound="StsVerb")


class MarkElement(Generic[T], StsMarkNode):
    """
    A DOM element. It contains a verb (which might have certain properties)
    and a sequence of children.
    """

    def __init__(
        self,
        verb: T,
        children: Sequence[StsMarkNode],
        start_char: int,
        end_char: int,
    ):
        super().__init__(start_char, end_char)
        self._verb = verb
        self._children = children
        self._mark_id = None

    _insert_node_splice_error = (
        "Splitting in the inside a mark element is not supported. "
        "For example <a>foo<b>p</a NOT ALLOWED. Splices the 'b' tag>op</b>. "
        "Maybe in future implementation could be supported by "
        "closing and reopening the tag, but not currently"
    )

    def insert_node(
        self,
        verb: StsVerb,
        start_char: int,
        end_char: int,
    ):
        """
        Inserts a node into this element at a given span. See
        the test cases in `test_transformation_spec.py` for some examples.
        """
        if self._frozen:
            msg = "Cannot modify frozen mark"
            raise RuntimeError(msg)
        if start_char < self.mark_start_original_char:
            msg = "Cannot start before start char"
            raise ValueError(msg)
        if start_char > end_char:
            msg = "Start char cannot be greater than end char"
            raise ValueError(msg)
        if end_char > self._original_end_char:
            msg = "End char cannot be greater than original end char"
            raise ValueError(msg)
        if (
            start_char == self._original_start_char
            and end_char == self._original_end_char
        ):
            self.children = [
                MarkElement(verb, self._children, start_char, end_char),
            ]
            return None
        if len(self._children) == 0:
            msg = "I don't think this should every happen"
            raise RuntimeError(msg)
        new_children_self = []
        # Find the starting child
        for current_original_child_idx, child in enumerate(self._children):
            if start_char >= child.mark_end_original_char:
                new_children_self.append(child)
            else:
                break
        # Now `child` will be set as the child that contains the start of where
        #   we want to split. Now let's split it
        new_node = MarkElement(
            verb,
            [],
            start_char,
            end_char,
        )
        if isinstance(child, MarkText):
            # MarkText won't have more children, so we can just split it directly
            before, inner, after = child.split_span(start_char, end_char)
            if before is not None and start_char < child.mark_end_original_char:
                new_children_self.append(before)
            new_children_self.append(new_node)
            if inner is not None:
                new_node.children.append(inner)
            if after is not None:
                new_children_self.append(after)
            current_original_child_idx += 1  # We now used up the text
        elif isinstance(child, MarkElement):
            new_children_self.append(new_node)
            if end_char <= child.mark_end_original_char:
                if child.mark_start_original_char < start_char:
                    msg = (
                        "Child mark start original character less than start character."
                    )
                    raise Exception(msg)
                child.insert_node(verb, start_char, end_char)
                return None
        else:
            msg = "Unknown mark type"
            raise TypeError(msg)
        # scan to the end for children of our new node
        for child in self._children[current_original_child_idx:]:
            if (
                child.mark_start_original_char < end_char
                and child.mark_end_original_char <= end_char
            ):
                new_node.children.append(child)
            elif (
                child.mark_start_original_char < end_char
                and child.mark_end_original_char > end_char
            ):
                # Need to split
                if isinstance(child, MarkText):
                    before, inner, after = child.split_span(
                        start_char,
                        end_char,
                    )
                    if before is not None:
                        new_node.children.append(before)
                    if inner is not None:
                        new_node.children.append(inner)
                    if after is not None:
                        new_children_self.append(after)
                else:
                    raise ValueError(self._insert_node_splice_error)
            elif child.mark_start_original_char >= end_char:
                new_children_self.append(child)
            else:
                raise ValueError(self._insert_node_splice_error)
        self.children = new_children_self
        return self

    @property
    def verb(self):
        return self._verb

    @verb.setter
    def verb(self, val: StsVerb):
        if self._frozen:
            msg = "Cannot modify frozen mark"
            raise RuntimeError(msg)
        self._verb = val

    @property
    def children(self):
        return self._children

    @children.setter
    def children(self, val: Sequence[StsMarkNode]):
        if self._frozen:
            msg = "Cannot modify frozen mark"
            raise RuntimeError(msg)
        self._children = val

    def get_tag(self):
        return str(self.verb.__class__.__name__)

    def get_attributes(self) -> dict[str, str]:
        fields = dataclasses.fields(self.verb)
        out = {}
        for f in fields:
            val = getattr(self.verb, f.name)
            if val is not None:
                out[f.name] = str(val)
        out["_original_start_char"] = str(self._original_start_char)
        out["_original_end_char"] = str(self._original_end_char)
        return out

    def freeze(self):
        super().freeze()
        for child in self._children:
            child.freeze()
        self._children = tuple(self._children)
        return self

    def __eq__(self, other):
        if not isinstance(other, MarkElement):
            return NotImplemented
        return (
            self._verb == other._verb
            and self._original_start_char == other._original_start_char
            and self._original_end_char == other._original_end_char
            and self._children == other._children
        )

    def as_xml(self, have_attributes: bool = True) -> str:
        tag = self.get_tag()
        attributes = {}
        if self._frozen:
            attributes["id"] = self.mark_id
        attributes.update(self.get_attributes())
        if have_attributes:
            attr_str = " ".join(
                f"{key}={saxutils.quoteattr(val)}" for key, val in attributes.items()
            )
        else:
            attr_str = ""
        if len(self._children) == 0:
            return f"<{tag} {attr_str} />"
        children = (child.as_xml() for child in self._children)
        return f"<{tag} {attr_str}>{''.join(children)}</{tag}>"

    def __repr__(self):
        return (
            f"MarkElement({self._verb}, children={self._children}, "
            f"{self._original_start_char}, {self._original_end_char})"
        )

    @property
    def mark_id(self) -> str:
        """Returns a unique string for representing this mark"""
        if self.verb.operation_is_static():
            return self.verb.static_mark_id
        if not self._frozen:
            msg = "Cannot get mark key for unfrozen mark"
            raise RuntimeError(msg)
        # NOTE: This probably could fail with multiple marks on the same character
        if self._mark_id is None:
            self._mark_id = hash_string_base62(
                f"tag:{self.get_tag()},{self._original_start_char},"
                f"{self._original_end_char},{len(self._children)}",
                digest_size_bytes=4,
            )
        return self._mark_id


@dataclass(frozen=True)
class SolveStep:
    """A step for constructing a solution. Writes a value at a certain mark"""

    path: str
    mark_id: MARK_KEY_TYPE
    value: str

    def dict_serialize(self):
        return {
            "path": self.path,
            "mark_id": self.mark_id,
            "value": self.value,
        }


"""
Standard Parser:

Utilities for walking through a markup keeping track of things
like the visibility state.
"""


@dataclass(frozen=True, slots=True)
class StandardParserStates:
    node: "MarkElement" = None
    is_visible: bool = True
    is_editable: bool = False
    parent_state: Optional["StandardParserStates"] = None
    after_visit_children_callbacks: list[callable] = field(default_factory=list)
    """These callables are called after the children of the current node.
    A possible use case is to add a closing tag to the current node.
    It is not clear this is good design but is useful for now."""


def iterate_parser_states_for_node(
    node,
    parent_state: StandardParserStates = None,
) -> Generator[StandardParserStates, bool, None]:
    """
    A similar coroutine to `depth_first_iterate`, but yields parser states
    that contains information about the current state (such as visible or invisible).
    """
    has_parent_state = parent_state is not None
    if not has_parent_state:
        parent_state = StandardParserStates()
    new_visible = parent_state.is_visible
    new_editable = parent_state.is_editable
    if isinstance(node, MarkElement):
        if isinstance(node.verb, StsInvisible):
            new_visible = False
        elif isinstance(node.verb, StsPlaceTransforming):
            new_editable = True
        elif isinstance(node.verb, StsNormalText | StsValueQuery):
            pass
        else:
            msg = f"Unknown verb {node.verb}"
            raise TypeError(msg)
    new_state = StandardParserStates(
        node,
        is_visible=new_visible,
        is_editable=new_editable,
        parent_state=parent_state if has_parent_state else None,
    )
    visit_children = yield new_state
    if visit_children is None or visit_children:
        for child in node.children:
            yield from iterate_parser_states_for_node(child, new_state)
    for listener in new_state.after_visit_children_callbacks:
        listener()
