#!/usr/bin/env python3

"""
Written by Ulf Hermjakob, USC/ISI  March-June 2024
uroman is a universal romanizer. It converts text in any script to the Latin alphabet.
This script is a Python reimplementation of an earlier Perl script, with some improvements.
The tool has been tested on 250 languages, with 100 or more sentences each.
This script is still under development and large-scale testing. Feedback welcome.
This script provides token-size caching (for faster runtimes).
Output formats include
  (1) best romanization string
  (2) best romanization edges ("best path"; incl. start and end positions with respect to the original string)
  (3) best romanization with alternatives (as applicable for ambiguous romanization)
  (4) best romanization full lattice (all edges, including superseded sub-edges)
See below for 'sample calls' under main()
"""


from __future__ import annotations
import argparse
from collections import defaultdict
# from memory_profiler import profile
import datetime
from enum import Enum
from fractions import Fraction
import gc
import json
import math
import os
from pathlib import Path
import pstats
import regex
import sys
from typing import List, Tuple
import unicodedata as ud
DEFAULT_ROM_MAX_CACHE_SIZE = 65536
PROFILE_FLAG = "--profile"  # also used in argparse processing
if PROFILE_FLAG in sys.argv:
    import cProfile

__version__ = '1.3.1'
__last_mod_date__ = 'June 27, 2024'
__description__ = "uroman is a universal romanizer. It converts text in any script to the standard Latin alphabet."

# UTILITIES


def timer(func):
    def wrapper(*args, **kwargs):
        start_time = datetime.datetime.now()
        print(f"Calling: {func.__name__}{args}")
        print(f"Start time: {start_time:%A, %B %d, %Y at %H:%M}")
        result = func(*args, **kwargs)
        end_time = datetime.datetime.now()
        time_diff = (end_time-start_time).total_seconds()
        print(f"End time: {end_time:%A, %B %d, %Y at %H:%M}")
        print(f"Duration: {time_diff} seconds")
        return result
    return wrapper


def slot_value_in_double_colon_del_list(line: str, slot: str, default: str | list | None = None) -> str | list | None:
    """For a given slot, e.g. 'cost', get its value from a line such as '::s1 of course ::s2 ::cost 0.3' -> 0.3
    The value can be an empty string, as for ::s2 in the example above."""
    m = regex.match(fr'(?:.*\s)?::{slot}(|\s+\S.*?)(?:\s+::\S.*|\s*)$', line)
    return m.group(1).strip() if m else default


def has_value_in_double_colon_del_list(line: str, slot: str) -> bool:
    return isinstance(slot_value_in_double_colon_del_list(line, slot), str)


def dequote_string(s: str) -> str:
    if isinstance(s, str):
        m = regex.match(r'''\s*(['"“])(.*)(['"”])\s*$''', s)
        if m and ((m.group(1) + m.group(3)) in ("''", '""', '“”')):
            return m.group(2)
    return s


def last_chr(s: str) -> str:
    if len(s):
        return s[len(s)-1]
    else:
        ''


def ud_numeric(char: str) -> int | float | None:
    try:
        num_f = ud.numeric(char)
        return int(num_f) if num_f.is_integer() else num_f
    except (ValueError, TypeError):
        return None


def robust_str_to_num(num_s: str, filename: str = None, line_number: int | None = None, silent: bool = False) \
        -> int | float | None:
    if isinstance(num_s, str):
        try:
            return float(num_s) if "." in num_s else int(num_s)
        except ValueError:
            if not silent:
                sys.stderr.write(f'Cannot convert "{num_s}" to a number')
                if line_number:
                    sys.stderr.write(f' line: {line_number}')
                if filename:
                    sys.stderr.write(f' file: {filename}')
                sys.stderr.write(f'\n')
    elif isinstance(num_s, float) or isinstance(num_s, int):
        return num_s
    return None


def first_non_none(*args):
    for arg in args:
        if arg is not None:
            return arg
    return None


def any_not_none(*args) -> bool:
    for arg in args:
        if arg is not None:
            return True
    return False


def add_non_none_to_dict(d: dict, key: str, value) -> None:
    if value is not None:
        d[key] = value


def fraction_char2fraction(fraction_char: str, fraction_value: float | None = None,
                           uroman: Uroman | None = None) -> Fraction | None:
    s = ''
    fraction = None
    for ud_decomp_elem in ud.decomposition(fraction_char).split():
        try:
            s += chr(int(ud_decomp_elem, 16))
        except ValueError:
            s += ud_decomp_elem
    if m := regex.match(r'<fraction>(\d+)⁄(\d+)$', s):
        numerator_s, denominator_s = m.group(1, 2)
        try:
            fraction = Fraction(int(numerator_s), int(denominator_s))
        except ValueError:
            fraction = None
    if (fraction is None) and uroman and fraction_value:
        if numerator_denominator := uroman.unicode_float2fraction(fraction_value):
            try:
                fraction = Fraction(numerator_denominator[0], numerator_denominator[1])
            except ValueError:
                fraction = None
    return fraction


def chr_name(char: str) -> str:
    """robust version of ud.name; see related Uroman.char_name() that includes names not included in UnicodeData.txt"""
    try:
        return ud.name(char)
    except (ValueError, TypeError):
        return ''


def args_get(key: str, args: argparse.Namespace | None = None):
    return vars(args)[key] if args and (key in args) else None


class DictClass:
    def __init__(self, **kw_args):
        for kw_arg in kw_args:
            kw_arg2 = kw_arg.replace('_', '-')
            value = kw_args[kw_arg]
            if not (value in (None, [], False)):
                self.__dict__[kw_arg2] = value

    def __repr__(self):
        return str(self.__dict__)

    def __getitem__(self, key, default=None):
        return self.__dict__[key] if key in self.__dict__ else default

    def __bool__(self):
        return len(self.__dict__) > 0


class RomRule(DictClass):
    # key: source string
    # typical attributes: s (source), t (target), prov (provenance), lcodes (language codes)
    # t_alts=t_alts (target alternatives), use_only_at_start_of_word, dont_use_at_start_of_word,
    # use_only_at_end_of_word, dont_use_at_end_of_word, use_only_for_whole_word
    pass


class Script(DictClass):
    # key: lower case script_name
    # typical attributes: script_name, direction, abugida_default_vowels, alt_script_names, languages
    pass


class RomFormat(Enum):
    """Output format of romanization"""
    STR = 'str'          # simple string
    EDGES = 'edges'      # list of edges (includes character offsets in original string)
    ALTS = 'alts'        # lattice including alternative edges
    LATTICE = 'lattice'  # lattice including alternative and superseded edges

    def __str__(self):
        return self.value


class Uroman:
    """This class loads and maintains uroman data independent of any specific text corpus.
    Typically, only a single instance will be used. (In contrast to multiple lattice instances, one per text.)
    Methods include some testing. And finally methods to romanize a string (romanize_string()) or an entire file
    (romanize_file())."""
    def __init__(self, data_dir: Path | None = None, **args):  # args: load_log, rebuild_ud_props
        self.data_dir = data_dir or self.default_data_dir(**args)
        self.rom_rules = defaultdict(list)
        self.scripts = defaultdict(Script)
        self.dict_bool = defaultdict(bool)
        self.dict_str = defaultdict(str)
        self.dict_int = defaultdict(int)
        self.dict_num = defaultdict(lambda: None)   # values are int (most common), float, or str ("1/2")
        # num_props key: txt
        # values:  {"txt": "\u137b", "rom": "100", "value": 100, "type": "base", "mult": 1, "script": "Ethiopic"}
        self.num_props = defaultdict(dict)
        self.dict_set = defaultdict(set)
        self.fraction_connectors = {}
        self.minus_signs = {}
        self.plus_signs = {}
        self.float2fraction = {}  # caching
        gc.disable()
        self.rom_cache = {}   # key: (s, lcode) value: t
        self.rom_cache_size = 0
        self.rom_max_cache_size = args.get('cache_size', 0)
        self.cache_p = (self.rom_max_cache_size != 0)
        self.hangul_rom = {}
        self.stats = defaultdict(int)  # stats, e.g. for unprocessed numbers
        self.abugida_cache = {}  # key: (script, char_rom) value: (base_rom, base_rom_plus_abugida_vowel, modified rom)
        self.load_resource_files(self.data_dir, args.get('load_log', False),
                                 args.get('rebuild_ud_props', False),
                                 args.get('rebuild_num_props', False))
        gc.enable()
        self.n_error_messages_output = 0
        self.n_non_utf8_characters = 0

    @staticmethod
    def default_data_dir(**args) -> Path:
        root_dir = Path(__file__).parent
        data_dir = (root_dir / "data").resolve()
        mini_test_dir = (root_dir / "mini-test").resolve()
        if args.get('verbose'):
            sys.stderr.write(f"data_dir: {str(data_dir)}\n")
            sys.stderr.write(f"mini_test_dir: {str(mini_test_dir)}\n")
        return data_dir

    def reset_cache(self, cache_size: int = DEFAULT_ROM_MAX_CACHE_SIZE):
        self.rom_cache = {}
        self.rom_cache_size = 0
        self.rom_max_cache_size = cache_size
        self.cache_p = (self.rom_max_cache_size != 0)

    # noinspection SpellCheckingInspection
    def second_rom_filter(self, c: str, rom: str, name: str | None) -> Tuple[str | None, str]:
        """Much of this code will eventually move the old Perl code to generate cleaner primary data"""
        if rom and (' ' in rom):
            if name is None:
                name = self.chr_name(c)
            if "MYANMAR VOWEL SIGN KAYAH" in name:
                if m := regex.search(r'kayah\s+(\S+)\s*$', rom):
                    return m.group(1), name
            if "MENDE KIKAKUI SYLLABLE" in name:
                if m := regex.search(r'm\d+\s+(\S+)\s*$', rom):
                    return m.group(1), name
            if regex.search(r'\S\s+\S', rom):
                return c, name
        return None, name

    def load_rom_file(self, filename: str, provenance: str, file_format: str = None, load_log: bool = True):
        """Reads in and processes the 3 main romanization data files: (1) romanization-auto-table.txt
        which was automatically generated from UnicodeData.txt (2) UnicodeDataOverwrite.txt that "corrects"
        some entries in romanization-auto-table.txt and (3) romanization-table.txt which was largely manually
        created and allows complex romanization rules, some for specific languages, some for specific contexts."""
        n_entries = 0
        try:
            f = open(filename, 'r', encoding='utf-8')
        except FileNotFoundError:
            sys.stderr.write(f'Cannot open file {filename}\n')
            return
        with (f):
            for line_number, line in enumerate(f, 1):
                if line.startswith('#'):
                    continue
                if regex.match(r'^\s*$', line):  # blank line
                    continue
                line = regex.sub(r'\s{2,}#.*$', '', line)
                if file_format == 'u2r':
                    t_at_end_of_syllable = None
                    u = dequote_string(slot_value_in_double_colon_del_list(line, 'u'))
                    try:
                        cp = int(u, 16)
                        s = chr(cp)
                    except ValueError:
                        continue
                    t = dequote_string(slot_value_in_double_colon_del_list(line, 'r'))
                    if name := slot_value_in_double_colon_del_list(line, 'name'):
                        self.dict_str[('name', s)] = name
                    if pic := slot_value_in_double_colon_del_list(line, 'pic'):
                        self.dict_str[('pic', s)] = pic
                    if tone_mark := slot_value_in_double_colon_del_list(line, 'tone-mark'):
                        self.dict_str[('tone-mark', s)] = tone_mark
                    if syllable_info := slot_value_in_double_colon_del_list(line, 'syllable-info'):
                        self.dict_str[('syllable-info', s)] = syllable_info
                else:
                    s = dequote_string(slot_value_in_double_colon_del_list(line, 's'))
                    t = dequote_string(slot_value_in_double_colon_del_list(line, 't'))
                    t_at_end_of_syllable = dequote_string(slot_value_in_double_colon_del_list(line,
                                                                                              't-end-of-syllable'))
                if (num_s := slot_value_in_double_colon_del_list(line, 'num')) is not None:
                    num = robust_str_to_num(num_s)
                    self.dict_num[s] = (num_s if (num is None) else num)
                is_minus_sign = has_value_in_double_colon_del_list(line, 'is-minus-sign')
                if is_minus_sign:
                    self.minus_signs[s] = True
                is_plus_sign = has_value_in_double_colon_del_list(line, 'is-plus-sign')
                if is_plus_sign:
                    self.plus_signs[s] = True
                is_decimal_point = has_value_in_double_colon_del_list(line, 'is-decimal-point')
                is_large_power = has_value_in_double_colon_del_list(line, 'is-large-power')
                fraction_connector = slot_value_in_double_colon_del_list(line, 'fraction-connector')
                if fraction_connector:
                    self.fraction_connectors[s] = True
                percentage_marker = slot_value_in_double_colon_del_list(line, 'percentage-marker')
                int_frac_connector = slot_value_in_double_colon_del_list(line, 'int-frac-connector')
                lcode_s = slot_value_in_double_colon_del_list(line, 'lcode')
                lcodes = regex.split(r'[,;]\s*', lcode_s) if lcode_s else []
                use_only_at_start_of_word = has_value_in_double_colon_del_list(line, 'use-only-at-start-of-word')
                dont_use_at_start_of_word = has_value_in_double_colon_del_list(line, 'dont-use-at-start-of-word')
                use_only_at_end_of_word = has_value_in_double_colon_del_list(line, 'use-only-at-end-of-word')
                dont_use_at_end_of_word = has_value_in_double_colon_del_list(line, 'dont-use-at-end-of-word')
                use_only_for_whole_word = has_value_in_double_colon_del_list(line, 'use-only-for-whole-word')
                num_s = slot_value_in_double_colon_del_list(line, 'num')
                num = robust_str_to_num(num_s, filename, line_number, silent=False)
                t_alt_s = slot_value_in_double_colon_del_list(line, 't-alt')
                t_alts = regex.split(r'[,;]\s*', t_alt_s) if t_alt_s else []
                t_alts = list(map(dequote_string, t_alts))
                t_mod, name2 = self.second_rom_filter(s, t, None)
                if t_mod and (t_mod != t):
                    if t != s:
                        pass  # sys.stderr.write(f"UPDATE: {s} {name2} {t} -> {t_mod}\n")
                    t = t_mod
                if s is not None:
                    for bool_key in ('is-large-power', 'is-minus-sign', 'is-plus-sign', 'is-decimal-point'):
                        bool_value = eval(bool_key.replace('-', '_'))
                        if bool_value:
                            self.dict_bool[(bool_key, s)] = True
                    if any_not_none(t, num, is_minus_sign, is_plus_sign, is_decimal_point, is_large_power,
                                    fraction_connector, percentage_marker, int_frac_connector):
                        self.register_s_prefix(s)
                        n_entries += 1
                        # if regex.match(r'[\u2800-\u28FF]', s): print("Braille", s, t)
                        restrictions = [lcodes, use_only_at_start_of_word, dont_use_at_start_of_word,
                                        use_only_at_end_of_word, dont_use_at_end_of_word, use_only_for_whole_word]
                        n_restrictions = len([restr for restr in restrictions if restr])
                        provenance2 = provenance
                        if (t is None) and (num is not None) and (provenance2 == "rom"):
                            provenance2 = "num"
                        new_rom_rule = RomRule(s=s, t=t, prov=provenance2, lcodes=lcodes, t_alts=t_alts, num=num,
                                               use_only_at_start_of_word=use_only_at_start_of_word,
                                               dont_use_at_start_of_word=dont_use_at_start_of_word,
                                               use_only_at_end_of_word=use_only_at_end_of_word,
                                               dont_use_at_end_of_word=dont_use_at_end_of_word,
                                               use_only_for_whole_word=use_only_for_whole_word,
                                               t_at_end_of_syllable=t_at_end_of_syllable,
                                               n_restr=n_restrictions,
                                               is_minus_sign=is_minus_sign,
                                               is_plus_sign=is_plus_sign,
                                               is_decimal_point=is_decimal_point,
                                               fraction_connector=fraction_connector,
                                               percentage_marker=percentage_marker,
                                               int_frac_connector=int_frac_connector,
                                               is_large_power=is_large_power)
                        old_rom_rules = self.rom_rules[s]
                        if ((len(old_rom_rules) == 1) and (old_rom_rules[0]['prov'] in ('ud', 'ow'))
                                and not (lcodes or use_only_at_start_of_word or dont_use_at_start_of_word
                                         or use_only_at_end_of_word or dont_use_at_end_of_word
                                         or use_only_for_whole_word)):
                            self.rom_rules[s] = [new_rom_rule]  # overwrite
                        else:
                            self.rom_rules[s].append(new_rom_rule)
        # Thai
        thai_cancellation_mark = '\u0E4C'
        # cancellation applies to preceding letter incl. any vowel modifier letter
        # noinspection SpellCheckingInspection (e.g. ศักดิ์สิทธิ์ -> saksit)
        for cp in range(0x0E01, 0x0E4C):   # Thai
            c = chr(cp)
            s = c + thai_cancellation_mark
            new_rom_rule = RomRule(s=s, t='', prov='auto cancel letter')
            if not self.rom_rules[s]:
                self.rom_rules[s] = [new_rom_rule]
                self.register_s_prefix(s)
        thai_consonants = list(map(chr, range(0x0E01, 0x0E2F)))
        thai_vowel_modifiers = ['\u0E31', '\u0E47'] + list(map(chr, range(0x0E33, 0x0E3B)))
        for c1 in thai_consonants:
            for v in thai_vowel_modifiers:
                s = c1 + v + thai_cancellation_mark
                new_rom_rule = RomRule(s=s, t='', prov='auto cancel syllable')
                if not self.rom_rules[s]:
                    self.rom_rules[s] = [new_rom_rule]
                    self.register_s_prefix(s)
        if load_log:
            sys.stderr.write(f'Loaded {n_entries} from {filename}\n')

    def load_script_file(self, filename: str, load_log: bool = True):
        """Reads in (typically from Scripts.txt) information about various scripts such as Devanagari,
        incl. information such as the default abugida vowel letter (e.g. "a")."""
        n_entries, max_n_script_name_components = 0, 0
        try:
            f = open(filename, 'r', encoding='utf-8')
        except FileNotFoundError:
            sys.stderr.write(f'Cannot open file {filename}\n')
            return
        with f:
            for line_number, line in enumerate(f, 1):
                if line.startswith('#'):
                    continue
                if regex.match(r'^\s*$', line):  # blank line
                    continue
                line = regex.sub(r'\s{2,}#.*$', '', line)
                if script_name := slot_value_in_double_colon_del_list(line, 'script-name'):
                    lc_script_name = script_name.lower()
                    if lc_script_name in self.scripts:
                        sys.stderr.write(f'** Ignoring duplicate script "{script_name}" '
                                         f'in line {line_number} of {filename}\n')
                    else:
                        n_entries += 1
                        direction = slot_value_in_double_colon_del_list(line, 'direction')
                        abugida_default_vowel_s = slot_value_in_double_colon_del_list(line,
                                                                                      'abugida-default-vowel')
                        abugida_default_vowels = regex.split(r'[,;]\s*', abugida_default_vowel_s) \
                            if abugida_default_vowel_s else []
                        alt_script_name_s = slot_value_in_double_colon_del_list(line, 'alt-script-name')
                        alt_script_names = regex.split(r'[,;]\s*', alt_script_name_s) if alt_script_name_s else []
                        language_s = slot_value_in_double_colon_del_list(line, 'language')
                        languages = regex.split(r'[,;]\s*', language_s) if language_s else []
                        new_script = Script(script_name=script_name, alt_script_names=alt_script_names,
                                            languages=languages, direction=direction,
                                            abugida_default_vowels=abugida_default_vowels)
                        self.scripts[lc_script_name] = new_script
                        for language in languages:
                            self.dict_set[('scripts', language)].add(script_name)
                        for alt_script_name in alt_script_names:
                            lc_alt_script_name = alt_script_name.lower()
                            if lc_alt_script_name in self.scripts:
                                sys.stderr.write(f'** Ignoring duplicate alternative script name "{script_name}" '
                                                 f'in line {line_number} of {filename}\n')
                            else:
                                self.scripts[lc_alt_script_name] = new_script
                    n_script_name_components = len(script_name.split())
                    if n_script_name_components > max_n_script_name_components:
                        max_n_script_name_components = n_script_name_components
        if max_n_script_name_components:
            self.dict_int['max_n_script_name_components'] = max_n_script_name_components
        if load_log:
            sys.stderr.write(f'Loaded {n_entries} script descriptions from {filename}'
                             f' (max_n_scripts_name_components: {max_n_script_name_components})\n')

    def extract_script_name(self, script_name_plus: str, full_char_name: str = None) -> str | None:
        """Using info from Scripts.txt, this script selects the script name from a Unicode,
        e.g. given "OLD HUNGARIAN CAPITAL LETTER A", extract "Old Hungarian"."""
        if full_char_name and script_name_plus == full_char_name:
            return None
        while script_name_plus:
            if script_name_plus.lower() in self.scripts:
                if script := self.scripts[script_name_plus.lower()]:
                    if script_name := script['script-name']:
                        return script_name
            script_name_plus = regex.sub(r'\s*\S*\s*$', '', script_name_plus)
        return None

    def load_unicode_data_props(self, filename: str, load_log: bool = True):
        """Loads Unicode derived data from (1) UnicodeDataProps.txt, (2) UnicodeDataPropsHangul.txt
        and UnicodeDataPropsCJK.txt with a list of valid script-specific characters."""
        n_script, n_script_char, n_script_vowel_sign, n_script_medial_consonant_sign, n_script_virama = 0, 0, 0, 0, 0
        try:
            f = open(filename, 'r', encoding='utf-8')
        except FileNotFoundError:
            sys.stderr.write(f'Cannot open file {filename}\n')
            return
        with f:
            for line_number, line in enumerate(f, 1):
                if line.startswith('#'):
                    continue
                if regex.match(r'^\s*$', line):  # blank line
                    continue
                line = regex.sub(r'\s{2,}#.*$', '', line)
                if script_name := slot_value_in_double_colon_del_list(line, 'script-name'):
                    n_script += 1
                    for char in slot_value_in_double_colon_del_list(line, 'char', []):
                        self.dict_str[('script', char)] = script_name
                        n_script_char += 1
                    for char in slot_value_in_double_colon_del_list(line, 'numeral', []):
                        self.dict_str[('script', char)] = script_name
                        n_script_char += 1
                    for char in slot_value_in_double_colon_del_list(line, 'vowel-sign', []):
                        self.dict_bool[('is-vowel-sign', char)] = True
                        n_script_vowel_sign += 1
                    for char in slot_value_in_double_colon_del_list(line, 'medial-consonant-sign', []):
                        self.dict_bool[('is-medial-consonant-sign', char)] = True
                        n_script_medial_consonant_sign += 1
                    for char in slot_value_in_double_colon_del_list(line, 'sign-virama', []):
                        self.dict_bool[('is-virama', char)] = True
                        n_script_virama += 1
        if load_log:
            sys.stderr.write(f'Loaded from {filename} mappings of {n_script_char:,d} characters '
                             f'to {n_script} script{"" if n_script == 1 else "s"}')
            if n_script_vowel_sign or n_script_virama or n_script_medial_consonant_sign:
                sys.stderr.write(f', with a total of {n_script_vowel_sign} vowel signs, '
                                 f'{n_script_medial_consonant_sign} medial consonant signs '
                                 f'and {n_script_virama} viramas')
            sys.stderr.write('.\n')

    def load_num_props(self, filename: str, load_log: bool = True):
        """Loads Unicode derived data from (1) UnicodeDataProps.txt, (2) UnicodeDataPropsHangul.txt
        and UnicodeDataPropsCJK.txt with a list of valid script-specific characters."""
        n_entries = 0
        try:
            f = open(filename, 'r', encoding='utf-8')
        except FileNotFoundError:
            sys.stderr.write(f'Cannot open file {filename}\n')
            return
        with f:
            for line_number, line in enumerate(f, 1):
                if line.startswith('#'):
                    continue
                if regex.match(r'^\s*$', line):  # blank line
                    continue
                d = json.loads(line)
                if isinstance(d, dict):
                    if txt := d.get('txt'):
                        self.num_props[txt] = d
                        n_entries += 1
                    else:
                        sys.stderr.write(f'Missing txt in l.{line_number} in file {filename}: {line.strip()}\n')
                    for bool_key in ('is-large-power',):
                        if d.get(bool_key):
                            self.dict_bool[(bool_key, txt)] = True
                else:
                    sys.stderr.write(f'json in l.{line_number} in file {filename} not a dict: {line.strip()}\n')
        if load_log:
            sys.stderr.write(f'Loaded {n_entries} entries from {filename}\n')

    @staticmethod
    def de_accent_pinyin(s: str) -> str:
        """De-accents a string from "liú" to "liu" and "ü" to "u" (to help process file Chinese_to_Pinyin.txt)."""
        result = ''
        for char in s:
            if decomp := ud.decomposition(char).split():
                try:
                    decomp_chars = [chr(int(x, 16)) for x in decomp]
                    letters = [x for x in decomp_chars if ud.category(x).startswith('L')]
                except ValueError:
                    sys.stderr.write(f'Cannot decode {decomp}\n')
                    continue
                if len(letters) == 1:
                    result += letters[0]
                else:
                    sys.stderr.write(f'Cannot decode {decomp} (expected 1 letter)\n')
            else:
                result += char
        result = result.replace('ü', 'u')
        return result

    def register_s_prefix(self, s: str):
        for prefix_len in range(1, len(s) + 1):
            self.dict_bool[('s-prefix', s[:prefix_len])] = True

    def load_chinese_pinyin_file(self, filename: str, load_log: bool = True):
        """Loads file Chinese_to_Pinyin.txt which maps Chinese characters to their Latin form."""
        n_entries = 0
        try:
            f = open(filename, 'r', encoding='utf-8')
        except FileNotFoundError:
            sys.stderr.write(f'Cannot open file {filename}\n')
            return
        with f:
            for line_number, line in enumerate(f, 1):
                if line.startswith('#'):
                    continue
                if regex.match(r'^\s*$', line):  # blank line
                    continue
                try:
                    chinese, pinyin = line.rstrip().split()
                    rom = self.de_accent_pinyin(pinyin)
                except ValueError:
                    sys.stderr.write(f'Cannot process line {line_number} in file {filename}: {line}')
                else:
                    s = chinese
                    new_rom_rule = RomRule(s=s, t=rom, prov='rom pinyin', lcodes=[])
                    self.rom_rules[chinese].append(new_rom_rule)
                    self.register_s_prefix(s)
                    n_entries += 1
        if load_log:
            sys.stderr.write(f'Loaded {n_entries} script descriptions from {filename}\n')

    @staticmethod
    def add_char_to_rebuild_unicode_data_dict(d: dict, script_name: str, prop_class: str, char: str):
        d['script-names'].add(script_name)
        key = (script_name, prop_class)
        if key in d:
            d[key].append(char)
        else:
            d[key] = [char]

    def rebuild_unicode_data_props(self, out_filename: str, cjk: str = None, hangul: str = None):
        """This functions rebuilds UnicodeDataProps*.txt This might be useful when a new UnicodeData.txt
        version is released, or additional information is extracted from Unicode to UnicodeDataProps.txt
        Regular users normally never have to call this function."""
        d = {'script-names': set()}
        vowel_s = ''
        n_script_refs = 0
        codepoint = -1
        prop_classes = {'char'}
        while codepoint < 0xF0000:
            codepoint += 1
            c = chr(codepoint)
            if not (char_name := self.chr_name(c)):
                continue
            # noinspection SpellCheckingInspection
            for prop_name_comp2 in ('VOWEL SIGN',
                                    ('MEDIAL CONSONANT SIGN', 'CONSONANT SIGN MEDIAL', 'CONSONANT SIGN SHAN MEDIAL',
                                     'CONSONANT SIGN MON MEDIAL'),
                                    ('SIGN VIRAMA', 'SIGN ASAT', 'AL-LAKUNA', 'SIGN COENG', 'SIGN PAMAAEH',
                                     'CHARACTER PHINTHU'),
                                    ('NUMERAL', 'NUMBER', 'DIGIT', 'FRACTION')):
                if prop_name_comp2 and isinstance(prop_name_comp2, tuple):
                    prop_list = prop_name_comp2
                else:
                    prop_list = (prop_name_comp2,)
                for prop_name_comp in prop_list:
                    prop_class = prop_list[0].lower().replace(' ', '-')
                    if prop_class not in prop_classes:
                        prop_classes.add(prop_class)
                    script_name_cand = regex.sub(fr'\s+{prop_name_comp}\b.*$', '', char_name)
                    if script_name := self.extract_script_name(script_name_cand, char_name):
                        self.add_char_to_rebuild_unicode_data_dict(d, script_name, prop_class, c)
            script_name_cand = regex.sub(r'\s+(CONSONANT|LETTER|LIGATURE|SIGN|SYLLABLE|SYLLABICS|VOWEL|'
                                         r'IDEOGRAPH|HIEROGLYPH|POINT|ACCENT|CHARACTER|TIPPI|ADDAK|IRI|URA|'
                                         r'SYMBOL GENITIVE|SYMBOL COMPLETED|SYMBOL LOCATIVE|SYMBOL AFOREMENTIONED|'
                                         r'AU LENGTH MARK)\b.*$', '',
                                         char_name)
            if script_name := self.extract_script_name(script_name_cand, char_name):
                self.add_char_to_rebuild_unicode_data_dict(d, script_name, 'char', c)
                n_script_refs += 1
            rom = self.romanize_string(c)
            # noinspection SpellCheckingInspection
            if regex.match(r'^[aeiou]*[aeiouy]$', rom, regex.IGNORECASE):
                vowel_s += c

        # print(sorted(d['script-names']))
        prop_classes = sorted(prop_classes)
        out_filenames = [x for x in [out_filename, cjk, hangul] if x]
        cjk2 = cjk if cjk else out_filename
        hangul2 = hangul if hangul else out_filename
        for out_file in out_filenames:
            try:
                f_out = open(out_file, 'w', encoding='utf-8')
            except OSError:
                sys.stderr.write(f'Cannot write to file {out_file}\n')
                continue
            with f_out:
                for script_name in sorted(d['script-names']):
                    if script_name == 'CJK':
                        if out_file != cjk2:
                            continue
                    elif script_name == 'Hangul':
                        if out_file != hangul2:
                            continue
                    else:
                        if out_file != out_filename:
                            continue
                    prop_components = [f"::script-name {script_name}"]
                    for prop_class in prop_classes:
                        key = (script_name, prop_class)
                        if key in d:
                            if chars := ''.join(d[key]):
                                if prop_class in ('char',):
                                    prop_components.append(f"::n-{prop_class} {len(chars)}")
                                prop_components.append(f"::{prop_class} {chars}")
                    f_out.write(f"{' '.join(prop_components)}\n")
                if (out_file == out_filename) and vowel_s:
                    f_out.write(f'::vowels {vowel_s}\n')
        sys.stderr.write(f"Rebuilt {out_filenames} with {n_script_refs} characters "
                         f"for {len(d['script-names'])} scripts.\n")

    def rebuild_num_props(self, out_filename: str, err_filename: str):
        n_out, n_err = 0, 0
        with (open(out_filename, 'w', encoding='utf-8') as f_out,
              open(err_filename, 'w', encoding='utf-8') as f_err):
            codepoint = -1
            while codepoint < 0xF0000:
                codepoint += 1
                char = chr(codepoint)
                num = first_non_none(ud_numeric(char),  # robust ud.numeric
                                     self.num_value(char))  # uroman table includes extra num values, e.g. for Egyptian
                if num is None:
                    continue
                result_dict = {}
                orig_txt = char
                value: int | float | None = None  # non-fraction-value(3 1/2) = 3
                fraction: Fraction | None = None  # fraction(3 1/2) = Fraction(1, 2)
                num_base = None  # num_base(500) = 100
                base_multiplier = None  # base_multiplier(500) = 5
                script = None
                is_large_power = self.dict_bool[('is-large-power', char)]
                # num_base is typically a power of 10: 1, 10, 100, 1000, 10000, 100000, 1000000, ...
                # exceptions might include 12 for the 'dozen' in popular English 'two dozen and one' (2*12+1=25)
                # exceptions might include 20 for the 'score' in archaic English 'four score and seven' (4*20+7=87)
                # noinspection SpellCheckingInspection   exceptions might include 20 for the 'vingt'
                # noinspection SpellCheckingInspection   as in standard French 'quatre-vingt-treize' (4*20+13=93)
                if script_name := self.chr_script_name(char):
                    script = script_name
                elif char in '0123456789':
                    script = 'ascii-digit'
                name = self.chr_name(char)
                exclude_from_number_processing = False
                for scrypt_type in ('SUPERSCRIPT', 'SUBSCRIPT',
                                    'CIRCLED', 'PARENTHESIZED', 'SEGMENTED', 'MATHEMATICAL', 'ROMAN NUMERAL',
                                    'FULL STOP', 'COMMA'):
                    if scrypt_type in name:
                        script = '*' + scrypt_type.lower().replace(' ', '-')
                        exclude_from_number_processing = True
                        break
                for scrypt_type in ('VULGAR FRACTION',):
                    if scrypt_type in name:
                        script = scrypt_type.lower().replace(' ', '-')
                        break
                if exclude_from_number_processing:
                    continue
                if isinstance(num, int):
                    value = num
                    if 0 <= num <= 9:
                        num_base = 1
                        base_multiplier = num
                        if "DIGIT" in name:
                            num_type = 'digit'
                        else:
                            # Chinese numbers 零 (0), 一 (1), ... 九 (9) have numeric values,
                            # but are NOT (full) digits
                            num_type = 'digit-like'
                    elif m := regex.match(r'([0-9]+?)(0*)$', str(num)):
                        base_multiplier = int(m.group(1))  # non_base_value(500) = 5
                        num_base = int('1' + m.group(2))
                        num_type = 'base' if base_multiplier == 1 else 'multi'
                    else:
                        num_type = 'other-int'  # Do such cases exist?
                elif ("FRACTION" in name) and (fraction := fraction_char2fraction(char, num, self)):
                    fraction = fraction
                    num_type = 'fraction'
                else:
                    num_type = 'other-num'  # Do such cases exist? Yes. Bengali currency numerators, ...
                value_s = '' if value is None else str(value)
                fraction_s = '' if fraction is None else f'{fraction.numerator}/{fraction.denominator}'
                fraction_list = None if fraction is None else [fraction.numerator, fraction.denominator]
                delimiter_s = ' ' if value_s and fraction_s else ''
                rom = (value_s + delimiter_s + fraction_s) or orig_txt
                add_non_none_to_dict(result_dict, 'txt', orig_txt)
                add_non_none_to_dict(result_dict, 'rom', rom)
                add_non_none_to_dict(result_dict, 'value', value)
                add_non_none_to_dict(result_dict, 'fraction', fraction_list)
                add_non_none_to_dict(result_dict, 'type', num_type)
                if is_large_power:
                    result_dict['is-large-power'] = True
                add_non_none_to_dict(result_dict, 'base', num_base)
                add_non_none_to_dict(result_dict, 'mult', base_multiplier)
                add_non_none_to_dict(result_dict, 'script', script)
                if num_type.startswith('other'):
                    add_non_none_to_dict(result_dict, 'name', name)
                    f_err.write(json.dumps(result_dict) + '\n')
                    n_err += 1
                else:
                    if not script:
                        add_non_none_to_dict(result_dict, 'name', name)
                    f_out.write(json.dumps(result_dict) + '\n')
                    n_out += 1
        sys.stderr.write(f'Processed {codepoint} codepoints,\n  wrote {n_out} lines to {out_filename}\n'
                         f'    and {n_err} lines to {err_filename}\n')

    def load_resource_files(self, data_dir: Path, load_log: bool = False,
                            rebuild_ud_props: bool = False, rebuild_num_props: bool = False):
        """Loads all resource files needed for romanization."""
        data_dir = data_dir
        if not isinstance(data_dir, Path):
            sys.stderr.write(f'Error: data_dir is of {type(data_dir)}, not a Path.\n'
                             f'       Cannot load any resource files.\n')
            return
        self.load_rom_file(os.path.join(data_dir, "romanization-auto-table.txt"),
                           'ud', file_format='rom', load_log=load_log)
        self.load_rom_file(os.path.join(data_dir, "UnicodeDataOverwrite.txt"),
                           'ow', file_format='u2r', load_log=load_log)
        self.load_rom_file(os.path.join(data_dir, "romanization-table.txt"),
                           'man', file_format='rom', load_log=load_log)
        self.load_chinese_pinyin_file(os.path.join(data_dir, "Chinese_to_Pinyin.txt"), load_log=load_log)
        self.load_script_file(os.path.join(data_dir, "Scripts.txt"), load_log=load_log)
        self.load_num_props(os.path.join(data_dir, "NumProps.jsonl"), load_log=load_log)
        for base_file in ("UnicodeDataProps.txt", "UnicodeDataPropsCJK.txt", "UnicodeDataPropsHangul.txt"):
            self.load_unicode_data_props(os.path.join(data_dir, base_file), load_log=load_log)
        if rebuild_ud_props:
            self.rebuild_unicode_data_props(os.path.join(data_dir, "UnicodeDataProps.txt"),
                                            cjk=os.path.join(data_dir, "UnicodeDataPropsCJK.txt"),
                                            hangul=os.path.join(data_dir, "UnicodeDataPropsHangul.txt"))
        if rebuild_num_props:
            self.rebuild_num_props(os.path.join(data_dir, "NumProps.jsonl"),
                                   os.path.join(data_dir, "NumPropsRejects.jsonl"))

    def unicode_hangul_romanization(self, s: str, pass_through_p: bool = False):
        """Special algorithmic solution to convert (Korean) Hangul characters to the Latin alphabet."""
        if cached_rom := self.hangul_rom.get(s, None):
            return cached_rom
        leads = "g gg n d dd r m b bb s ss - j jj c k t p h".split()
        vowels = "a ae ya yae eo e yeo ye o wa wai oe yo u weo we wi yu eu yi i".split()
        tails = "- g gg gs n nj nh d l lg lm lb ls lt lp lh m b bs s ss ng j c k t p h".split()
        result = ""
        for c in s:
            cp = ord(c)
            if 0xAC00 <= cp <= 0xD7A3:
                code = cp - 0xAC00
                lead_index = int(code / (28 * 21))
                vowel_index = int(code / 28) % 21
                tail_index = code % 28
                rom = leads[lead_index] + vowels[vowel_index] + tails[tail_index]
                rom = rom.replace('-', '')
                self.hangul_rom[c] = rom
                result += rom
            elif pass_through_p:
                result += c
        return result

    @staticmethod
    def char_is_nonspacing_mark(s) -> bool:
        """ Checks whether a character is a nonspacing mark, e.g. combining accents, points, vowel signs"""
        return (len(s) == 1) and (ud.category(s) == 'Mn')

    @staticmethod
    def char_is_format_char(s) -> bool:
        """ Checks whether a character is a formatting character, e.g. a zero-with joiner/non-joiner"""
        return (len(s) == 1) and (ud.category(s) == 'Cf')

    @staticmethod
    def char_is_space_separator(s) -> bool:
        """ Checks whether a character is a space,
            e.g. ' ', non-breakable space, en space, ideographic (Chinese) space, Ogham space mark
            but excluding \t, \r, \n"""
        return (len(s) == 1) and (ud.category(s) == 'Zs')

    def chr_name(self, char: str) -> str:
        try:
            return ud.name(char)
        except (ValueError, TypeError):
            if name := self.dict_str[('name', char)]:
                return name
        return ''

    def num_value(self, s: str) -> int | float | Fraction | None:
        """rom_rules include numeric values beyond UnicodeData.txt, e.g. for Egyptian numerals"""
        for rom_rule in self.rom_rules[s]:
            if (num := rom_rule['num']) is not None:
                return num
        return None

    def rom_rule_value(self, s: str, key: str):
        for rom_rule in self.rom_rules[s]:
            if (value := rom_rule.get(key)) is not None:
                return value
        return None

    def unicode_float2fraction(self, num: float, precision: float = 0.000001) -> Tuple[int, int] | None:
        """only for common unicode fractions"""
        if cached_value := self.float2fraction.get(num, None):
            return cached_value
        for numerator in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11):
            for denominator in (2, 3, 4, 5, 6, 8, 12, 16, 20, 32, 40, 64, 80, 160, 320):
                if abs(numerator / denominator - num) < precision:
                    result = numerator, denominator
                    self.float2fraction[num] = result
                    return result
        return None

    def chr_script_name(self, char: str) -> str:
        """For letters, diacritics, numerals etc."""
        return self.dict_str[('script', char)]

    def test_output_of_selected_scripts_and_rom_rules(self):
        """Low level test function that checks and displays romanization information."""
        output = ''
        for s in ("Oriya", "Chinese"):
            d = self.scripts[s.lower()]
            output += f'SCRIPT {s} {d}\n'
        for s in ('ƿ', 'β', 'и', 'μπ', '⠹', '亿', 'ちょ', 'и', '𓍧', '正', '分之', 'ऽ', 'ศ', 'ด์', 'ढ़', 'ड़'):
            d = self.rom_rules[s]
            output += f'DICT {s} {d}\n'
        for s in ('ƿ', 'β', 'न', 'ु'):
            output += f'SCRIPT-NAME {s} {self.chr_script_name(s)}\n'
        for s in ('万', '\uF8F7', '\U00013368', '\U0001308B', '\u0E48', '\u0E40'):
            name = self.chr_name(s)
            num = self.dict_num[s]
            pic = self.dict_str[('pic', s)]
            tone_mark = self.dict_str[('tone-mark', s)]
            syllable_info = self.dict_str[('syllable-info', s)]
            is_large_power = self.dict_bool[('is-large-power', s)]
            output += f'PROPS {s}'
            if name:
                output += f'  name: {name}'
            if num:
                output += f'  num: {num} ({type(num).__name__})'
            if pic:
                output += f'  pic: {pic}'
            if tone_mark:
                output += f'  tone-mark: {tone_mark}'
            if syllable_info:
                output += f'  syllable-info: {syllable_info}'
            if is_large_power:
                output += f'  is-large-power: {is_large_power}'
            output += '\n'
        mayan12 = '\U0001D2EC'
        egyptian600 = '𓍧'
        runic90 = '𐍁'
        klingon2 = '\uF8F2'
        for offset, c in enumerate(f'9九万萬百፲፱፻፸¾0²₂AⅫ⑫൵{runic90}{mayan12}{egyptian600}{klingon2}'):
            output += f'NUM-EDGE: {NumEdge(offset, offset+1, c, self)}\n'
        for s in ('\u00bc', '\u0968'):
            output += f'NUM-PROPS: {self.num_props[s]}\n'
        print(output)

    def test_romanization(self, **args):
        """A few full cases of romanization testing."""
        # noinspection SpellCheckingInspection
        tests = [('ألاسكا', None), ('यह एक अच्छा अनुवाद है.', 'hin'), ('ちょっとまってください', 'kor'),
                 ('Μπανγκαλόρ', 'ell'), ('Зеленський', 'ukr'), ('കേരളം', 'mal')]
        for test in tests:
            s = test[0]
            lcode = test[1] if len(test) >= 2 else None
            rom = self.romanize_string(s, lcode=lcode, **args)
            sys.stderr.write(f'ROM {s} -> {rom}\n')
        n_alerts = 0
        codepoint = -1
        while codepoint < 0xF0000:
            codepoint += 1
            c = chr(codepoint)
            rom = self.romanize_string(c)
            if regex.search(r'\s', rom) and regex.search(r'\S', rom):
                name = self.chr_name(c)
                sys.stderr.write(f'U+{codepoint:04X} {c} {name}  {rom}\n')
                n_alerts += 1
        sys.stderr.write(f'{n_alerts} alerts for roms with spaces\n')

    def romanize_file(self, input_filename: str | None = None, output_filename: str | None = None,
                      lcode: str | None = None, direct_input: List[str] = None, **args):
        """Script to apply romanization to an entire file. Input and output files needed.
        Language code (lcode) recommended."""
        f_in_to_be_closed, f_out_to_be_closed = False, False
        if direct_input and (input_filename is None):
            f_in = direct_input  # list of lines
        elif isinstance(input_filename, str):
            try:
                f_in = open(input_filename, 'r', encoding='utf-8', errors='surrogateescape')
                f_in_to_be_closed = True
            except OSError:
                sys.stderr.write(f'Error in romanize_file: Cannot open file {input_filename}\n')
                f_in = None
            except UnicodeDecodeError:
                sys.stderr.write(f'Error in romanize_file: File not in UTF-8: {input_filename}\n')
                f_in = None
        elif input_filename is None:
            f_in = sys.stdin
        else:
            sys.stderr.write(f"Error in romanize_file: argument 'input_filename' {input_filename} "
                             f"is of wrong type: {type(input_filename)} (should be str)\n")
            f_in = None
        if isinstance(output_filename, str):
            try:
                f_out = open(str(output_filename), 'w', encoding='utf-8')
                f_out_to_be_closed = True
            except OSError:
                sys.stderr.write(f'Error in romanize_file: Cannot write to file {output_filename}\n')
                f_out = None
        elif output_filename is None:
            f_out = sys.stdout
        else:
            sys.stderr.write(f"Error in romanize_file: argument 'output_filename' {output_filename} "
                             f"is of wrong type: {type(output_filename)} (should be str)\n")
            f_out = None
        self.n_non_utf8_characters = 0
        if f_in and f_out:
            max_lines = args.get('max_lines')
            progress_dots_output = False
            try:
                for line_number, line in enumerate(f_in, 1):
                    if non_utf8_chars := regex.findall(r'[\uDC80-\uDCFF]', line):
                        repl_char = '\uFFFD'
                        line2 = regex.sub(r'[\uDC80-\uDCFF]', repl_char, line)
                        n_non_utf8_chars = len(non_utf8_chars)
                        self.n_non_utf8_characters += n_non_utf8_chars
                        max_n_error_messages = 10
                        if self.n_error_messages_output < max_n_error_messages:
                            s_ending = '' if n_non_utf8_chars == 1 else 's'
                            sys.stderr.write(f"Detected encoding error: file {input_filename} line {line_number} "
                                             f"contains {n_non_utf8_chars} non-UTF-8 character{s_ending} "
                                             f"(replaced by {repl_char}): {line2.rstrip()}\n")
                            self.n_error_messages_output += 1
                        elif self.n_error_messages_output == max_n_error_messages:
                            sys.stderr.write(f"Too many errors. No further errors reported.\n")
                            self.n_error_messages_output += 1
                        line = line2
                    if m := regex.match(r'(::lcode\s+)([a-z]{3})(\s+)(.*)$', line):
                        lcode_kw, lcode2, space, snt = m.group(1, 2, 3, 4)
                        rom_result = self.romanize_string(snt, lcode2 or lcode, **args)
                        if args.get('rom_format', RomFormat.STR) == RomFormat.STR:
                            lcode_prefix = f"{lcode_kw}{lcode2}{space}"
                            f_out.write(lcode_prefix + rom_result + '\n')
                        else:
                            lcode_prefix = f'[0, 0, "", "lcode: {lcode2}"]'  # meta edge with lcode info
                            prefixed_edges = [lcode_prefix] + self.romanize_string(snt, lcode2 or lcode, **args)
                            f_out.write(Edge.json_str(prefixed_edges) + '\n')
                    else:
                        f_out.write(Edge.json_str(self.romanize_string(line.rstrip('\n'), lcode, **args)) + '\n')
                    if not args.get('silent'):
                        if line_number % 100 == 0:
                            if line_number % 1000 == 0:
                                sys.stderr.write(str(line_number))
                            else:
                                sys.stderr.write('.')
                            progress_dots_output = True
                            sys.stderr.flush()
                            gc.collect()
                    if max_lines and line_number >= max_lines:
                        break
            except UnicodeDecodeError as error:
                sys.stderr.write(f"UnicodeDecodeError: {error}\n")
                sys.stderr.write(f"   Please make sure the input stream is in Unicode (UTF-8).\n")
                sys.stderr.write(f"   Consider setting the encoding to UTF-8,"
                                 f" e.g. set/export PYTHONIOENCODING=UTF-8\n")
                sys.stderr.write(f"   Consider using uroman with -i/--input_filename option, "
                                 f"which is more robust with respect to encoding errors than piping.\n")
            if progress_dots_output:
                sys.stderr.write('\n')
                sys.stderr.flush()
        if f_in_to_be_closed:
            f_in.close()
        if f_out_to_be_closed:
            f_out.close()
        if input_filename and self.n_non_utf8_characters:
            sys.stderr.write(f"Total number of non-UTF-8 characters in {input_filename}: "
                             f"{self.n_non_utf8_characters}\n")

    @staticmethod
    def apply_any_offset_to_cached_rom_result(cached_rom_result: str | List[Edge], offset: int = 0) \
            -> str | List[Edge]:
        if isinstance(cached_rom_result, str):
            return cached_rom_result
        elif offset == 0:
            return cached_rom_result
        else:
            return [Edge(edge.start + offset, edge.end + offset, edge.txt, edge.type) for edge in cached_rom_result]

    @staticmethod
    def decode_unicode_escapes(s: str) -> str:
        if regex.search(r'\\[xuU][0-9A-Fa-f]{2}', s):
            result = ''
            rest = s
            while m := regex.match(r'(.*?)(\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8})(.*)$', rest):
                pre, core, rest = m.group(1, 2, 3)
                cp = int(core[2:], 16)
                # escape only for non-ASCII, specifically not for \x22, \x25 (quote, apostrophe)
                if cp > 0x80:
                    result += pre + chr(cp)
                else:
                    result += pre + core
            result += rest + ('\n' if s.endswith('\n') else '')
            return result
        else:
            return s

    def romanize_string_core(self, s: str, lcode: str | None, rom_format: RomFormat, offset: int = 0, **args) \
            -> str | List[Edge]:
        """Script to support token-by-token romanization with caching for higher speed."""
        if self.cache_p:
            cached_rom = self.rom_cache.get((s, lcode, rom_format))
            if cached_rom is not None:
                return self.apply_any_offset_to_cached_rom_result(cached_rom, offset)
        lat = Lattice(s, uroman=self, lcode=lcode)
        lat.pick_tibetan_vowel_edge(**args)
        lat.prep_braille(**args)
        lat.add_romanization(**args)
        lat.add_numbers(self, **args)
        lat.add_braille_numbers(**args)
        lat.add_rom_fall_back_singles(**args)
        if rom_format == RomFormat.LATTICE:
            all_edges = lat.all_edges(0, len(s))
            lat.add_alternatives(all_edges)
            if self.rom_cache_size < self.rom_max_cache_size:
                self.rom_cache[(s, lcode, rom_format)] = all_edges
                self.rom_cache_size += 1
            result = self.apply_any_offset_to_cached_rom_result(all_edges, offset)
        else:
            best_edges = lat.best_rom_edge_path(0, len(s))
            if rom_format in (RomFormat.EDGES, RomFormat.ALTS):
                if rom_format == RomFormat.ALTS:
                    lat.add_alternatives(best_edges)
                if self.rom_cache_size < self.rom_max_cache_size:
                    self.rom_cache[(s, lcode, rom_format)] = best_edges
                    self.rom_cache_size += 1
                result = self.apply_any_offset_to_cached_rom_result(best_edges, offset)
            else:
                rom = lat.edge_path_to_surf(best_edges)
                del lat
                if self.rom_cache_size < self.rom_max_cache_size:
                    self.rom_cache[(s, lcode, rom_format)] = rom
                    self.rom_cache_size += 1
                result = rom
        return result

    def romanize_string(self, s: str, lcode: str | None = None, rom_format: RomFormat = RomFormat.STR, **args) \
            -> str | List[Edge]:
        """Main entry point for romanizing a string. Recommended argument: lcode (language code).
        recursive only used for development.
        Method returns a string or a list of edges (with start and end offsets)."""
        lcode = lcode or args.get('lcode', None)
        # print('rom::', s, 'lcode:', lcode, 'print-lattice:', print_lattice_p)

        if args.get('decode_unicode'):
            s = self.decode_unicode_escapes(s)
        # with caching (for string format output only for now)
        if self.cache_p:
            rest, offset = s, 0
            result = '' if rom_format == RomFormat.STR else []
            while m3 := regex.match(r'(.*?)([.,; ]*[ 。་][.,; ]*)(.*)$', rest):
                pre, delimiter, rest = m3.group(1, 2, 3)
                result += self.romanize_string_core(pre, lcode, rom_format, offset, **args)
                offset += len(pre)
                result += self.romanize_string_core(delimiter, lcode, rom_format, offset, **args)
                offset += len(delimiter)
            result += self.romanize_string_core(rest, lcode, rom_format, offset, **args)
            return result
        else:
            return self.romanize_string_core(s, lcode, rom_format, 0, **args)


class Edge:
    """This class defines edges that span part of a sentence with a specific romanization.
    There might be multiple edges for a given span. The edges in turn are part of the
    romanization lattice."""
    def __init__(self, start: int, end: int, s: str, annotation: str = None):
        self.start = start
        self.end = end
        self.txt = s
        self.type = annotation

    def __str__(self):
        return f'[{self.start}-{self.end}] {self.txt} ({self.type})'

    def __repr__(self):
        return str(self)

    def json(self) -> str:  # start - end - text - annotation
        return json.dumps([self.start, self.end, self.txt, self.type])

    @staticmethod
    def json_str(rom_result: List[Edge] | str) -> str:
        if isinstance(rom_result, str):
            return rom_result
        else:
            result = '['
            for edge in rom_result:
                if isinstance(edge, Edge):
                    result += edge.json()
                else:
                    result += str(edge)
            result += ']'
            return result


class NumEdge(Edge):
    def __init__(self, start: int, end: int, s: str, uroman: Uroman | None, active: bool = False):
        """For NumEdge, the s argument is in original language (not yet romanized)."""
        # For speed, much of this processing should at some point be cached in data files.
        Edge.__init__(self, start, end, s)
        self.orig_txt, self.txt = s, s
        self.value, self.fraction, self.num_base, self.base_multiplier = None, None, None, None
        self.type, self.script, self.is_large_power, self.active = None, None, False, active
        self.n_decimals = None
        self.value_s = None     # precision for 3.14159265358979323846264338327950288419716939937510582097494
        if start+1 == end:
            char = s[0]
            if d := uroman.num_props.get(char):
                self.active = True
                self.value = d.get('value')
                fraction_list = d.get('fraction')
                self.fraction = Fraction(fraction_list[0], fraction_list[1]) if fraction_list else None
                self.num_base = d.get('base')
                self.base_multiplier = d.get('mult')
                self.type = d.get('type')
                self.script = d.get('script')
                self.is_large_power = d.get('is-large-power')
                self.update()

    def update(self,
               value: int | float | None = None,
               value_s: str | None = None,
               fraction: Fraction | None = None,
               n_decimals: int | None = None,
               num_base: int | None = None,
               base_multiplier: int | float | None = None,
               script: str | None = None,
               e_type: str | None = None,
               orig_txt: str | None = None) -> str:
        self.value = first_non_none(value, self.value)
        self.value_s = first_non_none(value_s, self.value_s)
        self.fraction = first_non_none(fraction, self.fraction)
        self.n_decimals = first_non_none(n_decimals, self.n_decimals)
        self.num_base = first_non_none(num_base, self.num_base)
        self.base_multiplier = first_non_none(base_multiplier, self.base_multiplier)
        self.script = first_non_none(script, self.script)
        self.type = first_non_none(e_type, self.type)
        self.orig_txt = first_non_none(orig_txt, self.orig_txt)
        if self.value_s is not None:
            value_s = self.value_s
        elif self.value is None:
            value_s = ''
        elif isinstance(self.value, float) and (self.n_decimals is not None):
            value_s = first_non_none(self.value_s, f'{self.value:0.{self.n_decimals}f}')
        else:
            value_s = str(self.value)
        fraction_s = '' if self.fraction is None else f'{self.fraction.numerator}/{self.fraction.denominator}'
        delimiter_s = ' ' if value_s and fraction_s else ''
        self.txt = (value_s + delimiter_s + fraction_s) or self.orig_txt
        return self.txt

    def __str__(self):
        if self.num_base is not None:
            if self.base_multiplier is not None:
                b_clause = f'{self.base_multiplier}*{self.num_base}'
            else:
                b_clause = str(self.num_base)
        else:
            b_clause = None
        return (('' if self.active else ' *')
                + f'[{self.start}-{self.end}] {self.orig_txt} R:{self.txt} T:{self.type}'
                + (' LP' if self.is_large_power else '')
                + (f' B:{b_clause}' if (b_clause is not None) else '')
                + (f' V:{self.value}' if ((self.value is not None) and (str(self.value) != self.txt)) else '')
                + (f' VS:{self.value_s}' if ((self.value_s is not None) and (self.value_s != self.txt)) else '')
                + (f' F:.{self.n_decimals}f' if self.n_decimals else f'')
                + (f' S:{self.script}' if self.script else ''))


class Lattice:
    """Lattice for a specific romanization instance. Has edges."""
    def __init__(self, s: str, uroman: Uroman, lcode: str = None):
        self.s = s
        self.lcode = lcode
        self.lattice = defaultdict(set)
        self.max_vertex = len(s)
        self.uroman = uroman
        self.props = {}
        self.simple_top_rom_cache = {}
        self.contains_script = defaultdict(bool)
        self.check_for_scripts()

    def check_for_scripts(self):
        for c in self.s:
            script_name = self.uroman.chr_script_name(c)
            self.contains_script[script_name] = True
            if regex.search(r'[\u2800-\u28FF]', self.s):
                self.contains_script['Braille'] = True

    def add_edge(self, edge: Edge):
        self.lattice[(edge.start, edge.end)].add(edge)
        self.lattice[(edge.start, 'right')].add(edge.end)
        self.lattice[(edge.end, 'left')].add(edge.start)

    def __str__(self):
        edges = []
        for start in range(self.max_vertex):
            for end in self.lattice[(start, 'right')]:
                for edge in self.lattice[(start, end)]:
                    edges.append(f'[{start}-{end}] {edge.txt} ({edge.type})')
        return ' '.join(edges)

    @staticmethod
    def char_is_braille(c: str) -> bool:
        return 0x2800 <= ord(c[0]) <= 0x28FF

    # Help Tibet
    def char_is_subjoined_letter(self, c: str) -> bool:
        return "SUBJOINED LETTER" in self.uroman.chr_name(c)

    def char_is_regular_letter(self, c: str) -> bool:
        char_name = self.uroman.chr_name(c)
        return ("LETTER" in char_name) and not ("SUBJOINED" in char_name)

    def char_is_letter(self, c: str) -> bool:
        return "LETTER" in self.uroman.chr_name(c)

    def char_is_vowel_sign(self, c: str) -> bool:
        return self.uroman.dict_bool[('is-vowel-sign', c)]

    def char_is_letter_or_vowel_sign(self, c: str) -> bool:
        return self.char_is_letter(c) or self.char_is_vowel_sign(c)

    def is_at_start_of_word(self, position: int) -> bool:
        # return not regex.match(r'(?:\pL|\pM)', self.s[position-1:position])
        first_char = self.s[position]
        first_char_is_braille = self.char_is_braille(first_char)
        end = position
        if (preceded_by_alpha := self.props.get(('preceded_by_alpha', end), None)) in (True, False):
            return not preceded_by_alpha
        if end:
            prev_orig_letter = self.s[end-1:end]
            if prev_orig_letter.isalpha():
                self.props[('preceded_by_alpha', position)] = True
                return False
        for start in self.lattice[(end, 'left')]:
            for edge in self.lattice[(start, end)]:
                prev_letter = None if edge.txt == '' else edge.txt[-1]
                if len(edge.txt) and (prev_letter.isalpha() or (first_char_is_braille and (prev_letter in ["'"]))):
                    self.props[('preceded_by_alpha', position)] = True
                    return False
        self.props[('preceded_by_alpha', position)] = False
        return True

    def is_at_end_of_word(self, position: int) -> bool:
        if (cached_followed_by_alpha := self.props.get(('followed_by_alpha', position), None)) in (True, False):
            return not cached_followed_by_alpha
        start = position
        if start < self.max_vertex:
            next_orig_letter = self.s[start:start+1]
            if next_orig_letter.isalpha():
                self.props[('followed_by_alpha', position)] = True
                return False
        while (start+1 < self.max_vertex) \
                and self.uroman.char_is_nonspacing_mark(self.s[start]) \
                and ('NUKTA' in self.uroman.chr_name(self.s[start])):
            start += 1
        for end in range(start + 1, self.max_vertex + 1):
            s = self.s[start:end]
            if not self.uroman.dict_bool[('s-prefix', s)]:
                break
            for rom_rule in self.uroman.rom_rules[s]:
                rom = rom_rule['t']
                if (not rom_rule['use-only-at-start-of-word']) and regex.search(r'\pL', rom):
                    self.props[('followed_by_alpha', position)] = True
                    return False
        self.props[('followed_by_alpha', position)] = False
        return True

    def is_at_end_of_syllable(self, position: int) -> Tuple[bool, str]:
        """At least initially for Thai"""
        prev_char = self.s[position-2] if position >= 2 else None
        # char = self.s[position-1] if position >= 1 else None
        next_char = self.s[position] if position < self.max_vertex else None
        if self.uroman.dict_str[('tone-mark', next_char)]:
            adj_position = position + 1
            next_char = self.s[adj_position] if adj_position < self.max_vertex else None
            # print('TONE-MARK', position, next_char)
        else:
            adj_position = position
        next_char2 = self.s[adj_position + 1] if adj_position + 1 < self.max_vertex else None
        if prev_char is None:
            return False, 'start-of-string'
        if not regex.search(r'(?:\pL|\pM)$', prev_char):  # start of token
            return False, 'start-of-token'
        if self.uroman.dict_str[('syllable-info', prev_char)] == 'written-pre-consonant-spoken-post-consonant':
            return False, 'pre-post-vowel-on-left'
        if self.uroman.dict_str[('syllable-info', next_char)] == 'written-pre-consonant-spoken-post-consonant':
            return True, 'pre-post-vowel-on-right'
        if adj_position >= self.max_vertex:  # end of string
            return True, 'end-of-string'
        # if not self.char_is_letter_or_vowel_sign(next_char):  # end of token
        if not regex.match(r'(?:\pL|\pM)', next_char):  # end of token
            return True, 'end-of-token'
        if position > 0:
            left_edge = self.best_left_neighbor_edge(position-1)
            if left_edge and regex.search(r'[bcdfghjklmnpqrstvxz]$', left_edge.txt):
                return False, 'consonant-to-the-left'
        next_char_rom = first_non_none(self.simple_top_romanization_candidate_for_span(adj_position,
                                                                                       adj_position + 2,
                                                                                       simple_search=True),
                                       self.simple_top_romanization_candidate_for_span(adj_position,
                                                                                       adj_position + 1,
                                                                                       simple_search=True),
                                       "?")
        if not regex.match(r"[aeiou]", next_char_rom.lower()):  # followed by consonant
            return True, f'not-followed-by-vowel {next_char_rom}'
        if (next_char == '\u0E2D') and (next_char2 is not None):  # THAI CHARACTER O ANG
            next_char2_rom = first_non_none(self.simple_top_romanization_candidate_for_span(adj_position+1,
                                                                                            adj_position+2,
                                                                                            simple_search=True),
                                            "?")
            if regex.match(r"[aeiou]", next_char2_rom.lower()):
                return True, 'o-ang-followed-by-vowel'  # In that context Thai char. "o ang" is considered a consonant
        return False, 'not-at-syllable-end-by-default'

    def romanization_by_first_rule(self, s) -> str | None:
        try:
            return self.uroman.rom_rules[s][0]['t']
        except IndexError:
            return None

    def expand_rom_with_special_chars(self, rom: str, start: int, end: int, **args) \
            -> Tuple[str, int, int, str | None]:
        """This method contains a number of special romanization heuristics that typically modify
        an existing or preliminary edge based on context."""
        orig_start = start
        uroman = self.uroman
        full_string = self.s
        annot = None
        if rom == '':
            return rom, start, end, None
        prev_char = (full_string[start-1] if start >= 1 else '')
        first_char = full_string[start]
        last_char = full_string[end-1]
        next_char = (full_string[end] if end < len(full_string) else '')
        # \u2820 is the Braille character indicating that the next letter is upper case
        if (prev_char == '\u2820') and regex.match(r'[a-z]', rom):
            return rom[0].upper() + rom[1:], start-1, end, 'rom exp'
        # noinspection SpellCheckingInspection   Normalize multi-upper case THessalonike -> Thessalonike,
        # noinspection SpellCheckingInspection   but don't change THESSALONIKE
        if start+1 == end and rom.isupper() and next_char.islower():
            ablation = args.get('ablation', '')     # VERBOSE
            if not ('nocap' in ablation):
                rom = rom.capitalize()
        # Japanese small tsu (and Gurmukhi addak) used as consonant doubler:
        if (prev_char and prev_char in 'っッ\u0A71') \
                and (uroman.chr_script_name(prev_char) == uroman.chr_script_name(prev_char)) \
                and (m_double_consonant := regex.match(r'(ch|[bcdfghjklmnpqrstwz])', rom)):
            # return m_double_consonant.group(1).replace('ch', 't') + rom, start-1, end, 'rom exp'
            # expansion might additional apply to the right
            if prev_char in 'っッ':  # for Japanese, per Hepburn, use tch
                rom = m_double_consonant.group(1).replace('ch', 't') + rom
            else:
                rom = m_double_consonant.group(1).replace('ch', 'c') + rom
            start = start-1
            first_char = full_string[start]
            prev_char = (full_string[start-1] if start >= 1 else '')
        # Thai
        if uroman.chr_script_name(first_char) == 'Thai':
            if (start+1 == end) and regex.match(r'[bcdfghjklmnpqrstvwxyz]+$', rom):
                if uroman.dict_str[('syllable-info', prev_char)] == 'written-pre-consonant-spoken-post-consonant':
                    for vowel_prefix_len in [1]:
                        if vowel_prefix_len <= start:
                            for vowel_suffix_len in [3, 2, 1]:
                                if end + vowel_suffix_len <= len(full_string):
                                    pattern = (full_string[start-vowel_prefix_len: start]
                                               + '–'
                                               + full_string[end:end+vowel_suffix_len])
                                    if uroman.rom_rules[pattern]:
                                        vowel_rom_rule = uroman.rom_rules[pattern][0]
                                        vowel_rom = vowel_rom_rule['t']
                                        # print(f" PATTERN {pattern} ({full_string[start:end]}/{rom}) {rom}{vowel_rom}")
                                        return rom + vowel_rom, start-vowel_prefix_len, end+vowel_suffix_len, 'rom exp'
            if (uroman.chr_script_name(prev_char) == 'Thai') \
                    and (uroman.dict_str[('syllable-info', prev_char)]
                         == 'written-pre-consonant-spoken-post-consonant') \
                    and regex.match(r'[bcdfghjklmnpqrstvwxyz]', rom) \
                    and (vowel_rom := self.romanization_by_first_rule(prev_char)):
                return rom + vowel_rom, start-1, end, 'rom exp'
            # THAI CHARACTER O ANG
            if (first_char == '\u0E2D') and (end - start == 1):
                prev_script = uroman.chr_script_name(prev_char)
                next_script = uroman.chr_script_name(next_char)
                prev_rom = self.find_rom_edge_path_backwards(0, start, 1, return_str=True)
                next_rom = self.romanization_by_first_rule(next_char)
                # if not recursive:
                #     lc = uroman.romanize_string(full_string[:start], lcode=self.lcode, recursive=True)
                #     rc = uroman.romanize_string(full_string[end:], lcode=self.lcode, recursive=True)
                #     print('PP', start, end, prev_script, next_script, prev_rom, next_rom, '  LC:', lc[-40:],
                #           '  RC:', rc[:40])
                # delete THAI CHARACTER O ANG unless it is surrounded on both sides by a Thai consonant
                if not ((prev_script == 'Thai') and (next_script == 'Thai')
                        and regex.match(r'[bcdfghjklmnpqrstvwxz]+$', prev_rom)
                        and regex.match(r'[bcdfghjklmnpqrstvwxz]+$', next_rom)):
                    # if not recursive:
                    #     print(f'* DELETE O ANG {first_char} {start}-{end}   LC: {lc[-40:]}  RC: {rc[:40]}')
                    return '', start, end, 'rom del'
        # Coptic: consonant + grace-accent = e + consonant
        if next_char and (next_char == "\u0300") and (uroman.chr_script_name(last_char) == "Coptic")\
                and (not self.simple_top_romanization_candidate_for_span(orig_start, end+1)):
            rom = 'e' + rom
            end = end+1
            last_char = full_string[end - 1]
            next_char = (full_string[end] if end < len(full_string) else '')
            annot = 'rom exp'
        # Japanese small y: ki + small ya = kya etc.
        y_rom = None
        if (next_char and next_char in 'ゃゅょャュョ') \
                and (uroman.chr_script_name(last_char) == uroman.chr_script_name(next_char)) \
                and regex.search(r'([bcdfghjklmnpqrstvwxyz]i$)', rom) \
                and (y_rom := self.romanization_by_first_rule(next_char)) \
                and (not self.simple_top_romanization_candidate_for_span(orig_start, end+1)) \
                and (not self.simple_top_romanization_candidate_for_span(start, end+1)):
            rom = rom[:-1] + y_rom
            end = end+1
            last_char = full_string[end - 1]
            next_char = (full_string[end] if end < len(full_string) else '')
            annot = 'rom exp'
        # Japanese vowel lengthener (U+30FC)
        last_rom_char = last_chr(rom)
        if (next_char == 'ー') \
                and (uroman.chr_script_name(last_char) in ('Hiragana', 'Katakana')) \
                and (last_rom_char in 'aeiou'):
            return rom + last_rom_char, start, end+1, 'rom exp'
        # Virama (in Indian languages)
        if self.uroman.dict_bool[('is-virama', next_char)]:
            return rom, start, end + 1, "rom exp"
        if rom.startswith(' ') and ((start == 0) or (prev_char == ' ')):
            rom = rom[1:]
        if rom.endswith(' ') and ((end == len(full_string)+1) or (next_char == ' ')):
            rom = rom[:-1]
        return rom, start, end, annot

    def prep_braille(self, **_args) -> None:
        if self.contains_script['Braille']:
            dots6 = '\u2820'  # characters in following word are upper case
            all_caps = False
            for i, c in enumerate(self.s):
                if (i >= 1) and (self.s[i-1] == dots6) and (c == dots6):
                    all_caps = True
                elif all_caps:
                    if c in '\u2800':  # Braille space
                        all_caps = False
                    else:
                        self.props[('is-upper', i)] = True

    def pick_tibetan_vowel_edge(self, **args) -> None:
        if not self.contains_script['Tibetan']:
            return None
        verbose = bool(args.get('verbose'))
        s = self.s
        uroman = self.uroman
        tibetan_syllable = []
        tibetan_letter_positions = []
        for start in range(self.max_vertex):
            c = s[start]
            if (uroman.chr_script_name(c) == 'Tibetan') and self.char_is_letter_or_vowel_sign(c):
                tibetan_letter_positions.append(start)
            else:
                if tibetan_letter_positions:
                    tibetan_syllable.append(tibetan_letter_positions)
                    tibetan_letter_positions = []
        if tibetan_letter_positions:
            tibetan_syllable.append(tibetan_letter_positions)
        for tibetan_letter_positions in tibetan_syllable:
            vowel_pos = None
            orig_txt = ''
            roms = []
            subjoined_letter_positions = []
            first_letter_position = tibetan_letter_positions[0]
            for i in tibetan_letter_positions:
                c = s[i]
                orig_txt += c
                rom = first_non_none(self.simple_top_romanization_candidate_for_span(i, i+1), "?")
                self.props[('edge-vowel', i)] = None
                if self.char_is_vowel_sign(c) or (rom and regex.match(r"[aeiou]+$", rom)):
                    vowel_pos = i
                    self.props[('edge-vowel', i)] = True
                    # delete any syllable initial ' before vowel
                    if roms == ["'"]:
                        self.props[('edge-delete', i-1)] = True
                elif self.char_is_subjoined_letter(c):
                    subjoined_letter_positions.append(i)
                    if i > first_letter_position:
                        if c == "\u0FB0":
                            vowel_pos = i-1
                            self.props[('edge-vowel', i-1)] = True
                        else:
                            self.props[('edge-vowel', i-1)] = False
                    rom = regex.sub(r'([bcdfghjklmnpqrstvwxyz].*)a$', r'\1', rom)
                elif c == "\u0F60":  # Tibetan letter -a, romanized as an apostrophe ("'")
                    self.props[('edge-vowel', i)] = False
                    if i > first_letter_position:
                        vowel_pos = i-1
                        self.props[('edge-vowel', i-1)] = True
                        if i == tibetan_letter_positions[-1]:
                            self.props[('edge-delete', i)] = True
                    if roms and not (roms[-1] in "aeiou"):
                        rom = "a'"
                    else:
                        rom = "'"
                else:
                    rom = regex.sub(r'([bcdfghjklmnpqrstvwxyz].*)a$', r'\1', rom)
                roms.append(rom)
            if vowel_pos is not None:
                for i in tibetan_letter_positions:
                    if self.props.get(('edge-vowel', i)) is None:
                        self.props[('edge-vowel', i)] = False
            else:
                best_cost, best_vowel_pos, best_pre, best_post = math.inf, None, None, None
                n_letters = len(tibetan_letter_positions)
                for i in tibetan_letter_positions:
                    rel_pos = i - first_letter_position
                    pre, post = ''.join(roms[:rel_pos+1]), ''.join(roms[rel_pos+1:])
                    if self.props.get(('edge-vowel', i)) is False:
                        cost = 20
                        if cost < best_cost:
                            best_cost, best_vowel_pos, best_pre, best_post = cost, i, pre, post
                    elif n_letters == 1:
                        cost = 0
                        if cost < best_cost:
                            best_cost, best_vowel_pos, best_pre, best_post = cost, i, pre, post
                    elif n_letters == 2:
                        cost = 0 if i == 0 else 0.1
                        if cost < best_cost:
                            best_cost, best_vowel_pos, best_pre, best_post = cost, i, pre, post
                    else:
                        good_suffix = regex.match(r"(?:|[bcdfghjklmnpqrstvwxz]|bh|bs|ch|cs|dd|ddh|"
                                                  r"dh|dz|dzh|gh|gr|gs|kh|khs|kss|n|nn|nt|ms|ng|ngs|ns|ph|"
                                                  r"rm|sh|ss|th|ts|tsh|tt|tth|zh|zhs)'?$", post)
                        # noinspection SpellCheckingInspection
                        good_prefix = regex.match(r"'?(?:.|bd|br|brg|brgy|bs|bsh|bst|bt|bts|by|bz|bzh|"
                                                  r"ch|db|dby|dk|dm|dp|dpy|dr|"
                                                  r"gl|gn|gr|gs|gt|gy|gzh|kh|khr|khy|kr|ky|ld|lh|lt|mkh|mny|mth|mtsh|"
                                                  r"ny|ph|phr|phy|rgy|rk|el|rn|rny|rt|rts|"
                                                  r"sk|skr|sky|sl|sm|sn|sny|sp|spy|sr|st|th|ts|tsh)$", pre)
                        subjoined_suffix = all([x in subjoined_letter_positions
                                                for x in tibetan_letter_positions[rel_pos+2:]])
                        # print('GOOD', good_suffix, good_prefix, subjoined_suffix, f'{pre}a{post}',
                        #       subjoined_letter_positions, tibetan_letter_positions[rel_pos+2:])
                        if good_suffix and good_prefix:
                            cost = len(pre) * 0.1
                        elif good_suffix:
                            cost = len(pre)
                        elif subjoined_suffix and good_prefix:
                            cost = len(pre) * 0.3
                        elif subjoined_suffix:
                            cost = len(pre) * 0.5
                        else:
                            cost = math.inf
                    if cost < best_cost:
                        best_cost, best_vowel_pos, best_pre, best_post = cost, i, pre, post
                if best_vowel_pos is not None:
                    for i in tibetan_letter_positions:
                        if self.props.get(('edge-vowel', i)) is None:
                            value = (i == best_vowel_pos)
                            self.props[('edge-vowel', i)] = value
                if verbose:
                    best_cost = best_cost if isinstance(best_cost, int) else round(best_cost, 2)
                    sys.stderr.write(f'Tib. best cost: "{best_pre}a{best_post}"  o:{orig_txt}  c:{round(best_cost, 2)}'
                                     f'   p:{best_vowel_pos} {tibetan_letter_positions}\n')

    def add_default_abugida_vowel(self, rom: str, start: int, end: int, annotation: str = '') -> str:
        """Adds an abugida vowel (e.g. "a") where needed. Important for many languages in South Asia."""
        uroman = self.uroman
        s = self.s
        # noinspection PyBroadException
        try:
            first_s_char = s[start]
            last_s_char = s[end-1]
            script_name = uroman.chr_script_name(first_s_char)
            script = self.uroman.scripts[script_name.lower()]
            if not (abugida_default_vowels := script['abugida-default-vowels']):
                return rom
            key = (script, rom)
            if key in uroman.abugida_cache:
                base_rom, base_rom_plus_vowel, mod_rom = uroman.abugida_cache[key]
                rom = mod_rom
            else:
                vowels_regex1 = '|'.join(abugida_default_vowels)   # e.g. 'a' or 'a|o'
                vowels_regex2 = '|'.join(map(lambda x: x + '+', abugida_default_vowels))   # e.g. 'a+' or 'a+|o+'
                # noinspection SpellCheckingInspection
                if m := regex.match(fr'([cfghkmnqrstxy]?y)({vowels_regex2})-?$', rom):
                    base_rom = m.group(1)
                    base_rom_plus_vowel = base_rom + m.group(2)
                elif m := regex.match(fr'([bcdfghjklmnpqrstvwxyz]+)({vowels_regex1})-?$', rom):
                    base_rom = m.group(1)
                    base_rom_plus_vowel = base_rom + m.group(2)
                    if rom.endswith('-') and (start+1 == end) and rom[0].isalpha():
                        rom = rom[:-1]
                else:
                    base_rom = rom
                    base_rom_plus_vowel = base_rom + abugida_default_vowels[0]
                if (not regex.match(r"[bcdfghjklmnpqrstvwxyz]+$", base_rom)
                        and (not ((script_name == 'Tibetan') and (base_rom == "'")))):
                    base_rom, base_rom_plus_vowel = None, None
                uroman.abugida_cache[key] = (base_rom, base_rom_plus_vowel, rom)
            if base_rom is None:
                return rom
            if 'tail' in annotation:
                return rom
            prev_s_char = s[start-1] if start >= 1 else ''
            next_s_char = s[end] if len(s) > end else ''
            next2_s_char = s[end+1] if len(s) > end+1 else ''
            if script_name == 'Tibetan':
                if self.props.get(('edge-delete', start)):
                    return ''
                elif self.props.get(('edge-vowel', start)):
                    return base_rom_plus_vowel
                else:
                    return base_rom
            if (next_s_char and ((base_rom in "bcdfghklmnpqrstvwz") or (base_rom in ["ng"]))
                    and (next_s_char in "យ")):  # Khmer yo
                return base_rom
            if self.uroman.dict_bool[('is-vowel-sign', next_s_char)]:
                return base_rom
            if self.uroman.dict_bool[('is-medial-consonant-sign', next_s_char)]:
                return base_rom
            if self.char_is_subjoined_letter(next_s_char):
                return base_rom
            if self.uroman.char_is_nonspacing_mark(next_s_char) \
                    and self.uroman.dict_bool[('is-vowel-sign', next2_s_char)]:
                return base_rom
            if self.uroman.dict_bool[('is-virama', next_s_char)]:
                return base_rom
            if self.uroman.char_is_nonspacing_mark(next_s_char) \
                    and self.uroman.dict_bool[('is-virama', next2_s_char)]:
                return base_rom
            if self.uroman.dict_bool[('is-virama', prev_s_char)]:
                return base_rom_plus_vowel
            if self.is_at_start_of_word(start) and not regex.search('r[aeiou]', rom):
                return base_rom_plus_vowel
            # delete many final schwas from most Devanagari languages (except: Sanskrit)
            if self.is_at_end_of_word(end):
                if (script_name in ("Devanagari",)) and (self.lcode not in ('san',)):  # Sanskrit
                    return rom
                elif self.lcode in ('asm', 'ben', 'guj', 'kas', 'pan'):
                    return rom
                else:
                    return base_rom_plus_vowel
            if uroman.chr_script_name(prev_s_char) != script_name:
                return base_rom_plus_vowel
            if 'VOCALIC' in self.uroman.chr_name(last_s_char):
                return base_rom
            if uroman.chr_script_name(next_s_char) == script_name:
                return base_rom_plus_vowel
        except Exception:
            return rom
        else:
            pass
            # print('ABUGIDA', rom, start, script_name, script, abugida_default_vowels, prev_s_char, next_s_char)
        return rom

    def cand_is_valid(self, rom_rule: RomRule, start: int, end: int, rom: str) -> bool:
        if rom is None:
            return False
        if rom_rule['dont-use-at-start-of-word'] and self.is_at_start_of_word(start):
            return False
        if rom_rule['use-only-at-start-of-word'] and not self.is_at_start_of_word(start):
            return False
        if rom_rule['dont-use-at-end-of-word'] and self.is_at_end_of_word(end):
            return False
        if rom_rule['use-only-at-end-of-word'] and not self.is_at_end_of_word(end):
            return False
        if rom_rule['use-only-for-whole-word'] \
                and not (self.is_at_start_of_word(start) and self.is_at_end_of_word(end)):
            return False
        if (lcodes := rom_rule['lcodes']) and (self.lcode not in lcodes):
            return False
        return True

    # @profile
    def simple_sorted_romanization_candidates_for_span(self, start, end) -> List[str]:
        s = self.s[start:end]
        if not self.uroman.dict_bool[('s-prefix', s)]:
            return []
        rom_rule_candidates = []
        for rom_rule in self.uroman.rom_rules[s]:
            rom = rom_rule['t']
            if self.cand_is_valid(rom_rule, start, end, rom):
                rom_rule_candidates.append((rom_rule['n-restr'] or 0, rom_rule['t']))
        rom_rule_candidates.sort(reverse=True)
        return [x[1] for x in rom_rule_candidates]

    def simple_top_romanization_candidate_for_span(self, start, end, simple_search: bool = False) -> str | None:
        if (start < 0) or (end > self.max_vertex):
            return None
        span_range = (start, end)
        if (cached_result := self.simple_top_rom_cache.get(span_range)) is not None:
            return cached_result
        best_cand, best_n_restr, best_rom_rule = None, None, None
        for rom_rule in self.uroman.rom_rules[self.s[start:end]]:
            if self.cand_is_valid(rom_rule, start, end, rom_rule['t']):
                n_restr = rom_rule['n-restr'] or 0
                if best_n_restr is None or (n_restr > best_n_restr):
                    best_cand, best_n_restr, best_rom_rule = rom_rule['t'], n_restr, rom_rule
        if simple_search:
            return best_cand
        if best_rom_rule:
            t_at_end_of_syllable = best_rom_rule['t-at-end-of-syllable']
            # noinspection GrazieInspection
            if t_at_end_of_syllable is not None:
                is_at_end_of_syllable, rationale = self.is_at_end_of_syllable(end)
                if is_at_end_of_syllable:
                    best_cand = t_at_end_of_syllable
                # print(f"   SIMPLE {start}-{end} {best_cand} ({best_rom_rule['t']},{t_at_end_of_syllable}) "
                #       f"END:{is_at_end_of_syllable} ({rationale})")
        self.simple_top_rom_cache[span_range] = best_cand
        # if (best_rom_rule is not None) and ('cancel' in (prov := best_rom_rule['prov'])):
        #     sys.stderr.write(f'   Cancel {self.s} ({start}-{end}) {prov} {self.s[start:end]}\n')
        return best_cand

    def decomp_rom(self, char_position: int) -> str | None:
        """Input: decomposable character such as ﻼ or ½
        Output: la or 1/2"""
        full_string = self.s
        char = full_string[char_position]
        rom = None
        if ud_decomp_s := ud.decomposition(char):
            format_comps = []
            other_comps = []
            decomp_s = ''
            # name = self.uroman.chr_name(char)
            for ud_decomp_elem in ud_decomp_s.split():
                if ud_decomp_elem.startswith("<"):
                    format_comps.append(ud_decomp_elem)
                else:
                    try:
                        norm_char = chr(int(ud_decomp_elem, 16))
                    except ValueError:
                        other_comps.append(ud_decomp_elem)
                    else:
                        decomp_s += norm_char
            if (format_comps and (format_comps[0] not in ('<super>', '<sub>', '<noBreak>', '<compat>'))
                    and (not other_comps) and decomp_s):
                rom = self.uroman.romanize_string(decomp_s, self.lcode)
            # make sure to add a space for 23½ -> 23 1/2
            if rom and ud.numeric(char, None):
                rom = rom.replace('⁄', '/')
                if char_position >= 1 and ud.numeric(full_string[char_position-1], None):
                    rom = ' ' + rom
                if (char_position+1 < len(full_string)) and ud.numeric(full_string[char_position+1], None):
                    rom += ' '
        return rom

    def add_romanization(self, **args):
        """Adds a romanization edge to the romanization lattice."""
        for start in range(self.max_vertex):
            for end in range(start+1, self.max_vertex+1):
                if not self.uroman.dict_bool[('s-prefix', self.s[start:end])]:
                    break
                if (rom := self.simple_top_romanization_candidate_for_span(start, end)) is not None:
                    if self.contains_script['Braille'] and (start+1 == end):
                        if self.props.get(('is-upper', start)):
                            rom = rom.upper()
                    edge_annotation = 'rom'
                    if regex.match(r'\+(m|ng|n|h|r)', rom):
                        rom, edge_annotation = rom[1:], 'rom tail'
                    new_rom = self.add_default_abugida_vowel(rom, start, end, annotation=edge_annotation)
                    if new_rom.startswith(rom):
                        suffix = new_rom[len(rom):]
                        if suffix and regex.match(r'[aeiou]+$', suffix):
                            edge_annotation += f' c:{rom} s:{suffix}'
                    rom = new_rom
                    # orig_rom, orig_start, orig_end = rom, start, end
                    rom, start2, end2, exp_edge_annotation \
                        = self.expand_rom_with_special_chars(rom, start, end, annotation=edge_annotation,
                                                             recursive=args.get('recursive', False), **args)
                    edge_annotation = exp_edge_annotation or edge_annotation
                    # if (orig_rom, orig_start, orig_end) != (rom, start, end):
                    #     print(f"EXP {s} {orig_rom} {orig_start}-{orig_end} -> {rom} {start}-{end}")
                    # if rom != rom_orig: print('** Add ABUGIDA', rom, start, end, rom2)
                    self.add_edge(Edge(start2, end2, rom, edge_annotation))
            if start < len(self.s):
                char = self.s[start]
                cp = ord(char)
                # Korean Hangul characters
                if 0xAC00 <= cp <= 0xD7A3:
                    if rom := self.uroman.unicode_hangul_romanization(char):
                        self.add_edge(Edge(start, start+1, rom, 'rom'))
                # character decomposition
                if rom_decomp := self.decomp_rom(start):
                    self.add_edge(Edge(start, start + 1, rom_decomp, 'rom decomp'))

    @staticmethod
    def update_edge_list(edges, new_edge, old_edges) -> List[NumEdge]:
        new_edge_not_yet_added = True
        result = []
        for edge in edges:
            if edge in old_edges:
                edge.active = False
                if new_edge_not_yet_added:
                    result.append(new_edge)
                    new_edge_not_yet_added = False
            else:
                result.append(edge)
        if new_edge_not_yet_added:
            result.append(new_edge)
        return result

    @staticmethod
    def edge_is_digit(edge: Edge | None) -> bool:
        return (isinstance(edge, NumEdge)
                and (edge.value is not None)
                and isinstance(edge.value, int)
                and (edge.type == 'digit')
                and (0 <= edge.value <= 9)
                and (edge.end - edge.start == 1))

    @staticmethod
    def is_gap_null_edge(edge: Edge) -> bool:
        return isinstance(edge, NumEdge) and (edge.orig_txt in ('零', '〇'))

    @staticmethod
    def braille_digit(char: str) -> str | None:
        position = '\u281A\u2801\u2803\u2809\u2819\u2811\u280B\u281B\u2813\u280A'.find(char)  # Braille 0-9
        return str(position) if position >= 0 else None

    def add_braille_number(self, start: int, end: int, txt: str, **_args) -> None:
        new_edge = NumEdge(start, end, txt, self.uroman)
        new_edge.type = 'number'
        self.add_edge(new_edge)

    def add_braille_numbers(self, **_args):
        if self.contains_script['Braille']:
            s = self.s
            num_s, start = '', None
            for i in range(len(s)):
                char = s[i]
                if char == '\u283C':  # number mark
                    if start is None:
                        start = i
                elif (start is not None) and (digit_s := self.braille_digit(char)):
                    num_s += digit_s
                elif (start is not None) and (char == '\u2832'):  # period
                    num_s += '.'
                elif (start is not None) and (char == '\u2802'):  # comma
                    num_s += ','
                elif isinstance(start, int) and (num_s != ''):
                    self.add_braille_number(start, i, num_s)
                    num_s, start = '', None
            if (start is not None) and (num_s != ''):
                self.add_braille_number(start, len(s), num_s)

    # noinspection PyUnboundLocalVariable
    def add_numbers(self, uroman, **args):
        """Adds a numerical romanization edge to the romanization lattice, currently just for digits."""
        verbose = bool(args.get('verbose'))
        s = self.s
        num_edges = []
        for start in range(len(s)):
            char = s[start]
            if uroman.num_props[char]:
                new_edge = NumEdge(start, start + 1, char, uroman)
                num_edges.append(new_edge)
                if verbose:
                    print('NumEdge', new_edge)
                self.add_edge(new_edge)
        # D1 sequence of digits 1234
        for edge in num_edges:
            if self.edge_is_digit(edge) and edge.active:  # and (edge.value != 0):
                n_decimal_points = 0
                n_decimals = None
                new_value_s = str(edge.value)
                sub_edges = [edge]
                prev_edge = edge
                while True:
                    right_edge = self.best_right_neighbor_edge(prev_edge.end)
                    if self.edge_is_digit(right_edge):
                        sub_edges.append(right_edge)
                        new_value_s += str(right_edge.value)
                        if n_decimals is not None:
                            n_decimals += 1
                        prev_edge = right_edge
                    elif ((prev_edge.end < len(s)) and (s[prev_edge.end] == '.') and (n_decimal_points == 0)
                            and (right_edge2 := self.best_right_neighbor_edge(prev_edge.end + 1))
                            and self.edge_is_digit(right_edge2)):
                        if right_edge is None:
                            right_edge = Edge(prev_edge.end, prev_edge.end+1, s[prev_edge.end],
                                              'decimal period')
                            self.add_edge(right_edge)
                        sub_edges.append(right_edge)
                        sub_edges.append(right_edge2)
                        new_value_s += '.' + str(right_edge2.value)
                        n_decimal_points += 1
                        n_decimals = 1
                        prev_edge = right_edge2
                    else:
                        break
                if len(sub_edges) >= 2:
                    new_value = float(new_value_s) if '.' in new_value_s else int(new_value_s)
                    new_edge = NumEdge(sub_edges[0].start, sub_edges[-1].end, str(new_value), uroman, active=True)
                    new_edge.update(value=new_value, value_s=new_value_s, n_decimals=n_decimals, num_base=1, 
                                    e_type='D1', script=sub_edges[-1].script)
                    self.add_edge(new_edge)
                    num_edges = self.update_edge_list(num_edges, new_edge, sub_edges)
                    if verbose:
                        print(new_edge.type, new_edge)
        # G1 combine (*) "single digits" 2*100=200, 3*10= 30
        for edge in num_edges:
            if (isinstance(edge, NumEdge) and edge.active and (edge.num_base == 1)
                    and isinstance(edge.value, int) and (edge.value >= 1)):
                right_edge = self.best_right_neighbor_edge(edge.end, skip_num_edge=False)
                if (right_edge
                        and isinstance(right_edge, NumEdge)
                        and right_edge.active
                        and isinstance(right_edge.value, int)
                        and (right_edge.num_base > 1)
                        and (not right_edge.is_large_power)):
                    new_value = edge.value * right_edge.value
                    new_edge = NumEdge(edge.start, right_edge.end, str(new_value), uroman, active=True)
                    new_edge.update(value=new_value, num_base=right_edge.num_base, e_type='G1',
                                    orig_txt=edge.orig_txt + right_edge.orig_txt,
                                    script=right_edge.script)
                    self.add_edge(new_edge)
                    num_edges = self.update_edge_list(num_edges, new_edge, [edge, right_edge])
                    if verbose:
                        print(new_edge.type, new_edge)
        # G2 combine (+) G1 "single digits" 200+30+4=234 (within larger blocks of 1000, 1000000)
        for edge in num_edges:
            if isinstance(edge, NumEdge) and edge.active and isinstance(edge.value, int) and not edge.is_large_power:
                sub_edges = [edge]
                prev_edge = edge
                prev_non_edge = edge  # None if (edge.orig_txt in '零') else prev_edge
                right_edge, right_edge2 = None, None
                while (prev_edge
                       and (right_edge := self.best_right_neighbor_edge(prev_edge.end, skip_num_edge=False))
                       and isinstance(right_edge, NumEdge)
                       and right_edge.active
                       and isinstance(right_edge.value, int)
                       and (not right_edge.is_large_power)
                       and (self.is_gap_null_edge(prev_non_edge)
                            or ((prev_non_edge.num_base > right_edge.value)
                                and (prev_non_edge.num_base > right_edge.num_base)))):
                    sub_edges.append(right_edge)
                    prev_edge = right_edge
                    if not self.is_gap_null_edge(right_edge):
                        prev_non_edge = right_edge
                if len(sub_edges) >= 2:
                    new_value = sum([e.value for e in sub_edges])
                    new_edge = NumEdge(sub_edges[0].start, sub_edges[-1].end, str(new_value), uroman, active=True)

                    new_edge.update(value=new_value, num_base=sub_edges[-1].num_base, e_type='G2',
                                    orig_txt=''.join([e.orig_txt for e in sub_edges]),
                                    script=sub_edges[-1].script)
                    self.add_edge(new_edge)
                    num_edges = self.update_edge_list(num_edges, new_edge, sub_edges)
                    new_edge.type = 'G2'
                    if verbose:
                        print(new_edge.type, new_edge)
        # G3 combine (*) G2 blocks with large powers, e.g. 234*1000 = 234000
        for edge in num_edges:
            if (isinstance(edge, NumEdge) and edge.active and (not edge.is_large_power)
                    and (isinstance(edge.value, int) or isinstance(edge.value, float))):
                right_edge = self.best_right_neighbor_edge(edge.end, skip_num_edge=False)
                if (right_edge
                        and isinstance(right_edge, NumEdge)
                        and right_edge.active
                        and isinstance(right_edge.value, int)
                        and (right_edge.num_base > 1)
                        and right_edge.is_large_power):
                    new_value = round(edge.value * right_edge.value, 5)
                    if isinstance(new_value, float) and new_value.is_integer():
                        new_value = int(new_value)
                    new_edge = NumEdge(edge.start, right_edge.end, str(new_value), uroman, active=True)
                    new_edge.update(value=new_value, num_base=right_edge.num_base, e_type='G3',
                                    orig_txt=edge.orig_txt + right_edge.orig_txt,
                                    script=right_edge.script)
                    self.add_edge(new_edge)
                    num_edges = self.update_edge_list(num_edges, new_edge, [edge, right_edge])
                    if verbose:
                        print(new_edge.type, new_edge)
        # G4 combine (+) G3 blocks 234000+567=234567
        for edge in num_edges:
            if isinstance(edge, NumEdge) and edge.active and isinstance(edge.value, int):
                sub_edges = [edge]
                while ((prev_edge := sub_edges[-1])
                       and (right_edge := self.best_right_neighbor_edge(prev_edge.end, skip_num_edge=False))
                       and isinstance(right_edge, NumEdge)
                       and right_edge.active
                       and isinstance(right_edge.value, int)
                       and (prev_edge.num_base > right_edge.value)
                       and (prev_edge.num_base > right_edge.num_base)):
                    if ((prev_edge.script == 'CJK')
                            and (prev_edge.num_base >= 1000)
                            and ('tag' not in prev_edge.type)
                            and regex.match('10+$', str(prev_edge.num_base))
                            and (1 <= right_edge.value <= 9)
                            and (right_edge.start + 1 == right_edge.end)):
                        new_num_base = prev_edge.num_base // 10
                        new_value = new_num_base * right_edge.value
                        # print('DIGIT TAG', prev_edge, right_edge, new_value)
                        right_edge.value = new_value
                        right_edge.num_base = new_num_base
                        right_edge.type = 'G4tag'
                    sub_edges.append(right_edge)
                if len(sub_edges) >= 2:
                    new_value = sum([e.value for e in sub_edges])
                    new_edge = NumEdge(sub_edges[0].start, sub_edges[-1].end, str(new_value), uroman, active=True)
                    new_edge.update(value=new_value, num_base=sub_edges[-1].num_base, e_type='G4',
                                    orig_txt=''.join([e.orig_txt for e in sub_edges]),
                                    script=sub_edges[-1].script)
                    self.add_edge(new_edge)
                    num_edges = self.update_edge_list(num_edges, new_edge, sub_edges)
                    if verbose:
                        print(new_edge.type, new_edge)

        # G5 (Chinese) fractions, percentages
        for edge in num_edges:
            if edge.value is None:
                continue
            if not isinstance(edge.value, int):
                continue
            for fraction_connector in self.uroman.fraction_connectors:
                fraction_connector_end = edge.end+len(fraction_connector)
                if self.s[edge.end:fraction_connector_end] != fraction_connector:
                    continue
                right_edge = self.best_right_neighbor_edge(fraction_connector_end)
                if right_edge.value is None:
                    continue
                if edge.value == 100:
                    if (isinstance(right_edge.value, int) or isinstance(right_edge.value, float)) and (edge.value >= 0):
                        new_edge = Edge(edge.start, right_edge.end, f'{right_edge.value}%', 'percentage')
                        self.add_edge(new_edge)
                        num_edges = self.update_edge_list(num_edges, new_edge, [edge, right_edge])
                else:
                    if isinstance(right_edge.value, int) and (edge.value > 0):
                        new_edge = NumEdge(edge.start, right_edge.end, f'{right_edge.value}/{edge.value}',
                                           self.uroman, True)
                        new_edge.fraction = Fraction(right_edge.value, edge.value)
                        new_edge.type = 'fraction'
                        self.add_edge(new_edge)
                        num_edges = self.update_edge_list(num_edges, new_edge, [edge, right_edge])

        # G6 plus/minus signs
        for edge in num_edges:
            _left_edge = self.best_left_neighbor_edge(edge.start)
            for minus_sign in self.uroman.minus_signs:
                if self.s[edge.start-len(minus_sign):edge.start] == minus_sign:
                    new_edge = Edge(edge.start-len(minus_sign), edge.end, f'-{edge.txt}', f'{edge.type} -')
                    self.add_edge(new_edge)
            for plus_sign in self.uroman.plus_signs:
                if self.s[edge.start-len(plus_sign):edge.start] == plus_sign:
                    new_edge = Edge(edge.start-len(plus_sign), edge.end, f'+{edge.txt}', f'{edge.type} +')
                    self.add_edge(new_edge)

        # F1
        for edge in num_edges:
            # cushion fractions with spaces as needed: e.g. 23½ -> 23 1/2 or 十一五 -> 11 5
            if isinstance(edge, NumEdge) and regex.match(r'\d', edge.txt):
                left_edge = self.best_left_neighbor_edge(edge.start)
                if left_edge and regex.search(r'\d$', left_edge.txt):
                    if edge.fraction:
                        sep = ' '
                    else:
                        sep = '·'
                    edge.txt = sep + edge.txt

        # exceptions: mostly some single-digit number characters
        for edge in num_edges:
            if (isinstance(edge, NumEdge) and edge.active and (edge.value is not None)
                    and (((edge.value > 1000) and (edge.start + 1 == edge.end))
                         or (edge.orig_txt in '兩參参伍陆陸什')
                         or (edge.orig_txt in ('京兆', )))):
                edge.active = False
        if verbose:  # or (num_edges and any([e.type in ['G1', 'G2', 'G3', 'G4'] for e in num_edges])):
            if num_edges:
                print('actives:')
            for num_edge in num_edges:
                print(num_edge)
        for start in range(len(s)):
            start_char = s[start]
            if (best_edge := self.best_edge_in_span(start, start+1)) and isinstance(best_edge, NumEdge):
                continue
            if (num := ud_numeric(start_char)) is not None:
                name = self.uroman.chr_name(start_char)
                if ("DIGIT" in name) and isinstance(num, int) and (0 <= num <= 9):
                    # if start_char not in '0123456789': print('DIGIT', s[start], num, name)
                    self.add_edge(Edge(start, start + 1, str(num), 'num'))
                else:
                    uroman.stats[('*NUM', start_char, num)] += 1

    def add_rom_fall_back_singles(self, **_args):
        """For characters in the original string not covered by romanizations and numbers,
        add a fallback edge based on type, romanization of single char, or original char."""
        for start in range(self.max_vertex):
            end = start+1
            orig_char = self.s[start]
            if not self.lattice[(start, end)]:
                rom, edge_annotation = orig_char, 'orig'
                if self.uroman.char_is_nonspacing_mark(rom):
                    rom, edge_annotation = '', 'Mn'
                elif self.uroman.char_is_format_char(rom):  # e.g. zero-width non-joiner, zero-width joiner
                    rom, edge_annotation = '', 'Cf'
                elif ud.category(orig_char) == 'Co':
                    rom, edge_annotation = '', 'Co'
                elif rom == ' ':
                    edge_annotation = 'orig'
                # elif self.uroman.char_is_space_separator(rom):
                #     rom, edge_annotation = ' ', 'Zs'
                elif (rom2 := self.simple_top_romanization_candidate_for_span(start, end)) is not None:
                    rom = rom2
                    if regex.match(r'\+(m|ng|n|h|r)', rom):
                        rom = rom[1:]
                    edge_annotation = 'rom single'
                # else the original values still hold: rom, edge_annotation = orig_char, 'orig'
                self.add_edge(Edge(start, end, rom, edge_annotation))

    @staticmethod
    def add_new_edge(old_edges: List[Edge], start: int, end: int, new_rom: str, new_type: str, position: int | None,
                     old_edge_dict: dict)\
            -> None:
        if (start, end, new_rom) not in old_edge_dict:
            new_edge = Edge(start, end, new_rom, new_type)
            if position is None:
                old_edges.append(new_edge)
            else:
                old_edges.insert(position + 1, new_edge)
            old_edge_dict[(start, end, new_rom)] = new_edge
            # print(f'  ALT {start}-{end} {new_rom}')

    def add_alternatives(self, old_edges: List[Edge]) -> None:
        old_edge_dict = {}
        for old_edge in old_edges:
            old_edge_dict[(old_edge.start, old_edge.end, old_edge.txt)] = old_edge
        for position, old_edge in enumerate(old_edges):
            if old_edge.type.startswith('rom-alt'):
                continue   # not old
            start, end = old_edge.start, old_edge.end
            orig_s = self.s[start:end]
            old_rom = old_edge.txt
            if m := regex.search(r'\bc:([a-z]+)\s+s:([a-z]+)\b', old_edge.type):
                old_rom_core, old_rom_suffix = m.group(1, 2)
            else:
                old_rom_core, old_rom_suffix = None, None
                # print(f'    CORE:{old_rom_core} SUFFIX:{old_rom_suffix}')
            # self.lattice[(start, end)]:
            for rom_rule in self.uroman.rom_rules[orig_s]:
                rom_t = rom_rule['t']
                if self.cand_is_valid(rom_rule, start, end, rom_t):
                    rom_alts = rom_rule['t-alts']
                    rom_end_of_syllable = rom_rule['t-at-end-of-syllable']
                    if (rom_t in [old_rom, old_rom_core]) and rom_alts:
                        for rom_alt in rom_alts:
                            if old_rom_suffix and (rom_t == old_rom_core):
                                rom_alt += old_rom_suffix
                            self.add_new_edge(old_edges, start, end, rom_alt, 'rom-alt', position,
                                              old_edge_dict)
                    if (rom_t == old_rom) and rom_end_of_syllable:
                        self.add_new_edge(old_edges, start, end, rom_t, 'rom-alt2', position, old_edge_dict)
                    if rom_end_of_syllable == old_rom:
                        self.add_new_edge(old_edges, start, end, rom_t, 'rom-alt3', position, old_edge_dict)

    def all_edges(self, start: int, end: int) -> List[Edge]:
        result = []
        for start2 in range(start, end):
            for end2 in sorted(list(self.lattice[(start2, 'right')]), reverse=True):
                if end2 <= end:
                    result.extend(self.lattice[(start2, end2)])
                else:
                    break
        return result

    def best_edge_in_span(self, start: int, end: int, skip_num_edge: bool = False) -> Edge | None:
        edges = self.lattice[(start, end)]
        # if len(edges) >= 2: print('Multi edge', start2, end2, self.s[start2:end2], edges)
        decomp_edge, other_edge, rom_edge = None, None, None
        for edge in edges:
            if isinstance(edge, NumEdge):
                if skip_num_edge:
                    continue
                if edge.active:
                    return edge
            if edge.type.startswith('rom decomp'):
                if decomp_edge is None:
                    decomp_edge = edge  # plan C
            elif regex.match(r'(?:rom|num)', edge.type):
                if rom_edge is None:
                    rom_edge = edge  # plan B
            elif other_edge is None:
                other_edge = edge  # plan D
        return rom_edge or decomp_edge or other_edge

    def best_right_neighbor_edge(self, start: int, skip_num_edge: bool = False) -> Edge | None:
        for end in sorted(list(self.lattice[(start, 'right')]), reverse=True):
            if best_edge := self.best_edge_in_span(start, end, skip_num_edge=skip_num_edge):
                return best_edge
        return None

    def best_left_neighbor_edge(self, end: int, skip_num_edge: bool = False) -> Edge | None:
        for start in sorted(list(self.lattice[(end, 'left')])):
            if best_edge := self.best_edge_in_span(start, end, skip_num_edge=skip_num_edge):
                return best_edge
        return None

    def best_rom_edge_path(self, start: int, end: int, skip_num_edge: bool = False) -> List[Edge]:
        """Finds the best romanization edge path through the romanization lattice, including
        non-romanized pieces such as ASCII and non-ASCII punctuation."""
        result = []
        start2 = start
        while start2 < end:
            if best_edge := self.best_right_neighbor_edge(start2, skip_num_edge=skip_num_edge):
                result.append(best_edge)
                start2 = best_edge.end
            else:  # should not happen
                start2 += 1
        return result

    def find_rom_edge_path_backwards(self, start: int, end: int, min_char: int | None = None,
                                     return_str: bool = False, skip_num_edge: bool = False) -> List[Edge] | str:
        """Finds a partial best path on the left from a start position to provide left contexts for
        romanization rules. Can return a string or a list of edges. Is typically used for a short context,
        as specified by min_char."""
        result_edges = []
        rom = ''
        end2 = end
        while start < end2:
            old_end2 = end2
            if new_edge := self.best_left_neighbor_edge(end2, skip_num_edge=skip_num_edge):
                result_edges = [new_edge] + result_edges
                rom = new_edge.txt + rom
                end2 = new_edge.start
            if min_char and len(rom) >= min_char:
                break
            if old_end2 >= end2:
                end2 -= 1
        if return_str:
            return rom
        else:
            return result_edges

    @staticmethod
    def edge_path_to_surf(edges) -> str:
        result = ''
        for edge in edges:
            result += edge.txt
        return result


# @timer
def main():
    """This function provides a user interface, either using argparse for a command line interface,
    or providing direct function calls.
    First, a uroman object will have to created, loading uroman data (directory must be provided,
    listed as default). This only needs to be done once.
    After that you can romanize from file to file, or just romanize a string."""

    parser = argparse.ArgumentParser()
    parser.add_argument('direct_input', nargs='*', type=str)
    parser.add_argument('--data_dir', type=Path, default=None, help='uroman resource dir')
    parser.add_argument('-i', '--input_filename', type=str, help='default: sys.stdin')
    parser.add_argument('-o', '--output_filename', type=str, help='default: sys.stdout')
    parser.add_argument('-l', '--lcode', type=str, default=None,
                        help='ISO 639-3 language code, e.g. eng')
    # parser.add_argument('-f', '--rom_format', type=RomFormat, default=RomFormat.STR, help:'alt: RomFormat.EDGES')
    parser.add_argument('-f', '--rom_format', type=RomFormat, default=RomFormat.STR,
                        choices=list(RomFormat), help="Output format of romanization. 'edges' provides offsets")
    # The remaining arguments are mostly for development and test
    parser.add_argument('--max_lines', type=int, default=None, help='limit uroman to first n lines')
    parser.add_argument('--load_log', action='count', default=0, help='report load stats (boolean)')
    parser.add_argument('--test', action='count', default=0, help='perform/display a few tests')
    parser.add_argument('-d', '--decode_unicode', action='count', default=0,
                        help='decodes Unicode escape notation, e.g. \\u03B4 to δ')
    parser.add_argument('-v', '--verbose', action='count', default=0)
    parser.add_argument('--rebuild_ud_props', action='count', default=0,
                        help='rebuild UnicodeDataProps files (for development mode only)')
    parser.add_argument('--rebuild_num_props', action='count', default=0,
                        help='rebuild NumProps file (for development mode only)')
    parser.add_argument('-c', '--cache_size', type=int, default=DEFAULT_ROM_MAX_CACHE_SIZE,
                        help='for speed')
    parser.add_argument('--silent', action='count', default=0, help='suppress ... progress')
    parser.add_argument('-a', '--ablation', type=str, default='', help='for development mode: nocap')
    parser.add_argument('--stats', action='count', default=0, help='for development mode: numbers')
    parser.add_argument('--ignore_args', action='count', default=0, help='for usage illustration only')
    parser.add_argument(PROFILE_FLAG, type=argparse.FileType('w', encoding='utf-8', errors='ignore'),
                        default=None, metavar='PROFILE-FILENAME', help='(optional output for performance analysis)')
    parser.add_argument('--version', action='version',
                        version=f'uroman {__version__}   last modified: {__last_mod_date__}')
    args = parser.parse_args()
    # copy selected (minor) args from argparse.Namespace to dict
    args_dict = {'rom_format': args.rom_format, 'load_log': args.load_log, 'test': args.test, 'stats': args.stats,
                 'cache_size': args.cache_size, 'max_lines': args.max_lines, 'verbose': args.verbose,
                 'rebuild_ud_props': args.rebuild_ud_props, 'rebuild_num_props': args.rebuild_num_props,
                 'ablation': args.ablation, 'silent': args.silent, 'decode_unicode': args.decode_unicode}
    pr = None
    if args.profile:
        gc.enable()
        gc.set_debug(gc.DEBUG_STATS)
        gc.set_debug(gc.DEBUG_LEAK)
        pr = cProfile.Profile()
        pr.enable()
    # noinspection SpellCheckingInspection
    '''Sample calls:
    uroman.py --help
    uroman.py -i ../test/multi-script.txt -o ../test/multi-script-out2.txt
    uroman.py  < ../test/multi-script.txt  > ../test/multi-script-out2.txt
    uroman.py Игорь
    uroman.py Игорь --lcode ukr
    uroman.py ألاسكا 서울 Καλιφόρνια
    uroman.py ちょっとまってください -f edges
    uroman.py "महात्मा गांधी" -f lattice
    uroman.py สวัสดี --load_log
    uroman.py --test
    uroman.py --ignore_args
    uroman.py Բարեւ -o ../test/tmp-out.txt -f edges
    # In double input cases such as in the line below,
    # the input-file's romanization is sent to stdout, while the direct-input romanization is sent to stderr
    uroman.py ⴰⵣⵓⵍ -i ../test/multi-script.txt > ../test/multi-script-out2.txt
        '''

    if args.ignore_args:
        # minimal calls
        uroman = Uroman(args.data_dir)
        s, s2, s3, s4 = 'Игорь', 'ちょっとまってください', 'ka‍n‍ne', 'महात्मा गांधी  '
        print(s, uroman.romanize_string(s))
        print(s, uroman.romanize_string(s, lcode='ukr'))
        print(s2, Edge.json_str(uroman.romanize_string(s2, rom_format=RomFormat.EDGES)))
        print(s3, Edge.json_str(uroman.romanize_string(s3, rom_format=RomFormat.EDGES)))
        print(s4, Edge.json_str(uroman.romanize_string(s4, rom_format=RomFormat.LATTICE)))
        # Note that ../test/multi-script.txt has several lines starting with ::lcode eng etc.
        # This allows users to select specific language codes to specific lines, overwriting the overall --lcodes
        uroman.romanize_file(input_filename='../test/multi-script.txt',
                             output_filename='../test/multi-script-out3.txt')
    else:
        # build a Uroman object (once for many applications and different scripts and languages)
        # uroman = Uroman(args.data_dir, load_log=args.load_log, rebuild_ud_props=args.rebuild_ud_props,
        #                 rebuild_num_props=args.rebuild_num_props)
        uroman = Uroman(args.data_dir, **args_dict)
        romanize_file_p = (args.input_filename or args.output_filename
                           or not (args.direct_input or args.test or args.ignore_args
                                   or args.rebuild_ud_props or args.rebuild_num_props))
        # Romanize any positional arguments, interpreted as strings to be romanized.
        for s in args.direct_input:
            result = uroman.romanize_string(s.rstrip('\n'), lcode=args.lcode, **args_dict)
            result_json = Edge.json_str(result)
            if romanize_file_p:
                # input from both file/stdin (to file/stdout) and direct-input (to stderr)
                if args.input_filename:
                    sys.stderr.write(result_json + '\n')
                # input from direct-input (but not from file/stdin) to stdout
                # else pass
            # no file/stdin or file/stdout, so we write romanization of direct-input to stdout
            else:
                print(result_json)
        # If provided, apply romanization to an entire file.
        if romanize_file_p:
            uroman.romanize_file(args.input_filename, args.output_filename, lcode=args.lcode,
                                 direct_input=args.direct_input, **args_dict)
        if args.test:
            uroman.test_output_of_selected_scripts_and_rom_rules()
            uroman.test_romanization()
        if uroman.stats and args.stats:
            stats100 = {k: uroman.stats[k] for k in list(dict(uroman.stats))[:100]}
            sys.stderr.write(f'Stats: {stats100} ...\n')
    if args.profile:
        if pr:
            pr.disable()
            ps = pstats.Stats(pr, stream=args.profile).sort_stats(pstats.SortKey.TIME)
            ps.print_stats()
        print(gc.get_stats())


if __name__ == "__main__":
    main()
