# AUTOGENERATED! DO NOT EDIT! File to edit: 02_chars.ipynb (unless otherwise specified).

__all__ = ['chardatadir', 'charname', 'charcategory', 'charsubcategory', 'charblock', 'blockfamily', 'unicodeblocks',
           'unicodecategories', 'unicodefamilies', 'NormChange', 'NormResult', 'TextNormalizer']

# Cell
import pandas as pd
pd.options.display.max_rows = 100
pd.options.display.max_columns = 50

# Cell
from .core import *

# Cell
chardatadir = config.libdata / "chars"

# Cell
from unicodedata import name as unicodename
from unicodedata import category as unicodecategory

unicodeblocks = pd.read_csv(chardatadir / "unicode_blocks.csv", sep=";")
unicodeblocks["StartNum"] = unicodeblocks["Start"].map(lambda hex:int(hex,16))
unicodeblocks["EndNum"] = unicodeblocks["End"].map(lambda hex:int(hex,16))

unicodecategories = pd.read_csv(chardatadir / "unicode_categories.csv", sep=";")

unicodefamilies = pd.read_csv(chardatadir / "unicode_families.csv", sep=";")

def charname(char):
    return unicodename(char,f"Char {ord(char)}").title()

def charcategory(char):
    catcode = unicodecategory(char)
    catrow = unicodecategories.loc[unicodecategories["Code"]==catcode]
    return catrow["Category"].iloc[0]

def charsubcategory(char):
    catcode = unicodecategory(char)
    catrow = unicodecategories.loc[unicodecategories["Code"]==catcode]
    return catrow["Subcategory"].iloc[0]

def charblock(char):
    charnum = ord(char)
    blockrow = unicodeblocks[(unicodeblocks["StartNum"]<= charnum) & (unicodeblocks["EndNum"]>= charnum)]
    return blockrow["Block"].iloc[0]

def blockfamily(block):
    filteredseries = unicodefamilies[unicodefamilies["UnicodeBlock"]==block]
    if len(filteredseries) == 0:
        family= "Other"
    else:
        family = filteredseries["CharFamily"].iloc[0]
    return family

# Cell
class NormChange():

    def __init__(self, layer, index, charsInput, charsOutput, removedInfo=None):
        self.layer, self.index, self.input, self.output, self.removedInfo = layer, index, charsInput, charsOutput, removedInfo

    def __repr__(self):
        return f"{self.layer} - {self.index} : {self.input} => {self.output}"

# Cell
class NormResult():

    def __init__(self, inputText, transformsDescs):
        self.input, self.transforms = inputText, transformsDescs
        self.layerChanges = None
        self.output = ""

    def addChange(self, layer, index, charsInput, charsOutput, removedInfo=None):
        if self.layerChanges is None:
            self.layerChanges = []
        if layer > (len(self.layerChanges)-1):
            for i in range(0,layer-len(self.layerChanges)+1):
                self.layerChanges.append([])
        changes = self.layerChanges[layer]
        change = NormChange(layer,index,charsInput,charsOutput,removedInfo)
        changes.append(change)

    def describeChanges(self):
         if self.layerChanges is None:
            return 'No change'
         else:
            desc = StringIO()
            previousString = self.input
            for changes in self.layerChanges:
                if len(changes) == 0: continue
                layer = changes[0].layer
                layerDesc = self.transforms[layer]
                desc.write(layerDesc+"\n")
                dispInparts = []
                outparts = []
                dispOutparts = []
                lastIndex = 0
                for change in changes:
                    if change.index > lastIndex:
                        samePart = previousString[lastIndex:change.index]
                        dispInparts.append(samePart)
                        outparts.append(samePart)
                        dispOutparts.append(samePart)
                    dispInpart = change.input
                    outpart = change.output
                    dispOutpart = outpart
                    if len(dispInpart)>len(outpart):
                        dispOutpart = outpart + ("_"*(len(dispInpart)-len(outpart)))
                    elif len(outpart)>len(dispInpart):
                        dispInpart = dispInpart + (" "*(len(outpart)-len(dispInpart)))
                    dispInparts.append(' ['+dispInpart+'] ')
                    outparts.append(outpart)
                    dispOutparts.append(' ['+dispOutpart+'] ')
                    lastIndex = change.index + len(change.input)
                if lastIndex < len(previousString):
                    samePart = previousString[lastIndex:]
                    dispInparts.append(samePart)
                    outparts.append(samePart)
                    dispOutparts.append(samePart)
                previousString = "".join(outparts)
                desc.write(" < ")
                for inpart in dispInparts:
                    desc.write(inpart)
                desc.write('\n')
                desc.write(" < ")
                for outpart in dispOutparts:
                    desc.write(outpart)
                desc.write('\n')
            return desc.getvalue()

    def mapOutputIndexToInput(self,outputIndex):
        inputIndex = outputIndex
        for changes in self.layerChanges:
            outputIndex = inputIndex
            for change in changes:
                if outputIndex < change.index:
                    break
                elif outputIndex > (change.index + len(change.output)):
                    inputIndex = inputIndex + (len(change.input)-len(change.output))
                else:
                    inputIndex = inputIndex -(outputIndex-change.index)
                    break
        return inputIndex

    def setOutput(self, outputText):
        self.output = outputText

    def __repr__(self):
        return self.output

# Cell
from functools import partial
from operator import itemgetter
from io import StringIO

class TextNormalizer():

    def __init__(self):

        # 1. Load Unicode character set data for latin script
        # 1.1 Frequent encoding errors : windows1252 read as iso8859-1
        dfencodingwin1252 = pd.read_csv(chardatadir / "windows1252-iso8859-errors.csv", sep=";")
        win1252errorchars = {}
        for rowidx,row in dfencodingwin1252.iterrows():
            win1252errorchars[row["Char"]] = row["DecodedChar"]
        # 1.2 Frequent encoding errors : utf8 read as windows1252
        dfencodingutf8 = pd.read_csv(chardatadir / "utf8-windows1252-errors.csv", sep=";")
        utf8errorchars = {}
        for rowidx,row in dfencodingutf8.iterrows():
            utf8errorchars[row["ErrorSubstring"]] = row["DecodedChar"]
        utf8errorshdict = self.buildhierarchicaldict(utf8errorchars)
        # 1.3 Frequent encoding errors : windows1252 read as utf8
        dfencodingwin1252utf8 = pd.read_csv(chardatadir / "windows1252-utf8-errors.csv", sep=";")
        win1252utf8errorchars = {}
        for rowidx,row in dfencodingwin1252utf8.iterrows():
            win1252utf8errorchars[row["Char"]] = row["DecodedChars"]
        # 1.4 Unicode combining chars
        dfcombiningchars = pd.read_csv(chardatadir / "combiningdiacritics.csv", sep=";")
        combiningchars = {}
        for rowidx,row in dfcombiningchars.iterrows():
            combiningchars[row["BaseChar"]+row["Char"]] = row["CombinedChar"]
        combiningcharshdict = self.buildhierarchicaldict(combiningchars)
        # 1.5 Control chars
        dfcontrolchars = pd.read_csv(chardatadir / "controlchars.csv", sep=";")
        dfcontrolchars.loc[0,"Char"] = chr(0) # chr(0) can't be saved in CSV file
        controlchars = set(dfcontrolchars["Char"])
        # 1.6 Latin letter symbols
        dflatinsymbols = pd.read_csv(chardatadir / "latinsymbols.csv", sep=";")
        latinlettersnolayout = {}
        latinlettersremovedlayout = {}
        for rowidx,row in dflatinsymbols.iterrows():
            latinlettersnolayout[row["Char"]] = row["NormString"]
            latinlettersremovedlayout[row["Char"]] = row["Layout"]
        # 1.7 Latin letters
        dflatinletters = pd.read_csv(chardatadir / "latinletters.csv", sep=";")
        latinletterstoupper = {}
        for rowidx,row in dflatinletters.iterrows():
            if row["Char"] != row["UpperChar"]:
                latinletterstoupper[row["Char"]] = row["UpperChar"]
        latinlettersnodiacritics = {}
        latinlettersremoveddiacritics = {}
        for rowidx,row in dflatinletters.iterrows():
            if row["IsDiacritic"]:
                latinlettersnodiacritics[row["Char"]] = row["BaseChar"]
                latinlettersremoveddiacritics[row["Char"]] = row["Diacritics"]
        latinlettersnoligatures = {}
        for rowidx,row in dflatinletters.iterrows():
            if row["IsLigature"]:
                latinlettersnoligatures[row["Char"]] = row["MultiChars"]
        # 1.8 Latin numbers and number symbols
        dflatinnumbers = pd.read_csv(chardatadir / "latinnumbers.csv", sep=";")
        latinnumbersnolayout = {}
        latinnumbersremovedlayout = {}
        for rowidx,row in dflatinnumbers.iterrows():
            if rowidx < 10:
                continue
            latinnumbersnolayout[row["Char"]] = row["NormString"]
            latinnumbersremovedlayout[row["Char"]] = row["Layout"]
        # 1.9 Variations on frequent chars to normalize
        dfnormchars = pd.read_csv(chardatadir / "normalizedchars.csv", sep=";")
        normalizedchars = {}
        for rowidx,row in dfnormchars.iterrows():
            normalizedchars[row["Char"]] = row["NormChar"]
        # 1.10 Optional replacement of cyrillic and greek chars looking like latin letters
        dfcgnormchars = pd.read_csv(chardatadir / "cyrillic-greek-chars.csv", sep=";")
        cgnormalizedchars = {}
        for rowidx,row in dfcgnormchars.iterrows():
            cgnormalizedchars[row["Char"]] = row["NormChar"]
        # 1.11 Final supported french charset
        dfsupportedchars = pd.read_csv(chardatadir / "charset-fr.csv", sep=";", quotechar='"')
        dfsupportedchars.loc[0,"Char"] = chr(0) # chr(0) can't be saved in CSV file
        supportedchars = set(dfsupportedchars["Char"])

        # 2.1 List successive transformations
        self.transformsDescs = []
        transforms = []
        self.transformsDescs.append("Fix encoding errors : windows1252 read as iso8859-1")
        transforms.append(partial(self.replacechars1to1, 0, win1252errorchars))
        self.transformsDescs.append("Fix encoding errors : utf8 read as windows1252")
        transforms.append(partial(self.replacecharsNto1, 1, utf8errorshdict))
        self.transformsDescs.append("Fix encoding errors :  windows1252 read as utf8")
        transforms.append(partial(self.replacechars1toN, 2, win1252utf8errorchars))
        self.transformsDescs.append("Merge Unicode combining chars")
        transforms.append(partial(self.replacecharsNto1, 3, combiningcharshdict))
        self.transformsDescs.append("Ignore control chars")
        transforms.append(partial(self.ignorechars, 4, controlchars))
        self.transformsDescs.append("Replace latin letter symbols")
        transforms.append(partial(self.replacechars1toN, 5, latinlettersnolayout))
        self.transformsDescs.append("Replace latin letter ligatures")
        transforms.append(partial(self.replacechars1toN, 6, latinlettersnoligatures))
        self.transformsDescs.append("Replace latin number symbols")
        transforms.append(partial(self.replacechars1toN, 7, latinnumbersnolayout))
        self.transformsDescs.append("Normalize equivalent chars")
        transforms.append(partial(self.replacechars1to1, 8, normalizedchars))
        self.transformsDescs.append("Replace cyrillic and greek chars looking like latin letters")
        transforms.append(partial(self.replacechars1to1,9, cgnormalizedchars))
        self.transformsDescs.append("Replace infrequent chars : latin letters with diacritics")
        transforms.append(partial(self.replacecharsnotinset, 10, supportedchars, latinlettersnodiacritics))
        self.transformsDescs.append("Replace infrequent chars : other scripts")
        transforms.append(partial(self.replaceotherscripts, 11, supportedchars))
        self.transformsDescs.append("Replace infrequent chars : symbols")
        transforms.append(partial(self.replacesymbols, 12, supportedchars))
        self.transformsDescs.append("Replace infrequent chars : chars to ignore")
        transforms.append(partial(self.ignoreotherchars, 13, supportedchars))

        # 2.2 Combine all transformations
        def func(x,y):
            ci = transforms[0](x,y)
            for transform in transforms[1:]:
                ci = transform(ci,y)
            return ci
        self.transformsFunc = func

    def __repr__(self):
        desc = StringIO()
        for idx,transformDesc in enumerate(self.transformsDescs):
            desc.write(f'{idx+1} - {transformDesc}\n')
        return desc.getvalue()

    def __call__(self, inputText):
        result = NormResult(inputText, self.transformsDescs)
        result.setOutput(self.tostring(self.transformsFunc(inputText,result)))
        return result

    @staticmethod
    def buildhierarchicaldict(idict):
        hdict = {}
        odicts = []
        for key in idict:
            if len(key) > 1:
                firstchar = key[0]
                remainingstring = key[1:]
                if not firstchar in hdict:
                    newdict = {}
                    hdict[firstchar] = newdict
                    odicts.append((firstchar,newdict))
                hdict[firstchar][remainingstring] = idict[key]
        for pkey,odict in odicts:
            dictwithlongkey = False
            for key in odict:
                if len(key)>1:
                    dictwithlongkey = True
                    break
            if dictwithlongkey:
                hdict[pkey] = TextNormalizer.buildhierarchicaldict(odict)
        return hdict

    @staticmethod
    def ignorechars(layer, charset, chariterator, result):
        for index,char in enumerate(chariterator):
            if not char in charset:
                yield char
            else:
                result.addChange(layer, index, char, '')

    @staticmethod
    def replacechars1to1(layer, chardict, chariterator, result):
        for index,char in enumerate(chariterator):
            if char in chardict:
                resChar = chardict[char]
                result.addChange(layer, index, char, resChar)
                yield resChar
            else:
                yield char

    @staticmethod
    def replacechars1toN(layer, chardict, chariterator, result):
        for index,char in enumerate(chariterator):
            if char in chardict:
                resStr = chardict[char]
                result.addChange(layer, index, char, resStr)
                for outchar in resStr:
                    yield outchar
            else:
                yield char

    @staticmethod
    def replacecharsNto1(layer, hierarchicaldict, chariterator, result):
        candidatechars = []
        candidatedicts = []
        for index,char in enumerate(chariterator):
            # Try to match previously started patterns
            if len(candidatechars)>0:
                for idx,candidatedict in enumerate(candidatedicts):
                    if not candidatedict is None:
                        if char in candidatedict:
                            value = candidatedict[char]
                            if isinstance(value,dict):
                                candidatedicts[idx] = value
                            else:
                                # Success : found a char to return
                                for ridx in range(0,idx):
                                    yield candidatechars[ridx]
                                replacedStr = "".join(candidatechars[idx:]) + char
                                result.addChange(layer, index-len(replacedStr)+1, replacedStr, value)
                                candidatechars = []
                                candidatedicts = []
                                char = None
                                yield value
                                break
                        else:
                            candidatedicts[idx] = None
                # Clean oldest failed attemps and return accumulated chars
                while len(candidatedicts)>0 and candidatedicts[0] is None:
                    candidatedicts.pop(0)
                    yield candidatechars.pop(0)
            # Handle the current char
            if not char is None:
                if len(candidatechars)==0:
                    if char in hierarchicaldict:
                        value = hierarchicaldict[char]
                        if isinstance(value,dict):
                            candidatechars.append(char)
                            candidatedicts.append(value)
                        else:
                            result.addChange(layer, index, char, value)
                            yield value
                    else:
                        yield char
                else:
                    candidatechars.append(char)
                    if char in hierarchicaldict:
                        value = hierarchicaldict[char]
                        candidatedicts.append(value)
                    else:
                        candidatedicts.append(None)
        if len(candidatechars)>0:
            for char in candidatechars:
                yield char

    @staticmethod
    def replacecharsnotinset(layer, charset, replacedict, chariterator, result):
        for index,char in enumerate(chariterator):
            if char in charset:
                yield char
            else:
                if char in replacedict:
                    resChar = replacedict[char]
                    result.addChange(layer, index, char, resChar)
                    yield resChar
                else:
                    yield char

    @staticmethod
    def replaceotherscripts(layer, charset, chariterator, result):
        for index,char in enumerate(chariterator):
            if char in charset:
                yield char
            else:
                family = blockfamily(charblock(char))
                if not family in ("Symbols","Ignore"):
                    resStr = chr(65532) + str(ord(char)) + '_'
                    result.addChange(layer, index, char, resStr)
                    for outchar in resStr:
                        yield outchar
                else:
                    yield char

    @staticmethod
    def replacesymbols(layer, charset, chariterator, result):
        for index,char in enumerate(chariterator):
            if char in charset:
                yield char
            else:
                family = blockfamily(charblock(char))
                if family == "Symbols":
                    resStr ='$' + charname(char).replace(' ','') + '_'
                    result.addChange(layer, index, char, resStr)
                    for outchar in resStr:
                        yield outchar
                else:
                    yield char

    @staticmethod
    def ignoreotherchars(layer, charset, chariterator, result):
        for index,char in enumerate(chariterator):
            if char in charset:
                yield char
            else:
                family = blockfamily(charblock(char))
                if family == "Ignore":
                    result.addChange(layer, index, char, '')
                else:
                    yield char

    @staticmethod
    def tostring(iterator):
        return "".join(iterator)