from abc import ABC, abstractmethod

from hanziconv import HanziConv


class BaseTokenizer(ABC):

    @abstractmethod
    def freq(self, tk):
        pass

    @abstractmethod
    def tag(self, tk):
        pass

    @abstractmethod
    def tokenize(self, line):
        pass

    @abstractmethod
    def fine_grained_tokenize(self, tks):
        pass

    @abstractmethod
    def add_tok(self, token, freq, tag):
        pass

    @staticmethod
    def _strQ2B(ustring):
        """Convert full-width characters to half-width characters"""
        rstring = ""
        for uchar in ustring:
            inside_code = ord(uchar)
            if inside_code == 0x3000:
                inside_code = 0x0020
            else:
                inside_code -= 0xFEE0
            if (
                inside_code < 0x0020 or inside_code > 0x7E
            ):  # After the conversion, if it's not a half-width character, return the original character.
                rstring += uchar
            else:
                rstring += chr(inside_code)
        return rstring

    strQ2B = _strQ2B

    @staticmethod
    def _tradi2simp(line):
        return HanziConv.toSimplified(line)

    tradi2simp = _tradi2simp
