"""
.. module:: groups
   :synopsis: Classes for handling Kleio groups and elements.

.. moduleauthor: Joaquim Ramos de Carvalho

Kleio Groups are the building blocks for transcription of historical sources.

The classes in this package can be used to generate Kleio sources from data
fetched from databases or read from csv or json files.

The base classes are:

KKleio
    A Kleio document. It should include a single KSource group.

KSource
    A Historical Source. Can contain a variable number
    of KAct groups.

KAct
    A historical act. Can contain a variable number
    of KPerson or KObject.

In normal usage the basic groups are extended for a particular context:



.. code-block:: python


        from timelink.kleio.groups import KKleio, KSource, KAct,
        KAbstraction, KPerson, KLs, KAtr


        kleio = KKleio

        fonte = KSource.extend('fonte',
                                also=['tipo', 'data', 'ano', 'substitui',
                                        'loc', 'ref', 'obs'])

        lista = KAct.extend('lista',
                            position=['id', 'dia', 'mes', 'ano'],
                            guaranteed=['id', 'ano', 'mes', 'dia'],
                            also=['data', 'tipo', 'loc', 'obs'])

        auc = KAbstraction.extend('auc', position=['name', ''], also=[
            'level', 'id'], guaranteed=['id'])

        n = KPerson.extend('n', position=['nome', 'sexo'], guaranteed=[
            'id', 'nome', 'sexo'], also=['mesmo_que', 'obs'])

        pai = KPerson.extend('pai', position=['nome'], guaranteed=[
            'id', 'nome'], also=['mesmo_que', 'obs'])

        mae = KPerson.extend('mae', position=['nome'], guaranteed=[
            'id', 'nome'], also=['mesmo_que', 'obs'])

        ls = KLs.extend('ls',
                        position=['type', 'value', 'data'],
                        also=['data', 'obs'])

        atr = KAtr.extend(
            'atr', position=['type', 'value', 'data'], also=['data', 'obs'])

        n.allow_as_part(pai)

        n.allow_as_part(mae)



"""
import json
import textwrap
import warnings
from os import linesep as nl
from typing import Any, Union, Type, Tuple, List, Dict

from box import Box

from timelink.kleio.utilities import quote_long_text


class KElement:
    """Represents an Element in Kleio language.

     While *Groups* represent historical entities (people, objects, events)
     *Elements* encapsulate basic items of information (name, gender, date).

    The value of an Element can have three possible "aspects":

     1) "core":  the actual information for the element
     2) "original" (optional), the original wording when relevant
     3) "comment" (optional), a comment of the value.

     Example in Kleio notation::

         person$Joaquim Carvalho%Joachim Carvº#Family name added in the margin

     Can be generated by ::

         n = KElement('name','Joaquim Carvalho',original='Joachim Carvº',
                 comment='Family name added in the margin')
         id = KElement('id','p-jrc')
         person = KPerson(id=id,name=n)

     TODO: does not deal with multiple values in elements.
     Check KElement in MHK where core, comment and original are Vectors
     better to create a KAspect class.

    """

    name: str = None
    core: Any  # must have a str representation.
    comment: str = None
    original: str = None

    @property
    def element_class(self):
        """Return the name of the class of this element."""
        return self.__class__.name

    def __init__(
        self,
        name: str = None,
        core: Any = None,
        comment=None,
        original=None,
        element_class=None,
    ):
        """
        Args:
            name: name of the Element. If None then self._name is used.
            core: the core aspect of the Element. Must have __str__
                    or a tuple (core,comment,original). If a tuple
                    optional arguments are disregarded.
            comment: Optional; The comment aspect of the Element.
            original: Optional; The original aspect of the Element.
            element_class: Optional; in groups generated from kleio
                translations this is set by the translator based
                on the the source=parameter of the str file.
                If absent it is set here to the name of the element.
        """
        if name is not None:
            self.name = name

        if type(core) is tuple and len(core) == 3:
            self.core = core[0]
            self.comment = core[1]
            self.original = core[2]
        else:
            self.core = core
            if comment is not None:
                self.comment = comment
            if original is not None:
                self.original = original

    def __str__(self):
        return self.core

    def __int__(self):
        return int(str(self))

    @classmethod
    def extend(cls, name: str):
        """
        Creates a new KElement class that extends this one.
        This allow creating new KElement
        names for different languages and keep the behaviour of
        a specific element type (e.g. mapping "data" elements
        behave like "date" elements).

        When an element of a group is set with a atomic value
        or a tuple, a new KElement of the class with the same
        name of the element in the group is used to store the
        group.

        :param name:
        :return:
        """
        new_kelement = type(name, (cls,), {})
        new_kelement.name = name
        return new_kelement

    @classmethod
    def get_subclasses(cls):
        """Generator for subclasses of this KElement"""
        for subclass in cls.__subclasses__():
            yield from subclass.get_subclasses()
            yield subclass

    @classmethod
    def all_subclasses(cls):
        """List of all the subclasses of this KElement"""
        return list(cls.get_subclasses())

    @classmethod
    def get_class_for(cls, name: str):
        """
        Search in KElement subclasses and return the one
        with the same element name as the argument.

        if more than one apply return the more specialized (longer __mro__)

        If not found return None

        :param name: name of an element
        :return: KElement or a subclass
        """
        search_list = sorted(
            [(a, len(a.__mro__)) for a in cls.all_subclasses()], key=lambda mro: -mro[1]
        )
        for eclass, mro in search_list:
            if eclass.name == name:
                return eclass
        return None

    @classmethod
    def inherits_from(cls):
        return [ke for ke in cls.__mro__ if ke is not object and ke is not None]

    @classmethod
    def get_classes_for(cls, name: str):
        """Search in Element subclass and return all that
        have the same element name as the argument

        see KElement.get_class_for(name) for getting the more specialized"""
        return [sc for sc in KElement.all_subclasses() if sc.name == name]

    def inherited_names(self):
        """Return the list of names in the KElement subclasses
        this element inherits from including its own"""
        return [
            ke.name
            for ke in self.__class__.__mro__
            if ke is not object and ke.name is not None
        ]

    def extends(self, name: str):
        """True if name is in the the classes this element inherits from"""
        return name in self.inherited_names()

    def is_empty(self):
        """True if all aspects of the element are None or empty string"""
        e = [
            x for x in [self.core, self.comment, self.original] if x is None or x == ""
        ]
        if len(e) == 3:
            return True
        else:
            return False

    def to_tuple(self):
        """Return Element as a tuple (core,comment,original)"""
        return self.core, self.comment, self.original

    def to_kleio(self, name=True, force_show=False):
        """
        Return element as a kleio string: element=core#comment%original
        To avoid rendering the element name set name=False
        If the current element extends the element invisible_
        then this method returns ''. This is useful to present
        some elements to be rendered as part of groups.

        :param name: if True(default) prefix value with self.name=
        :param force_show: if True render element even if it inherits from
                            invisible_
        :return: a valid Kleio representation of this element value
        """
        if self.extends("invisible_") and force_show is False:
            return ""
        if self.is_empty():
            return ""
        c = self.core
        cc = self.comment
        o = self.original
        if c is None:
            c = ""
        else:
            c = quote_long_text(str(c))
        if cc is None:
            cc = ""
        else:
            cc = "#" + quote_long_text(str(cc))
        if o is None:
            o = ""
        else:
            o = "%" + quote_long_text(str(o))
        kleio = c + cc + o
        if name:
            kleio = f"{self.name}={kleio}"
        return kleio

    def to_dict(self, name=False):
        """Return Element as a dict {core:_, comment:_, original:_}
        add name=True to add name to dictionary:
        {name:_, core:_, comment:_, original:_}"""
        if name:
            return {
                "name": self.name,
                "core": self.core,
                "comment": self.comment,
                "original": self.original,
            }
        else:
            return {
                "core": self.core,
                "comment": self.comment,
                "original": self.original,
            }

    def to_dots(self):
        return Box(self.to_dict())


# Default KElement classes.
# element name=id; identification=yes
# element name=type
# element name=loc
# element name=obs
# element name=ref
# element name=value
# element name=origin
# element name=destination
# element name=entity
# element name=same_as
# element name=xsame_as
# element name=name
# element name=sex
# element name=destname
# element name=destination
# element name=summary;
# element name=description;
# element name=replace;

# use this for mixins to mark this should show in Kleio
# still can override with KElement.to_kleio(force_show=True
KleioNoShow = KElement.extend("invisible_")


class KDate(KElement):
    name = "date"

    def __init__(
        self,
        date: Any = None,
        core=None,
        comment=None,
        original=None,
        element_class=None,
    ):
        if core is not None:
            date = core  # to allow core setting in generic code
        super().__init__(self.name, date, comment, original, element_class)


class KDay(KElement):
    name = "day"

    def __init__(
        self,
        day: Any = None,
        core=None,
        comment=None,
        original=None,
        element_class=None,
    ):
        if core is not None:
            day = core  # to allow core setting in generic code
        super().__init__(self.name, day, comment, original, element_class)
        if type(self.core) is str:
            self.core = int(self.core)
        if self.core != 0 and (self.core < 1 or self.core > 31):
            raise ValueError("Day value must be between 1 and 31")


class KMonth(KElement):
    name = "month"

    def __init__(
        self,
        month: Any = None,
        core=None,
        comment=None,
        original=None,
        element_class=None,
    ):
        if core is not None:
            month = core  # to allow core
        super().__init__(self.name, month, comment, original, element_class)

        self.core = int(self.core)

        if type(self.core) is str:
            self.core = int(self.core)
        if self.core != 0 and (self.core < 1 or self.core > 12):
            raise ValueError("Month value must be between 1 and 12")


class KYear(KElement):
    """
    Represents a year.

    To have value checking do KYear.set_limits((lower,upper))

    """

    name = "year"

    def __init__(self, year: Any = None, core=None, comment=None, original=None):
        if core is not None:
            year = core

        super().__init__(self.name, year, comment, original)

        self.core = int(self.core)


class KType(KElement):
    """
    Represents a type of object or abstraction
    """

    name = "type"


class KId(KElement):
    """
    Represents an unique id for a group.

    """

    name = "id"


class KEntityInAttribute(KId, KleioNoShow):
    name = "entity"


class KOriginInRel(KId, KleioNoShow):
    name = "origin"


class KReplace(KElement):
    """
    Represents the id of a group to be replaced.

    Example:

        source$new-id/replace=old-id
    """

    name = "replace"


class KSameAs(KElement):
    """
    Represents the id of a group that describes the
    same real world entity has the one with this element.

    Used in the same file. Translators should check if
    the id corresponds to a group in the same file and
    file an error otherwise

    Example:
        person$Bob Dylan/id=bob-dylan
        .....
        person$Robert Allan Zimmerman/sameas=bob-dylan
    """

    name = "sameas"


class KXSameAs(KElement):
    """
    Same meaning as KSameAs used when id is not
    in the file.

    The difference between KSameAs and KXSameAs is
    just for error checking during translation.
    Translators will raise error if a KSameAs id
    is not found in the same file, but only a warning
    for KXSameAs.

    """

    name = "xsameas"


class KName(KElement):
    """
    Name of person
    """

    name = "name"


class KSex(KElement):
    """
    male / female ...
    """

    name = "sex"


class KDescription(KElement):
    """
    Similar to name, for objects
    """

    name = "description"


class KGroup:
    """
    KGroup(*positional_elements ,**more_elements)

    Abstract Kleio Group.

    To define a Kleio Group extend this class and set default value for _name,
    or use extend(name,position, guaranteed, also).

    Define allowed elements in the default values for
     _position, _guaranteed, _also (list of strings) or call allowed_as_element

    Use _part to list allowed enclosed groups, or call allow_as_part

    For an example see timelink.kleio.groups.KPerson



    """

    # Class scoped list of reserved element names for system use
    _builtin_elements = [
        "line",
        "level",
        "order",
        "inside",
        "groupname",
        "pom_class_id",
        "id",
    ]

    id = None
    _name: str = "kgroup"
    _position: list = []  # list of positional elements
    _guaranteed: list = []  # list of required elements
    _also: list = []  # list of optional elements
    _part: list = []  # allowed sub groups

    _extends: Type["KGroup"]  # name of group this is based on
    _pom_class_id: str = "entity"  # Id of PomSom mapper for this group
    _element_check = True  # if true validates element assignment
    _elementsd: Type[Dict[str, KElement]]  # Current elements
    _inside: Type["KGroup"]  # group that includes this

    # The following fields are generated during the translation process.
    # If the Kleio groups are generated programatically (for instance when
    # importing from csv files or from databases) then the cls.include method
    # will update these values
    _line: int  # line in the source file
    _level: int  # nesting always +1  than enclosing group
    _order: int  # sequential number of this group in the original source
    # class scoped counters to ensure proper numbering across different groups
    _global_sequence: int = 1  # global sequence count
    _global_line: int = 1  # global line count

    # TODO to_kleio_str generates the definition of a group
    #      for a kleio str file. recurse=yes
    #      collects included groups and generates for those also.

    @property
    def kname(self):
        return self.unpack_from_kelement(self._name)

    @kname.setter
    def kname(self, value):
        self._name = self.pack_as_kelement("kname", value, element_class="name")

    @property
    def inside(self):
        return self._inside

    def get_container_id(self):
        if self._inside is None:
            gid = None
        elif KGroup.is_kgroup(self._inside):
            gid = self.unpack_from_kelement(self.inside.id)
        return gid

    @inside.setter
    def inside(self, value):
        if KGroup.is_kgroup(value):
            self._inside = value
        else:
            self._inside = KGroup(id=str(value), element_check=False)

    @property
    def line(self):
        return self.unpack_from_kelement(self._line)

    @line.setter
    def line(self, value):
        self._line = self.pack_as_kelement("line", value)

    @property
    def level(self):
        return self.unpack_from_kelement(self._level)

    @level.setter
    def level(self, value):
        self._level = self.pack_as_kelement("level", value)

    @property
    def order(self):
        return self.unpack_from_kelement(self._order)

    @order.setter
    def order(self, value):
        self._order = self.pack_as_kelement("order", value)

    @property
    def pom_class_id(self):
        return self.unpack_from_kelement(self._pom_class_id)

    @pom_class_id.setter
    def pom_class_id(self, pcid):
        self._pom_class_id = self.pack_as_kelement("pom_class_id")

    @classmethod
    def extend(
        cls,
        name: str,
        position: Union[list, str, None] = None,
        guaranteed: Union[list, str, None] = None,
        also: Union[list, str, None] = None,
        part: Union[list, str, None] = None,
    ):
        """
        Create a new group definition by extending this one
        :param name:  name of the new group
        :param position: list of positional elements
        :param guaranteed: list of required elements
        :param also:    list of optional elements
        :param part: list of groups that can be included in this one
        :return: new group class
        """
        new_group = type(name, (cls,), {})
        new_group._name = name
        # todo: k,v in kwargs if in cls set if not error
        if type(position) is str:
            position = [position]
        if type(guaranteed) is str:
            guaranteed = [guaranteed]
        if type(also) is str:
            also = [also]
        if position is not None:
            new_group._position = position
        else:
            new_group._position = list(cls._position)
        if guaranteed is not None:
            new_group._guaranteed = guaranteed
        else:
            new_group._guaranteed = list(cls._guaranteed)
        if also is not None:
            new_group._also = also
        else:
            new_group._also = list(cls._also)
        if part is not None:
            new_group._part = part
        else:
            new_group._part = list(cls._part)
        new_group._extends = cls

        return new_group

    @classmethod
    def get_subclasses(cls):
        """Generator for subclasses of this group"""
        for subclass in cls.__subclasses__():
            yield from subclass.get_subclasses()
            yield subclass

    @classmethod
    def all_subclasses(cls):
        """List of all the subclasses of this group"""
        return list(cls.get_subclasses())

    @classmethod
    def is_kgroup(cls, g):
        """True g is an instance of a subclass of KGroup"""
        return "KGroup" in [c.__name__ for c in type(g).mro()]

    @classmethod
    def elements_allowed(cls) -> set:
        """Set of  Elements allowed in this Group"""
        return set(cls._guaranteed).union(set(cls._also)).union(cls._position)

    @classmethod
    def allow_as_element(
        cls, ename: Union[str, List[str]], guaranteed=False, also=True, position=None
    ):
        """
        Add element or list to list of allowed elements for this group.
        Optionally define if element(s) is positional, required or optional
        :param ename:  name of element
        :param guaranteed: if True this is element is added to list of
                required elements.
        :param also: if True this element is optional (default)
        :param position: int, this is a positional element, at this position
            (0 = first position)
        :return: List of allowed elements
        """
        if type(ename) is List:
            for e in ename:
                cls.allow_as_element(
                    e, guaranteed=guaranteed, position=position, also=also
                )
            return
        elif type(ename) is str:
            if guaranteed:
                if ename not in cls._guaranteed:
                    cls._guaranteed.append(ename)
            if position is not None:
                if ename in cls._position:
                    cls._position.remove(ename)
                cls._position.insert(position, ename)
            if ename not in cls.elements_allowed():
                cls._also.append(ename)
        else:
            raise TypeError("first argument must be string or list of strings")
        return

    @classmethod
    def allow_as_part(cls, g: Union[str, type]):
        """Allow g to be enclosed as part of this group.

        Arguments:
            g: the name of a group, or a subclass of KGroup.
               A string or class.
        """
        if g not in cls._part:
            cls._part.append(g)

    @classmethod
    def inc_sequence(cls):
        KGroup._global_sequence += 1
        return KGroup._global_sequence

    @classmethod
    def inc_line(cls):
        KGroup._global_line += 1
        return KGroup._global_line

    def __init__(self, *args, **kwargs):
        """
        Creates a new group with elements
        :param args: values for positional elements
        :param kwargs: values for optional elements.
            Use element_check=False to turn off checking of element names
        """
        self._containsd: dict = {}
        self.level = 1
        self.line = 1
        self.order = 1
        self._element_check = True
        self._elementsd = {}
        self._inside = None

        if len(args) > len(self._position):
            raise ValueError("Too many positional elements")
        n = 0
        # set the positional arguments according to "_position"
        for arg in args:
            e = self._position[n]
            # setattr(self, e, KElement(e, arg))
            self[e] = arg  # this will go through __setitem__
            n = n + 1
        # keyword arguments must be in one the element lists
        for (k, v) in kwargs.items():
            if k == "element_check":
                self._element_check = v
            if self._element_check and not self.is_allowed_as_element(k):
                raise ValueError(f"Element not allowed: {k}")
            self[k] = v
        # test if the compulsory (guaranteed) elements are present
        for g in self._guaranteed:
            if getattr(self, g, None) is None:
                raise TypeError(
                    f"{self.kname}: element {g} in _guaranteed "
                    f"is missing or with None value"
                )

    def include(self, group: Type["KGroup"]):
        """Include a group. `group`, or its class, must in _part list or
            extend a class in the part list.

        Returns self so it is possible to chain: g.include(g2).include(g3)"""

        allowed = self.is_allowed_as_part(group)
        if allowed is None:
            raise ValueError(f"Group {self.kname} cannot contain {group.kname}")

        group.level = self.level + 1
        group.line = KGroup.inc_line()
        group.order = KGroup.inc_sequence()

        # Hook to before input processing in the group being included
        if hasattr(group, "before_include") and callable(group.before_include):
            if not group.before_include(self):
                raise TypeError(
                    f"{group} includding aborted by "
                    f"group.before_include(self)"
                    f" returning False"
                )

        # new style, dictionary based
        k = self._containsd.keys()
        if allowed in k:
            self._containsd[allowed].append(group)
        else:
            self._containsd[allowed] = [group]
        group.inside = self
        # Hook to after input processing in the group being included
        if hasattr(group, "after_include") and callable(group.after_include):
            group.after_include(self)
        return self

    def before_include(self, container_group):
        """Method called before this group is included into another
        through KGroup.include(KGroup)."""

        if self.id is None:
            if container_group.id is None:
                raise ValueError(
                    "A group with no id cannot be included in another "
                    "group also without id"
                )
            gid = f"{container_group.id}-{self.order:02d}-{self.kname[0:3]}"
            self["id"] = gid
            return True

        return True

    def after_include(self, group):
        """Method called after a new group is included in this one
        through KGroup.include(KGroup)."""
        pass

    def is_allowed_as_part(self, group):
        """Test if a group can be included in the current one.

        For a group to be allowed for inclusion one of 3 conditions necessary:
            1. the kname of the group is in self._pars
            2. the type of the group is in self._pars
            3. the type of the group inherits from a type in self._pars

        Return key under which the group is allowed (kname, type or super type)
        Return None if not allowed
        """
        if not self.is_kgroup(group):
            raise TypeError("Argument must be subclass of KGroup")
        if group.kname not in self._part:
            allowed_classes = [c for c in self._part if type(c) is not str]
            super_classes = type(group).mro()
            r = list(set(super_classes).intersection(set(allowed_classes)))
            if len(r) == 0:
                allowed = None
            else:
                allowed = r[0]  # kname?
        else:
            allowed = group.kname

        return allowed

    def is_allowed_as_element(self, element_name):
        """
        Test if this element is allowed in this group.

        For an element to be allowed one the following must be true:

            1. part of KGroup._builtin
            2. part of position list
            3. part of guarenteed list
            4. part of also list

        Note that this function is unaffected by self._element_check
        :param element_name: name of element to check

        :return: True if element allowed False otherwise
        """
        return element_name in (
            self._builtin_elements + self._position + self._guaranteed + self._also
        )

    def includes(self, group: Type[Union[str, Type["KGroup"]]] = None) -> list:
        """Returns included groups.

        Groups are returned by the order in _pars.

        TODO this would better be a generator, yield instead of extend

        :param str group: filter by group name
        """
        if group is not None:
            if group in self._containsd.keys():
                return self._containsd[group]
            else:
                if type(group) is str:
                    gname = group
                elif KGroup.is_kgroup(group):
                    gname = group.kname

                inc_by_part_order = []
                classes_in_contains = [
                    c for c in self._containsd.keys() if hasattr(c, "_name")
                ]
                for class_in_contains in classes_in_contains:
                    if class_in_contains._name == gname:
                        inc_by_part_order.extend(self._containsd[class_in_contains])
                return inc_by_part_order
        else:  # no specific subgroup, we return by pars order
            inc_by_part_order = []
            for p in self._part:
                if p in self._containsd.keys():
                    inc_by_part_order.extend(self._containsd[p])

            return inc_by_part_order

    def attr(
        self,
        the_type: Union[str, KElement, Tuple[str, str, str]],
        value: Union[str, KElement, Tuple[str, str, str]],
        date: Union[str, KElement, Tuple[str, str, str]],
        obs=None,
    ):
        """Utility function to include a KAttribute in this KGroup

        The call::

            KGroup.attr('age','25','2021-08-08',obs='in May')

        is short hand for::

            KGroup.include(KAttr('age','25','2021-08-08',obs='in May'))

        Params google style

        :param str or tuple the_type: core or (core,org,comment)
        :param str or tuple value: core or (core,org,comment)
        :param str date: date as string in Kleio format, or (date,org,comment)
        :param str obs: options observation field

        """
        ka = globals()["KAttribute"]
        self.include(ka(the_type, value, date=date, obs=obs))
        return self

    def rel(
        self,
        the_type: Union[str, tuple],
        value: Union[str, tuple],
        destname: Union[str, tuple],
        destination: Union[str, tuple],
        date: Union[str, tuple] = None,
        obs: str = None,
    ):
        """include a relation in this KGroup"""
        kr = globals()["KRelation"]
        self.include(kr(the_type, value, destname, destination, date=date, obs=obs))

    def to_kleio(self, indent="") -> str:
        """Return a kleio representation of the group."""
        return self.__str__(indent=indent, recurse=True)

    def to_dict(
        self,
        allow_none: bool = False,
        include_str: bool = False,
        include_kleio: bool = False,
        redundant_keys: bool = True,
        include_builtin: bool = True,
    ):
        """Return group information as a dict.

        Params:
            allow_none bool = Include null values (default False)

            include_str = include a string represention of the element
                            (with # and % if necessary)

            include_kleio = include a kleio representation of the element
                            element_name=core#original%comment
            include_builtin = include also elements like line,level,... etc..


        Also available as property "get" so that
        group.to_dict()['id'] == group.get['id']

        Format of keys:
        group[element]: core value of element
                        also group[element_core]
        group[element_comment]: comment aspect of element

        group[element_original]: original aspect of element

        group[element_str] : string representation of element
                             (with # and % if necessary)
                             if include_str=True
        group[element_kleio]: kleio representation element=string
                              if include_kleio=True
        group[includes]: list of enclosed groups

        group[includes][subgroup]: list of enclosed groups of type subgroup

        if redundant_keys=True enclose subgroups can also be accessed
        in the plural form if no name conflict with existing elements:

            group['persons'] == group['includes']['persons']

            group['person']['id1'] ==
                [p for p in group['includes']['persons']
                if p.id='id1'][0]

        TODO add first(), last() where includes is allowed:
             group['first']['person'] =
                  group['includes']['person'][0]
        """
        kd = dict()
        kd["kleio_group"] = self._name
        elements_to_include = self.elements_allowed()
        if include_builtin:
            els = elements_to_include.union(set(self._builtin_elements)) - set(
                ["inside"]
            )
        for e in els:
            v: KElement = getattr(self, e, None)
            if v is not None:
                if issubclass(type(v), KElement):
                    core, comment, original = v.to_tuple()
                    kd[e] = core
                    kd[e + "_core"] = core
                    kd[e + "_comment"] = comment
                    kd[e + "_original"] = original
                    if include_str:
                        kd[e + "_str"] = str(v)
                    if include_kleio:
                        kd[e + "_kleio"] = v.to_kleio()
                else:
                    kd[e] = v
        if not allow_none:
            kd = dict([(key, value) for key, value in kd.items() if value is not None])
        # we now includes subgroups
        ki = dict()
        # we now collect subgroups by name
        included = list(self.includes())
        for i in included:
            n = i.kname
            if n not in ki.keys():
                ki[n] = [
                    i.to_dict(
                        include_str=include_str,
                        include_kleio=include_kleio,
                        include_builtin=include_builtin,
                        redundant_keys=redundant_keys,
                    )
                ]
            else:
                ki[n].append(
                    i.to_dict(
                        include_str=include_str,
                        include_kleio=include_kleio,
                        include_builtin=include_builtin,
                        redundant_keys=redundant_keys,
                    )
                )
        if len(ki) > 0:
            kd["includes"] = ki
            # if there are no name conflicts and plural form
            # so g['includes']['act'] can be accessed as
            #    g['acts']
            if redundant_keys:
                for subgroup in ki.keys():
                    if subgroup + "s" not in self.elements_allowed():
                        kd[subgroup + "s"] = ki[subgroup]
                        # we include subgroup indexed by id
                        # so we can have source['act']['ac010]['person']['p01']
                        for group in ki[subgroup]:
                            gid = group.get("id", None)

                            if (
                                gid is not None  # noqa
                                and subgroup not in self.elements_allowed()
                            ):  # noqa

                                if subgroup not in kd.keys():
                                    kd[subgroup] = dict()
                                kd[subgroup][gid] = group
        return kd

    def to_json(self):
        return json.dumps(
            self.to_dict(
                include_str=False,
                include_kleio=False,
                redundant_keys=False,
                include_builtin=True,
            ),
            indent=4,
            allow_nan=False,
        )

    @property
    def get(self):
        return self.to_dict()

    def to_dots(self):
        return Box(self.to_dict())

    @property
    def dots(self):
        """
        Allows easy referencing of the dictionary representation of the group.

        It is very usefull in list comprehensiona, e.g.:

          >> 'Diamantina' in [ls.value for ls in person.dots.lss
                                if ls.type == 'nome-geografico']

          >> [ls.value for ls in person.dots.lss
                        if ls.type == 'nome-geografico']

        :return:
        """
        return self.to_dots()

    def __str__(self, indent="", recurse=False):
        sname = getattr(self, "_name", self.__class__.__name__)
        s = sname + "$"
        first = True
        out = []
        for e in self._position:
            v: KElement = getattr(self, e, None)
            if v is not None:
                if not first:
                    s = s + "/" + v.to_kleio(name=False)
                else:
                    s = s + v.to_kleio(name=False)
                    first = False
                out.append(e)
        more = sorted(
            list(
                set(self._guaranteed)
                .union(set(self._also))
                .union(self._position)
                .difference(out)
            )
        )
        # print(more)
        if "obs" in more:  # we like obs elements at the end
            more.remove("obs")
            more.append("obs")
        for e in more:
            m: Union[KElement, str] = getattr(self, e, None)
            if m is not None and (
                type(m) is str
                and m > ""
                or (issubclass(type(m), KElement) and m.to_kleio() > "")
            ):
                # m contains data, lets output
                if not first:
                    s = s + f"/{m.to_kleio()}"
                else:
                    s = s + f"{m.to_kleio()}"
                    first = False

        if recurse:
            for g in self.includes():
                s = s + nl + g.__str__(indent + " ", recurse=recurse)
        return textwrap.indent(s, indent)

    def __getitem__(self, arg):
        if self._element_check and arg not in self.elements_allowed():
            raise ValueError("Element does not exist in group")
        return getattr(self, arg)

    def __setitem__(self, arg, value):
        if self._element_check and not self.is_allowed_as_element(arg):
            raise ValueError(f"Element not allowed: {arg}")
        el = self.pack_as_kelement(arg, value)
        setattr(self, arg, el)
        self._elementsd[arg] = el

    def pack_as_kelement(self, arg, value):
        kelement = KElement.get_class_for(arg)
        if kelement is None:  # if there is no KElement class we create it
            kelement = KElement.extend(arg)
            warnings.warn(
                f"Created a KElement class for {arg}. "
                f"Better to create explicit or provide "
                f" synonyms= in group creation."
            )
        if not isinstance(value, KElement):  # we did not get a KElement
            # we get KElement class that matches the name
            # this is how we handle localized name of elements that
            # have a builtin meaning or are referred to by standard names
            # in PomSomMapping
            core = None
            comment = None
            original = None
            if type(value) is tuple and len(value) == 3:
                core, comment, original = value
            else:
                core = value
            el = kelement(arg, core=core, comment=comment, original=original)
        else:  # we got a KElement object, ensure it goes with the right class
            el = kelement(
                core=value.core, comment=value.comment, original=value.original
            )
        return el

    def unpack_from_kelement(self, value):
        """
        if value is a KElement return core if not return value as is.

        Useful to obtain the core value in elements that normally have
        no comment or original.

        :param value:
        :return:
        """
        if not isinstance(value, KElement):  # we did not get a KElement
            return value
        else:  # we got a KElement object
            return value.core

    def get_core(self, element, default=None):
        """get_core(element_name [, default])
        Returns the core value of an element
        """
        try:
            core = self.unpack_from_kelement(self[element])
        except Exception:
            core = None
        if core is None:
            return default
        else:
            return core

    def get_id(self):
        """Return the id of the group"""
        return self.unpack_from_kelement(self.id)

    def get_element_for_column(self, colspec, default=None):
        """
        Return the value of an element that matches a specific column in the
        database.

        An element matches a column if its name is equal to the column name
        or if it is an instance of KElement subclass with the same name.

        :param colspec: name of column, or name of POMClassAttributes
        :param default: default value if not element found
        :return: KElement, or whatever in default.

        """
        el: KElement
        for (
            name,
            el,
        ) in (
            self._elementsd.items()
        ):  # same name as column or in ancestors # noqa: E501
            if name == colspec:
                return el
        # Handles synonyms created by subclassing core KElements
        for el in self._elementsd.values():  # if name in inherited names
            if colspec in el.inherited_names():
                return el
        # handles multiple subclassing of core KElements
        for (
            el
        ) in (
            self._elementsd.values()
        ):  # check if there are alternative classes # noqa: E501
            # all classes for colspec
            targets = KElement.get_classes_for(colspec)
            # other classes for el
            alternatives = KElement.get_classes_for(el.name)
            # now check if there is a common path
            #   collect all the ancestors of all the classes for this el
            alt_ancestors = [(sc, sc.inherits_from()) for sc in alternatives]
            #   check if any of colspec classes are there
            for alternative, ancestors in alt_ancestors:
                common = set(ancestors).intersection(set(targets))
                if len(common) > 0:
                    return el
        return default


class KKleio(KGroup):
    """KKleio(structure,prefix=,obs=,translations=,translator=)

    Kleio notation document. Represents a file in Kleio notation.

    Elements:
        structure: The path to a Kleio structure file (default gacto2.str)
        prefix: Prefix to be added to all ids generated from this file
        translations: number of times this file was translated
        translator: name of the translator to be used (currently not used)
        obs: observations

    TODO: Could keep a list of the name of the groups included, at all levels
    """

    _name = "kleio"
    _position = ["structure"]
    _also = ["prefix", "translations", "translator", "obs"]
    _part = ["source", "aregister"]
    _pom_class_id: str = "entity"

    def __init__(self, *args, **kwargs):
        KGroup._global_line = 1
        KGroup._global_sequence = 1
        super().__init__(*args, **kwargs)


class KSource(KGroup):
    """Represents an Historical Source. Sources contain :class:`KAct` and
    may contain :class:`KAttribute`.

    Elements

    id
        An unique id for this source.
    type
        The type of the source (e.g. baptisms, marriages); optional.
    loc
        Location (name of archive, library, collection); optional.
    ref
        The call reference ("cota") of the source in the location; optional.
    date
        The date of the source. A string in timelink format; optional.

          - 1582
          - 1582-05-04
          - 1582:1609
          - >1582:<1702
    year
        A single year. A number. Deprecated, use date instead
    obs
        Observations on the source (can be long and multiline); optional.
    replace
        Id of source to be replaced. A string; optional.
        The source with this id is removed before importing this one.
        Used when changing the id of a file, old id should go here.

    Kleio str definition::

        part name=historical-source;
             guaranteed=id;
             also=type,date,year,loc,ref,obs,replace;
             position=id,year,type,ref;
             part=historical-act


    """

    _name = "source"
    _guaranteed = ["id"]
    _also = ["type", "date", "year", "loc", "ref", "replace", "obs", "kleiofile"]
    _position = ["id"]
    _part = ["act", "attr"]
    _pom_class_id: str = "source"


KKleio.allow_as_part(KSource)


class KAct(KGroup):
    """KAct(id,type,date[,day=,month=,year=,loc=,ref=,obs=])

    An Act is a record of an event in a specific date.

    Elements:
        id: an unique id for this act. A string.
        type: type of the act (baptism, marriage, contract...). A string.
        date: the date of the act. A string in timelink format.
        day,month,year: the date expressed in individual values
        loc: location of the act in the archive (if different from source)
        ref: call number, or page number in source.

    Kleio str definition:

    part 	name=historical-act;
            guaranteed=id,type,date;
            position=id,type,date;
            also=loc,ref,obs,day,month,year;
            arbitrary=person,object,geoentity,abstraction,ls,atr,rel

    """

    _name = "act"
    _guaranteed = ["id", "type", "date"]
    _position = ["id", "type", "date"]
    _also = ["loc", "ref", "obs", "day", "month", "year"]
    _part = ["person", "object", "geoentity", "abstraction", "ls", "attr", "rel"]
    _pom_class_id: str = "act"


KSource.allow_as_part(KAct)


class KPerson(KGroup):
    """KPerson(name,sex,id,obs=,same_as=,xsame_as=)

    Person in a historical source

    Elements:
        name: the name of the person. A string.
        sex:  the gender of the person. A string.
        id: an unique id for this person. A string, optional.
        obs: a note on the person. A string, optional.
        same_as: id of another instance of this person in the same file.
        xsame_as: id of another instance of this person in another file.

    Kleio str definition:

    part	name=person ;
            guaranteed=name,sex;
            also=id,obs,same_as;
            position=name,sex,id,same_as;
            arbitrary=atr,rel,ls
    """

    _name = "person"
    _guaranteed = ["name", "sex"]
    _also = ["id", "obs", "same_as"]
    _position = ["name", "sex", "id", "same_as", "xsame_as"]
    _part = ["rel", "attr"]
    _pom_class_id: str = "person"


KAct.allow_as_part(KPerson)


class KObject(KGroup):
    """KObject(name,type,id=,obs=,same_as=,xsame_as=)

    An object in a historical source.
    Object groups represent physical entities like
    houses, pieces of land, movable objects

    Elements:
        name: the name of the object. A string.
        type:  the . A string.
        id: an unique id for this person. A string, optional.
        obs: a note on the person. A string, optional.
        same_as: id of another instance of this object in the same file.
        xsame_as: id of another instance of this object in another file.

    Kleio str definition:

        part name=object;
             guaranteed=name;
             position=name,type;
             also=obs,id,same_as,xsame_as;
             arbitrary=atr,ls,rel
    """

    _name = "object"
    _guaranteed = ["name"]
    _also = ["id", "obs", "same_as"]
    _position = ["name", "sex", "id", "same_as", "xsame_as"]
    _part = ["rel", "attr"]
    _pom_class_id: str = "object"


KAct.allow_as_part(KObject)


class KAbstraction(KObject):
    """KAbstraction(name,type,id=,obs=,same_as=,xsame_as=)
    A synonym for object, used in non physical entities such as
    institutions.
    """

    pass


KAct.allow_as_part(KAbstraction)


class KAttribute(KGroup):
    """KAttribute(type,value,[date, obs=])

    Time varying attribute of a person, object, or other

    Attributes represent time-varying information about historical entities.
    Each attribute has a type ('address','profession', 'age'), a value and
    a date associated with the attribute.

    Elements:
        type: the type of the attribute. A String
        value: the value of the attribute. A string.
        date: the date of attribute. A string in Timelink format, optional.
        obs: a note on the attribute. A string optional.

     Kleio str definition :
        part	name=attribute ;
                guaranteed=type,value ;
                also=obs,date ;
                position=type,value,date

    """

    _name = "attr"
    _guaranteed = ["type", "value"]
    _also = ["date", "obs", "entity"]  # entity is automatically set
    _position = ["type", "value", "date"]
    _pom_class_id: str = "attribute"

    def before_include(self, container_group):
        """Method called before a new group is included in this one
        through KGroup.include(KGroup).
        """
        KGroup.before_include(self, container_group)
        self["entity"] = container_group.id
        return True

    def after_include(self, group):
        pass


KPerson.allow_as_part(KAttribute)
KAct.allow_as_part(KAttribute)
KSource.allow_as_part(KAttribute)


class KLs(KAttribute):
    """Synonym for KAttribute"""

    _name = "ls"


class KAtr(KAttribute):
    """Synonym for KAttribute"""

    _name = "atr"


class KRelation(KGroup):
    """KRelation(type,value,destname,destination[,date=,obs=])

    A relation between historical entities.

    Relations have a type, a value, a date and a destination.
    The origin of the relation is the entity represented by the
    group that includes the relation.

    Elements:
        type: the type of the relation. A String
        value: the value of the relation. A string.
        origin: the id of the origin of the destination. A string
        destination: the id of the destination of the relation. A string.
        destname: the name of the destination of the relation. A string
        date: the date of relation. A string in Timelink format, optional.
        obs: a note on the attribute. A string optional.

    Kleio stru definition:
        part 	name=relation ;
                position=type,value,destname,destination;
                guaranteed=type,value,destname,destination ;
                also=obs,date
    """

    _name = "rel"
    _position = ["type", "value", "destname", "destination"]
    _guaranteed = ["type", "value", "destname", "destination"]
    _also = ["obs", "date", "origin"]
    _pom_class_id: str = "relation"

    def before_include(self, container_group):
        """Method called before a new group is included in this one
        through KGroup.include(KGroup)."""
        KGroup.before_include(self, container_group)
        self["origin"] = container_group.id
        return True

    def after_include(self, group):
        pass


KPerson.allow_as_part(KRelation)
KAct.allow_as_part(KRelation)
KSource.allow_as_part(KRelation)
