#  2018-06-20 Friedrich Weber <friedrich.weber@netknights.it>
#             Add PeriodicTask, PeriodicTaskOption, PeriodicTaskLastRun
#  2018-25-09 Paul Lettich <paul.lettich@netknights.it>
#             Add decrease/reset methods to EventCounter
#  2017-10-30 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#             Add timeout and retries to radiuserver
#  2017-08-24 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#             Remote privacyIDEA Server
#  2017-08-11 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#             Add AuthCache
#  2017-04-19 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#             Add support for multiple challenge response token
#  2016-02-19 Cornelius Kölbel <cornelius@privacyidea.org>
#             Add radiusserver table
#  2015-08-27 Cornelius Kölbel <cornelius@privacyidea.org>
#             Add revocation of token
# Nov 11, 2014 Cornelius Kölbel, info@privacyidea.org
# http://www.privacyidea.org
#
# privacyIDEA is a fork of LinOTP. This model definition
# is based on the LinOTP model.
#
# This code is free software; you can redistribute it and/or
# modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
# License as published by the Free Software Foundation; either
# version 3 of the License, or any later version.
#
# This code is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""privacyIDEA Database definition"""

import binascii
import logging
import re
import traceback
from datetime import datetime, timedelta, timezone

from dateutil.tz import tzutc
from json import loads, dumps
from flask_sqlalchemy import SQLAlchemy

from privacyidea.lib.crypto import (encrypt, encryptPin, decryptPin,
                                    geturandom, hash, SecretObj, pass_hash,
                                    verify_pass_hash, get_rand_digit_str)
from sqlalchemy import and_
from sqlalchemy.schema import Sequence, CreateSequence
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.exc import IntegrityError
from .lib.log import log_with
from privacyidea.lib.utils import (is_true, convert_column_to_unicode,
                                   hexlify_and_unicode)
from privacyidea.lib.framework import get_app_config_value
from privacyidea.lib.error import DatabaseError

log = logging.getLogger(__name__)

#
#  After changing the database model do not forget to run
#  ./pi-manage db migrate
#  and edit the autogenerated script.
#

implicit_returning = True
PRIVACYIDEA_TIMESTAMP = "__timestamp__"
SAFE_STORE = "PI_DB_SAFE_STORE"

db = SQLAlchemy()


# Add fractions to the MySQL DataTime column type
@compiles(db.DateTime, "mysql")
def compile_datetime_mysql(type_, compiler, **kw):  # pragma: no cover
    return "DATETIME(6)"


# Fix creation of sequences on MariaDB (and MySQL, which does not support
# sequences anyway) with galera by adding INCREMENT BY 0 to CREATE SEQUENCE
@compiles(CreateSequence, 'mysql')
@compiles(CreateSequence, 'mariadb')
def increment_by_zero(element, compiler, **kw):  # pragma: no cover
    text = compiler.visit_create_sequence(element, **kw)
    text = text + " INCREMENT BY 0"
    return text


class MethodsMixin(object):
    """
    This class mixes in some common Class table functions like
    delete and save
    """

    def save(self):
        db.session.add(self)
        db.session.commit()
        if hasattr(self, 'id'):
            return self.id
        return None

    def delete(self):
        ret = self.id if hasattr(self, 'id') else None
        db.session.delete(self)
        db.session.commit()
        return ret


def save_config_timestamp(invalidate_config=True):
    """
    Save the current timestamp to the database, and optionally
    invalidate the current request-local config object.
    :param invalidate_config: defaults to True
    """
    c1 = Config.query.filter_by(Key=PRIVACYIDEA_TIMESTAMP).first()
    if c1:
        c1.Value = datetime.now().strftime("%s")
    else:
        new_timestamp = Config(PRIVACYIDEA_TIMESTAMP,
                               datetime.now().strftime("%s"),
                               Description="config timestamp. last changed.")
        db.session.add(new_timestamp)
    if invalidate_config:
        # We have just modified the config. From now on, the request handling
        # should operate on the *new* config. Hence, we need to invalidate
        # the current request-local config object. The next access to the config
        # during this request will reload the config from the database and create
        # a new request-local config object, which holds the *new* config.
        from privacyidea.lib.config import invalidate_config_object
        invalidate_config_object()


class TimestampMethodsMixin(object):
    """
    This class mixes in the table functions including update of the timestamp
    """

    def save(self):
        db.session.add(self)
        save_config_timestamp()
        db.session.commit()
        return self.id

    def delete(self):
        ret = self.id
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret


# token_container_association_table = (
#     db.Table('tokencontainertoken',
#              db.metadata,
#              db.Column('token_id', db.Integer, db.ForeignKey('token.id'), primary_key=True),
#              db.Column('container_id', db.Integer, db.ForeignKey('tokencontainer.id'), primary_key=True)))
#

class TokenContainerToken(MethodsMixin, db.Model):
    """
    Association table to link tokens to containers.
    """
    __tablename__ = 'tokencontainertoken'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    token_id = db.Column('token_id', db.Integer, db.ForeignKey('token.id'), primary_key=True)
    container_id = db.Column('container_id', db.Integer, db.ForeignKey('tokencontainer.id'), primary_key=True)


class Token(MethodsMixin, db.Model):
    """
    The "Token" table contains the basic token data.

    It contains data like
     * serial number
     * secret key
     * PINs
     * ...

    The table :py:class:`privacyidea.models.TokenOwner` contains the owner
    information of the specified token.
    The table :py:class:`privacyidea.models.TokenInfo` contains additional information
    that is specific to the tokentype.
    """
    __tablename__ = 'token'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("token_seq"),
                   primary_key=True,
                   nullable=False)
    description = db.Column(db.Unicode(80), default='')
    serial = db.Column(db.Unicode(40), default='',
                       unique=True,
                       nullable=False,
                       index=True)
    tokentype = db.Column(db.Unicode(30),
                          default='HOTP',
                          index=True)
    user_pin = db.Column(db.Unicode(512),
                         default='')  # encrypt
    user_pin_iv = db.Column(db.Unicode(32),
                            default='')  # encrypt
    so_pin = db.Column(db.Unicode(512),
                       default='')  # encrypt
    so_pin_iv = db.Column(db.Unicode(32),
                          default='')  # encrypt
    pin_seed = db.Column(db.Unicode(32),
                         default='')
    otplen = db.Column(db.Integer(),
                       default=6)
    pin_hash = db.Column(db.Unicode(512),
                         default='')  # hashed
    key_enc = db.Column(db.Unicode(2800),
                        default='')  # encrypt
    key_iv = db.Column(db.Unicode(32),
                       default='')
    maxfail = db.Column(db.Integer(),
                        default=10)
    active = db.Column(db.Boolean(),
                       nullable=False,
                       default=True)
    revoked = db.Column(db.Boolean(),
                        default=False)
    locked = db.Column(db.Boolean(),
                       default=False)
    failcount = db.Column(db.Integer(),
                          default=0)
    count = db.Column(db.Integer(),
                      default=0)
    count_window = db.Column(db.Integer(),
                             default=10)
    sync_window = db.Column(db.Integer(),
                            default=1000)
    rollout_state = db.Column(db.Unicode(10),
                              default='')
    info_list = db.relationship('TokenInfo', lazy='select', backref='token')
    # This creates an attribute "token" in the TokenOwner object
    owners = db.relationship('TokenOwner', lazy='dynamic', backref='token')

    # Container
    container = db.relationship('TokenContainer', secondary='tokencontainertoken', back_populates='tokens')

    def __init__(self, serial, tokentype="",
                 isactive=True, otplen=6,
                 otpkey="",
                 userid=None, resolver=None, realm=None,
                 **kwargs):
        super(Token, self).__init__(**kwargs)
        self.serial = '' + serial
        self.tokentype = tokentype
        self.count = 0
        self.failcount = 0
        self.maxfail = 10
        self.active = isactive
        self.revoked = False
        self.locked = False
        self.count_window = 10
        self.otplen = otplen
        self.pin_seed = ""
        self.set_otpkey(otpkey)

        # also create the user assignment
        if userid and resolver and realm:
            # We can not create the tokenrealm-connection and owner-connection, yet
            # since we need to token_id.
            token_id = self.save()
            realm_id = Realm.query.filter_by(name=realm).first().id
            tr = TokenRealm(realm_id=realm_id, token_id=token_id)
            if tr:
                db.session.add(tr)

            to = TokenOwner(token_id=token_id, user_id=userid, resolver=resolver, realm_id=realm_id)
            if to:
                db.session.add(to)

            if tr or to:
                db.session.commit()

    @property
    def first_owner(self):
        return self.owners.first()

    @property
    def all_owners(self):
        return self.owners.all()

    @log_with(log)
    def delete(self):
        # some DBs (e.g. DB2) run in a deadlock, if the TokenRealm entry
        # is deleted via key relation, so we delete it explicitly
        ret = self.id
        db.session.query(TokenRealm) \
            .filter(TokenRealm.token_id == self.id) \
            .delete()
        db.session.query(TokenOwner) \
            .filter(TokenOwner.token_id == self.id) \
            .delete()
        for mt in db.session.execute(db.select(MachineToken).filter(MachineToken.token_id == self.id)).scalars():
            mt.delete()
        db.session.query(Challenge) \
            .filter(Challenge.serial == self.serial) \
            .delete()
        db.session.query(TokenInfo) \
            .filter(TokenInfo.token_id == self.id) \
            .delete()
        db.session.query(TokenTokengroup) \
            .filter(TokenTokengroup.token_id == self.id) \
            .delete()
        if self.tokentype.lower() in ["webauthn", "passkey"]:
            db.session.query(TokenCredentialIdHash).filter(TokenCredentialIdHash.token_id == self.id).delete()

        db.session.delete(self)
        db.session.commit()
        return ret

    @staticmethod
    def _fix_spaces(data):
        """
        On MS SQL server empty fields ("") like the info
        are returned as a string with a space (" ").
        This functions helps to fix this.
        Also avoids running into errors, if the data is a None Type.

        :param data: a string from the database
        :type data: str
        :return: a stripped string
        :rtype: str
        """
        if data:
            data = data.strip()

        return data

    @log_with(log, hide_args=[1])
    def set_otpkey(self, otpkey, reset_failcount=True):
        iv = geturandom(16)
        self.key_enc = encrypt(otpkey, iv)
        length = len(self.key_enc)
        if length > Token.key_enc.property.columns[0].type.length:
            log.error("Key {0!s} exceeds database field {1:d}!".format(self.serial,
                                                                       length))
        self.key_iv = hexlify_and_unicode(iv)
        self.count = 0
        if reset_failcount is True:
            self.failcount = 0

    def set_tokengroups(self, tokengroups, add=False):
        """
        Set the list of the tokengroups.

        This is done by filling the :py:class:`privacyidea.models.TokenTokengroup` table.

        :param tokengroups: the tokengroups
        :type tokengroups: list[str]
        :param add: If set, the tokengroups are added. I.e. old tokengroups are not deleted
        :type add: bool
        """
        # delete old Tokengroups
        if not add:
            db.session.query(TokenTokengroup) \
                .filter(TokenTokengroup.token_id == self.id) \
                .delete()
        # add new Tokengroups
        # We must not set the same tokengroup more than once...
        # uniquify: tokengroups -> set(tokengroups)
        for tokengroup in set(tokengroups):
            # Get the id of the realm to add
            g = Tokengroup.query.filter_by(name=tokengroup).first()
            if g:
                # Check if TokenTokengroup already exists
                tg = TokenTokengroup.query.filter_by(token_id=self.id,
                                                     tokengroup_id=g.id).first()
                if not tg:
                    # If the Tokengroup is not yet attached to the token
                    Tg = TokenTokengroup(token_id=self.id, tokengroup_id=g.id)
                    db.session.add(Tg)
        db.session.commit()

    def set_realms(self, realms, add=False):
        """
        Set the list of the realms.

        This is done by filling the :py:class:`privacyidea.models.TokenRealm` table.

        :param realms: realms
        :type realms: list[str]
        :param add: If set, the realms are added. I.e. old realms are not deleted
        :type add: bool
        """
        # delete old TokenRealms
        if not add:
            db.session.query(TokenRealm).filter(TokenRealm.token_id == self.id).delete()
        # add new TokenRealms
        # We must not set the same realm more than once...
        # uniquify: realms -> set(realms)
        if self.first_owner:
            if self.first_owner.realm.name not in realms:
                realms.append(self.first_owner.realm.name)
                log.info(f"The realm of an assigned user cannot be removed from "
                         f"token {self.first_owner.token.serial} "
                         f"(realm: {self.first_owner.realm.name})")
        for realm in set(realms):
            # Get the id of the realm to add
            realm_db = Realm.query.filter_by(name=realm).first()
            if realm_db:
                # Check if tokenrealm already exists
                token_realm_db = TokenRealm.query.filter_by(token_id=self.id, realm_id=realm_db.id).first()
                if not token_realm_db:
                    # If the realm is not yet attached to the token
                    token_realm = TokenRealm(token_id=self.id, realm_id=realm_db.id)
                    db.session.add(token_realm)
        db.session.commit()

    def get_realms(self):
        """
        return a list of the assigned realms

        :return: the realms of the token
        :rtype: list
        """
        realms = []
        for tokenrealm in self.realm_list:
            realms.append(tokenrealm.realm.name)
        return realms

    @log_with(log)
    def set_user_pin(self, user_pin):
        iv = geturandom(16)
        self.user_pin = encrypt(user_pin, iv)
        self.user_pin_iv = hexlify_and_unicode(iv)

    @log_with(log)
    def get_otpkey(self):
        key = binascii.unhexlify(self.key_enc)
        iv = binascii.unhexlify(self.key_iv)
        secret = SecretObj(key, iv)
        return secret

    @log_with(log)
    def get_user_pin(self):
        """
        return the user_pin
        :rtype : the PIN as a secretObject
        """
        user_pin = self.user_pin or ''
        user_pin_iv = self.user_pin_iv or ''
        key = binascii.unhexlify(user_pin)
        iv = binascii.unhexlify(user_pin_iv)
        secret = SecretObj(key, iv)
        return secret

    def set_hashed_pin(self, pin):
        """
        Set the pin of the token in hashed format

        :param pin: the pin to hash
        :type pin: str
        :return: the hashed pin
        :rtype: str
        """
        self.pin_hash = pass_hash(pin)
        return self.pin_hash

    def get_hashed_pin(self, pin):
        """
        Calculate a hash from a pin
        Fix for working with MS SQL servers
        MS SQL servers sometimes return a '<space>' when the
        column is empty: ''

        :param pin: the pin to hash
        :type pin: str
        :return: hashed pin with current pin_seed
        :rtype: str
        """
        seed_str = self._fix_spaces(self.pin_seed)
        seed = binascii.unhexlify(seed_str)
        hashed_pin = hash(pin, seed)
        log.debug(f"hashed_pin: {hashed_pin}, pin: {pin!r}, seed: {self.pin_seed}")
        return hashed_pin

    @log_with(log)
    def set_description(self, desc):
        if desc is None:
            desc = ""
        length = len(desc)
        if length > Token.description.property.columns[0].type.length:
            desc = desc[:Token.description.property.columns[0].type.length]
        self.description = convert_column_to_unicode(desc)
        return self.description

    def set_pin(self, pin, hashed=True):
        """
        Set the OTP pin in a hashed way
        """
        real_pin = pin or ""
        if hashed is True:
            self.set_hashed_pin(real_pin)
            log.debug(f"set_pin hash: {self.pin_hash!r}")
        else:
            self.pin_hash = "@@" + encryptPin(real_pin)
            log.debug(f"set_pin encrypted: {self.pin_hash!r}")
        return self.pin_hash

    def check_pin(self, pin):
        res = False
        # check for a valid input
        if pin is not None:
            if self.is_pin_encrypted() is True:
                log.debug("we got an encrypted PIN!")
                token_pin = self.pin_hash[2:]
                decrypted_token_pin = decryptPin(token_pin)
                if decrypted_token_pin == pin:
                    res = True
            else:
                log.debug("we got a hashed PIN!")
                if self.pin_hash:
                    try:
                        # New PIN verification
                        return verify_pass_hash(pin, self.pin_hash)
                    except ValueError as _e:
                        # old PIN verification
                        pin_hash = self.get_hashed_pin(pin)
                else:
                    pin_hash = pin
                if pin_hash == (self.pin_hash or ""):
                    res = True
        return res

    def is_pin_encrypted(self, pin=None):
        ret = False
        if pin is None:
            pin = self.pin_hash or ""
        if pin.startswith("@@"):
            ret = True
        return ret

    def get_pin(self):
        ret = -1
        if self.is_pin_encrypted() is True:
            token_pin = self.pin_hash[2:]
            ret = decryptPin(token_pin)
        return ret

    def set_so_pin(self, security_officer_pin):
        """
        For smartcards this sets the security officer pin of the token

        :rtype : None
        """
        iv = geturandom(16)
        self.so_pin = encrypt(security_officer_pin, iv)
        self.so_pin_iv = hexlify_and_unicode(iv)
        return self.so_pin, self.so_pin_iv

    @log_with(log)
    def get(self, key=None, fallback=None, save=False):
        """
        simulate the dict behaviour to make challenge processing
        easier, as this will have to deal as well with
        'dict only challenges'

        :param key: the attribute name - in case of key is not provided, a dict
                    of all class attributes are returned
        :param fallback: if the attribute is not found,
                         the fallback is returned
        :param save: in case of all attributes and save==True, the timestamp is
                     converted to a string representation
        """
        if key is None:
            return self.get_vars(save=save)

        td = self.get_vars(save=save)
        return td.get(key, fallback)

    @log_with(log)
    def get_vars(self, save=False):
        log.debug('get_vars()')
        tokenowner = self.first_owner

        ret = {}
        ret['id'] = self.id
        ret['description'] = self.description
        ret['serial'] = self.serial
        ret['tokentype'] = self.tokentype
        ret['info'] = self.get_info()

        ret['resolver'] = "" if not tokenowner else tokenowner.resolver
        ret['user_id'] = "" if not tokenowner else tokenowner.user_id
        ret['otplen'] = self.otplen

        ret['maxfail'] = self.maxfail
        ret['active'] = self.active
        ret['revoked'] = self.revoked
        ret['locked'] = self.locked
        ret['failcount'] = self.failcount
        ret['count'] = self.count
        ret['count_window'] = self.count_window
        ret['sync_window'] = self.sync_window
        ret['rollout_state'] = self.rollout_state
        # list of Realm names
        realm_list = []
        for realm_entry in self.realm_list:
            realm_list.append(realm_entry.realm.name)
        ret['realms'] = realm_list
        # list of tokengroups
        tokengroup_list = []
        for tg_entry in self.tokengroup_list:
            tokengroup_list.append(tg_entry.tokengroup.name)
        ret['tokengroup'] = tokengroup_list
        return ret

    def __str__(self):
        return self.serial

    def __repr__(self):
        """
        return the token state as text

        :return: token state as string representation
        :rtype:  str
        """
        ldict = {}
        for attr in self.__dict__:
            key = "{0!r}".format(attr)
            val = "{0!r}".format(getattr(self, attr))
            ldict[key] = val
        res = "<{0!r} {1!r}>".format(self.__class__, ldict)
        return res

    def set_info(self, info):
        """
        Set the additional token info for this token

        Entries that end with ".type" are used as type for the keys.
        I.e. two entries sshkey="XYZ" and sshkey.type="password" will store
        the key sshkey as type "password".

        :param info: The key-values to set for this token
        :type info: dict
        """
        if not self.id:
            # If there is no ID to reference the token, we need to save the token
            self.save()
        types = {}
        for k, v in info.items():
            if k.endswith(".type"):
                types[".".join(k.split(".")[:-1])] = v
        for k, v in info.items():
            if not k.endswith(".type"):
                TokenInfo(self.id, k, v, Type=types.get(k)).save(persistent=False)
        db.session.commit()

    def del_info(self, key=None):
        """
        Deletes tokeninfo for a given token.
        If the key is omitted, all Tokeninfo is deleted.

        :param key: searches for the given key to delete the entry
        :return:
        """
        if key:
            tokeninfos = TokenInfo.query.filter_by(token_id=self.id, Key=key)
        else:
            tokeninfos = TokenInfo.query.filter_by(token_id=self.id)
        for ti in tokeninfos:
            ti.delete()

    def del_tokengroup(self, tokengroup=None, tokengroup_id=None):
        """
        Deletes the tokengroup from the given token.
        If tokengroup name and id are omitted, all tokengroups are deleted.

        :param tokengroup: The name of the tokengroup
        :type tokengroup: str
        :param tokengroup_id: The id of the tokengroup
        :type tokengroup_id: int
        :return:
        """
        if tokengroup:
            # We need to resolve the id of the tokengroup
            t = Tokengroup.query.filter_by(name=tokengroup).first()
            if not t:
                raise Exception("tokengroup does not exist")
            tokengroup_id = t.id
        if tokengroup_id:
            tokengroups = TokenTokengroup.query.filter_by(tokengroup_id=tokengroup_id, token_id=self.id)
        else:
            tokengroups = TokenTokengroup.query.filter_by(token_id=self.id)
        for tokengroup in tokengroups:
            tokengroup.delete()

    def get_info(self):
        """

        :return: The token info as dictionary
        """
        ret = {}
        for tokeninfo in self.info_list:
            if tokeninfo.Type:
                ret[tokeninfo.Key + ".type"] = tokeninfo.Type
            ret[tokeninfo.Key] = tokeninfo.Value
        return ret

    def update_type(self, typ):
        """
        in case the previous has been different type
        we must reset the counters
        But be aware, ray, this could also be upper and lower case mixing...
        """
        if self.tokentype.lower() != typ.lower():
            self.count = 0
            self.failcount = 0

        self.tokentype = typ
        return

    def update_otpkey(self, otpkey):
        """
        in case of a new hOtpKey we have to do some more things
        """
        if otpkey is not None:
            otp_key_secret = self.get_otpkey()
            if otp_key_secret.compare(otpkey) is False:
                log.debug('update token OtpKey - counter reset')
                self.set_otpkey(otpkey)

    def update_token(self, description=None, otpkey=None, pin=None):
        if description is not None:
            self.set_description(description)
        if pin is not None:
            self.set_pin(pin)
        if otpkey is not None:
            self.update_otpkey(otpkey)


class TokenInfo(MethodsMixin, db.Model):
    """
    The table "tokeninfo" is used to store additional, long information that
    is specific to the tokentype.
    E.g. the tokentype "TOTP" has additional entries in the tokeninfo table
    for "timeStep" and "timeWindow", which are stored in the
    column "Key" and "Value".

    The tokeninfo is reference by the foreign key to the "token" table.
    """
    __tablename__ = 'tokeninfo'
    id = db.Column(db.Integer, Sequence("tokeninfo_seq"), primary_key=True)
    Key = db.Column(db.Unicode(255),
                    nullable=False)
    Value = db.Column(db.UnicodeText(), default='')
    Type = db.Column(db.Unicode(100), default='')
    Description = db.Column(db.Unicode(2000), default='')
    token_id = db.Column(db.Integer(),
                         db.ForeignKey('token.id'), index=True)
    __table_args__ = (db.UniqueConstraint('token_id',
                                          'Key',
                                          name='tiix_2'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, token_id, Key, Value, Type=None, Description=None):
        """
        Create a new tokeninfo for a given token_id
        """
        self.token_id = token_id
        self.Key = Key
        self.Value = convert_column_to_unicode(Value)
        self.Type = Type
        self.Description = Description

    def save(self, persistent=True):
        ti_func = TokenInfo.query.filter_by(token_id=self.token_id, Key=self.Key).first
        ti = ti_func()
        if ti is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            if get_app_config_value(SAFE_STORE, False):
                ti = ti_func()
                ret = ti.id
            else:
                ret = self.id
        else:
            # update
            TokenInfo.query.filter_by(token_id=self.token_id,
                                      Key=self.Key).update({'Value': self.Value,
                                                            'Description': self.Description,
                                                            'Type': self.Type})
            ret = ti.id
        if persistent:
            db.session.commit()
        return ret


class CustomUserAttribute(MethodsMixin, db.Model):
    """
    The table "customuserattribute" is used to store additional, custom attributes
    for users.

    A user is identified by the user_id,  the resolver_id and the realm_id.

    The additional attributes are stored in Key and Value.
    The Type can hold extra information like e.g. an encrypted value / password.

    Note: Since the users are external, i.e. no objects in this database,
          there is no logic reference on a database level.
          Since users could be deleted from user stores
          without privacyIDEA realizing that, this table could pile up
          with remnants of attributes.
    """
    __tablename__ = 'customuserattribute'
    id = db.Column(db.Integer(), Sequence("customuserattribute_seq"), primary_key=True)
    user_id = db.Column(db.Unicode(320), default='', index=True)
    resolver = db.Column(db.Unicode(120), default='', index=True)
    realm_id = db.Column(db.Integer(), db.ForeignKey('realm.id'))
    Key = db.Column(db.Unicode(255), nullable=False)
    Value = db.Column(db.UnicodeText(), default='')
    Type = db.Column(db.Unicode(100), default='')

    def __init__(self, user_id, resolver, realm_id, Key, Value, Type=None):
        """
        Create a new customuserattribute for a user tuple
        """
        self.user_id = user_id
        self.resolver = resolver
        self.realm_id = realm_id
        self.Key = Key
        self.Value = convert_column_to_unicode(Value)
        self.Type = Type

    def save(self, persistent=True):
        ua = CustomUserAttribute.query.filter_by(user_id=self.user_id,
                                                 resolver=self.resolver,
                                                 realm_id=self.realm_id,
                                                 Key=self.Key).first()
        if ua is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            CustomUserAttribute.query.filter_by(user_id=self.user_id,
                                                resolver=self.resolver,
                                                realm_id=self.realm_id,
                                                Key=self.Key
                                                ).update({'Value': self.Value, 'Type': self.Type})
            ret = ua.id
        if persistent:
            db.session.commit()
        return ret


class Admin(db.Model):
    """
    The administrators for managing the system.
    To manage the administrators use the command pi-manage.

    In addition certain realms can be defined to be administrative realms.

    :param username: The username of the admin
    :type username: basestring
    :param password: The password of the admin (stored using PBKDF2,
       salt and pepper)
    :type password: basestring
    :param email: The email address of the admin (not used at the moment)
    :type email: basestring
    """
    __tablename__ = "admin"
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    username = db.Column(db.Unicode(120),
                         primary_key=True,
                         nullable=False)
    password = db.Column(db.Unicode(255))
    email = db.Column(db.Unicode(255))

    def save(self):
        c = Admin.query.filter_by(username=self.username).first()
        if c is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.username
        else:
            # update
            update_dict = {}
            if self.email:
                update_dict["email"] = self.email
            if self.password:
                update_dict["password"] = self.password
            Admin.query.filter_by(username=self.username) \
                .update(update_dict)
            ret = c.username
        db.session.commit()
        return ret

    def delete(self):
        db.session.delete(self)
        db.session.commit()


class Config(TimestampMethodsMixin, db.Model):
    """
    The config table holds all the system configuration in key value pairs.

    Additional configuration for realms, resolvers and machine resolvers is
    stored in specific tables.
    """
    __tablename__ = "config"
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    Key = db.Column(db.Unicode(255),
                    primary_key=True,
                    nullable=False)
    Value = db.Column(db.Unicode(2000), default='')
    Type = db.Column(db.Unicode(2000), default='')
    Description = db.Column(db.Unicode(2000), default='')

    @log_with(log)
    def __init__(self, Key, Value, Type='', Description=''):
        self.Key = convert_column_to_unicode(Key)
        self.Value = convert_column_to_unicode(Value)
        self.Type = convert_column_to_unicode(Type)
        self.Description = convert_column_to_unicode(Description)

    def __str__(self):
        return "<{0!s} ({1!s})>".format(self.Key, self.Type)

    def save(self):
        db.session.add(self)
        save_config_timestamp()
        db.session.commit()
        return self.Key

    def delete(self):
        ret = self.Key
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret


class Realm(TimestampMethodsMixin, db.Model):
    """
    The realm table contains the defined realms. User Resolvers can be
    grouped to realms. This very table contains just contains the names of
    the realms. The linking to resolvers is stored in the table "resolverrealm".
    """
    __tablename__ = 'realm'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("realm_seq"), primary_key=True,
                   nullable=False)
    name = db.Column(db.Unicode(255), default='',
                     unique=True, nullable=False)
    default = db.Column(db.Boolean(), default=False)
    option = db.Column(db.Unicode(40), default='')
    resolver_list = db.relationship('ResolverRealm',
                                    lazy='select',
                                    back_populates='realm')
    container = db.relationship('TokenContainer', secondary='tokencontainerrealm', back_populates='realms')

    @log_with(log)
    def __init__(self, realm):
        self.name = realm

    def delete(self):
        ret = self.id
        # delete all TokenRealm
        db.session.query(TokenRealm) \
            .filter(TokenRealm.realm_id == ret) \
            .delete()
        # delete all ResolverRealms
        db.session.query(ResolverRealm) \
            .filter(ResolverRealm.realm_id == ret) \
            .delete()
        # delete the realm
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret


class CAConnector(TimestampMethodsMixin, db.Model):
    """
    The table "caconnector" contains the names and types of the defined
    CA connectors. Each connector has a different configuration, that is
    stored in the table "caconnectorconfig".
    """
    __tablename__ = 'caconnector'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("caconnector_seq"), primary_key=True,
                   nullable=False)
    name = db.Column(db.Unicode(255), default="",
                     unique=True, nullable=False)
    catype = db.Column(db.Unicode(255), default="",
                       nullable=False)
    caconfig = db.relationship('CAConnectorConfig',
                               lazy='dynamic',
                               backref='caconnector')

    def __init__(self, name, catype):
        self.name = name
        self.catype = catype

    def delete(self):
        ret = self.id
        # delete all CAConnectorConfig
        db.session.query(CAConnectorConfig) \
            .filter(CAConnectorConfig.caconnector_id == ret) \
            .delete()
        # Delete the CA itself
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret


class CAConnectorConfig(db.Model):
    """
    Each CAConnector can have multiple configuration entries.
    Each CA Connector type can have different required config values. Therefor
    the configuration is stored in simple key/value pairs. If the type of a
    config entry is set to "password" the value of this config entry is stored
    encrypted.

    The config entries are referenced by the id of the resolver.
    """
    __tablename__ = 'caconnectorconfig'
    id = db.Column(db.Integer, Sequence("caconfig_seq"), primary_key=True)
    caconnector_id = db.Column(db.Integer,
                               db.ForeignKey('caconnector.id'))
    Key = db.Column(db.Unicode(255), nullable=False)
    Value = db.Column(db.Unicode(2000), default='')
    Type = db.Column(db.Unicode(2000), default='')
    Description = db.Column(db.Unicode(2000), default='')
    __table_args__ = (db.UniqueConstraint('caconnector_id',
                                          'Key',
                                          name='ccix_2'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, caconnector_id=None,
                 Key=None, Value=None,
                 caconnector=None,
                 Type="", Description=""):
        if caconnector_id:
            self.caconnector_id = caconnector_id
        elif caconnector:
            self.caconnector_id = CAConnector.query \
                .filter_by(name=caconnector) \
                .first() \
                .id
        self.Key = Key
        self.Value = convert_column_to_unicode(Value)
        self.Type = Type
        self.Description = Description

    def save(self):
        c = CAConnectorConfig.query.filter_by(caconnector_id=self.caconnector_id,
                                              Key=self.Key).first()
        save_config_timestamp()
        if c is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            CAConnectorConfig.query.filter_by(caconnector_id=self.caconnector_id,
                                              Key=self.Key
                                              ).update({'Value': self.Value,
                                                        'Type': self.Type,
                                                        'Descrip'
                                                        'tion': self.Description})
            ret = c.id
        db.session.commit()
        return ret


class NodeName(db.Model, TimestampMethodsMixin):
    __tablename__ = "nodename"
    # TODO: we can use the UUID type here when switching to SQLAlchemy 2.0
    #  <https://docs.sqlalchemy.org/en/20/core/custom_types.html#backend-agnostic-guid-type>
    id = db.Column(db.Unicode(36), primary_key=True)
    name = db.Column(db.Unicode(100), index=True)
    lastseen = db.Column(db.DateTime(), index=True, default=datetime.now(tz=tzutc()))


class Resolver(TimestampMethodsMixin, db.Model):
    """
    The table "resolver" contains the names and types of the defined User
    Resolvers. As each Resolver can have different required config values the
    configuration of the resolvers is stored in the table "resolverconfig".
    """
    __tablename__ = 'resolver'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("resolver_seq"), primary_key=True,
                   nullable=False)
    name = db.Column(db.Unicode(255), default="",
                     unique=True, nullable=False)
    rtype = db.Column(db.Unicode(255), default="",
                      nullable=False)
    # This creates an attribute "resolver" in the ResolverConfig object
    config_list = db.relationship('ResolverConfig',
                                  lazy='select')
    realm_list = db.relationship('ResolverRealm',
                                 lazy='select',
                                 back_populates='resolver')

    def __init__(self, name, rtype):
        self.name = name
        self.rtype = rtype

    def delete(self):
        ret = self.id
        # delete all ResolverConfig
        db.session.query(ResolverConfig) \
            .filter(ResolverConfig.resolver_id == ret) \
            .delete()
        # delete the Resolver itself
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret


class ResolverConfig(TimestampMethodsMixin, db.Model):
    """
    Each Resolver can have multiple configuration entries.
    Each Resolver type can have different required config values. Therefor
    the configuration is stored in simple key/value pairs. If the type of a
    config entry is set to "password" the value of this config entry is stored
    encrypted.

    The config entries are referenced by the id of the resolver.
    """
    __tablename__ = 'resolverconfig'
    id = db.Column(db.Integer, Sequence("resolverconf_seq"), primary_key=True)
    resolver_id = db.Column(db.Integer,
                            db.ForeignKey('resolver.id'))
    Key = db.Column(db.Unicode(255), nullable=False)
    Value = db.Column(db.Unicode(2000), default='')
    Type = db.Column(db.Unicode(2000), default='')
    Description = db.Column(db.Unicode(2000), default='')
    __table_args__ = (db.UniqueConstraint('resolver_id',
                                          'Key',
                                          name='rcix_2'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, resolver_id=None,
                 Key=None, Value=None,
                 resolver=None,
                 Type="", Description=""):
        if resolver_id:
            self.resolver_id = resolver_id
        elif resolver:
            self.resolver_id = Resolver.query \
                .filter_by(name=resolver) \
                .first() \
                .id
        self.Key = convert_column_to_unicode(Key)
        self.Value = convert_column_to_unicode(Value)
        self.Type = convert_column_to_unicode(Type)
        self.Description = convert_column_to_unicode(Description)

    def save(self):
        c = ResolverConfig.query.filter_by(resolver_id=self.resolver_id,
                                           Key=self.Key).first()
        if c is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            ResolverConfig.query.filter_by(resolver_id=self.resolver_id,
                                           Key=self.Key
                                           ).update({'Value': self.Value,
                                                     'Type': self.Type,
                                                     'Descrip'
                                                     'tion': self.Description})
            ret = c.id
        save_config_timestamp()
        db.session.commit()
        return ret


class ResolverRealm(TimestampMethodsMixin, db.Model):
    """
    This table stores which Resolver is located in which realm
    This is a N:M relation
    """
    __tablename__ = 'resolverrealm'
    id = db.Column(db.Integer, Sequence("resolverrealm_seq"), primary_key=True)
    resolver_id = db.Column(db.Integer, db.ForeignKey("resolver.id"))
    realm_id = db.Column(db.Integer, db.ForeignKey("realm.id"))
    # If there are several resolvers in a realm, the priority is used the
    # find a user first in a resolver with a higher priority (i.e. lower number)
    priority = db.Column(db.Integer)
    # TODO: with SQLAlchemy 2.0 db.UUID will be generally available
    node_uuid = db.Column(db.Unicode(36), default='')
    resolver = db.relationship(Resolver,
                               lazy="joined",
                               back_populates="realm_list")
    realm = db.relationship(Realm,
                            lazy="joined",
                            back_populates="resolver_list")
    __table_args__ = (db.UniqueConstraint('resolver_id',
                                          'realm_id',
                                          'node_uuid',
                                          name='rrix_2'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, resolver_id=None, realm_id=None,
                 resolver_name=None,
                 realm_name=None,
                 priority=None,
                 node_uuid=None,
                 node_name=None):
        self.resolver_id = None
        self.realm_id = None
        if priority:
            self.priority = int(priority)
        if resolver_id:
            self.resolver_id = resolver_id
        elif resolver_name:
            self.resolver_id = Resolver.query \
                .filter_by(name=resolver_name) \
                .first().id
        if realm_id:
            self.realm_id = realm_id
        elif realm_name:
            self.realm_id = Realm.query \
                .filter_by(name=realm_name) \
                .first().id
        if node_uuid:
            # Check if the node is already defined in the NodeName table
            if db.session.scalar(db.select(db.func.count(NodeName.id)).filter(NodeName.id == node_uuid)) > 0:
                self.node_uuid = node_uuid
            else:
                # Did not find a NodeName entry, adding a new one only if node_name is set
                if node_name:
                    self.node_uuid = NodeName(node_uuid, node_name).save().id
                else:
                    raise DatabaseError(f"No NodeName entry found for UUID {node_uuid}")

        elif node_name:
            # Get the UUID for the corresponding node name
            self.node_uuid = db.session.scalar(db.select(NodeName).filter(NodeName.name == node_name)).id


class TokenOwner(MethodsMixin, db.Model):
    """
    This tables stores the owner of a token.
    A token can be assigned to several users.
    """
    __tablename__ = 'tokenowner'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer(), Sequence("tokenowner_seq"), primary_key=True)
    token_id = db.Column(db.Integer(), db.ForeignKey('token.id'))
    resolver = db.Column(db.Unicode(120), default='', index=True)
    user_id = db.Column(db.Unicode(320), default='', index=True)
    realm_id = db.Column(db.Integer(), db.ForeignKey('realm.id'))
    # This creates an attribute "tokenowners" in the realm objects
    realm = db.relationship('Realm', lazy='joined', backref='tokenowners')

    def __init__(self, token_id=None, serial=None, user_id=None, resolver=None, realm_id=None, realmname=None):
        """
        Create a new token assignment to a user.

        :param token_id: The database ID of the token
        :param serial:  The alternate serial number of the token
        :param resolver: The identifying name of the resolver
        :param realm_id: The database ID of the realm
        :param realmname: The alternate name of realm
        """
        if realm_id is not None:
            self.realm_id = realm_id
        elif realmname:
            r = Realm.query.filter_by(name=realmname).first()
            self.realm_id = r.id
        if token_id is not None:
            self.token_id = token_id
        elif serial:
            r = Token.query.filter_by(serial=serial).first()
            self.token_id = r.id
        self.resolver = resolver
        self.user_id = user_id

    def save(self, persistent=True):
        to_func = TokenOwner.query.filter_by(token_id=self.token_id,
                                             user_id=self.user_id,
                                             realm_id=self.realm_id,
                                             resolver=self.resolver).first
        to = to_func()
        if to is None:
            # This very assignment does not exist, yet:
            db.session.add(self)
            db.session.commit()
            if get_app_config_value(SAFE_STORE, False):
                to = to_func()
                ret = to.id
            else:
                ret = self.id
        else:
            ret = to.id
            # There is nothing to update

        if persistent:
            db.session.commit()
        return ret


class TokenRealm(MethodsMixin, db.Model):
    """
    This table stores to which realms a token is assigned. A token is in the
    realm of the user it is assigned to. But a token can also be put into
    many additional realms.
    """
    __tablename__ = 'tokenrealm'
    id = db.Column(db.Integer(), Sequence("tokenrealm_seq"), primary_key=True)
    token_id = db.Column(db.Integer(),
                         db.ForeignKey('token.id'))
    realm_id = db.Column(db.Integer(),
                         db.ForeignKey('realm.id'))
    # This creates an attribute "realm_list" in the Token object
    token = db.relationship('Token',
                            lazy='joined',
                            backref='realm_list')
    # This creates an attribute "token_list" in the Realm object
    realm = db.relationship('Realm',
                            lazy='joined',
                            backref='token_list')
    __table_args__ = (db.UniqueConstraint('token_id',
                                          'realm_id',
                                          name='trix_2'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, realm_id=0, token_id=0, realmname=None):
        """
        Create a new TokenRealm entry.
        :param realm_id: The id of the realm
        :param token_id: The id of the token
        """
        log.debug("setting realm_id to {0:d}".format(realm_id))
        if realmname:
            r = Realm.query.filter_by(name=realmname).first()
            self.realm_id = r.id
        if realm_id:
            self.realm_id = realm_id
        self.token_id = token_id

    def save(self):
        """
        We only save this, if it does not exist, yet.
        """
        tr_func = TokenRealm.query.filter_by(realm_id=self.realm_id,
                                             token_id=self.token_id).first
        tr = tr_func()
        if tr is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            if get_app_config_value(SAFE_STORE, False):
                tr = tr_func()
                ret = tr.id
            else:
                ret = self.id
        else:
            ret = self.id
        return ret


class PasswordReset(MethodsMixin, db.Model):
    """
    Table for handling password resets.
    This table stores the recoverycodes sent to a given user

    The application should save the HASH of the recovery code. Just like the
    password for the Admins the application shall salt and pepper the hash of
    the recoverycode. A database admin will not be able to inject a rogue
    recovery code.

    A user can get several recoverycodes.
    A recovery code has a validity period

    Optional: The email to which the recoverycode was sent, can be stored.
    """
    __tablename__ = "passwordreset"
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer(), Sequence("pwreset_seq"), primary_key=True,
                   nullable=False)
    recoverycode = db.Column(db.Unicode(255), nullable=False)
    username = db.Column(db.Unicode(64), nullable=False, index=True)
    realm = db.Column(db.Unicode(64), nullable=False, index=True)
    resolver = db.Column(db.Unicode(64))
    email = db.Column(db.Unicode(255))
    timestamp = db.Column(db.DateTime, default=datetime.now())
    expiration = db.Column(db.DateTime)

    @log_with(log)
    def __init__(self, recoverycode, username, realm, resolver="", email=None,
                 timestamp=None, expiration=None, expiration_seconds=3600):
        # The default expiration time is 60 minutes
        self.recoverycode = recoverycode
        self.username = username
        self.realm = realm
        self.resolver = resolver
        self.email = email
        self.timestamp = timestamp or datetime.now()
        self.expiration = expiration or datetime.now() + \
                          timedelta(seconds=expiration_seconds)


class Challenge(MethodsMixin, db.Model):
    """
    Table for handling of the generic challenges.
    """
    __tablename__ = "challenge"
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer(), Sequence("challenge_seq"), primary_key=True,
                   nullable=False)
    transaction_id = db.Column(db.Unicode(64), nullable=False, index=True)
    data = db.Column(db.Unicode(512), default='')
    challenge = db.Column(db.Unicode(512), default='')
    session = db.Column(db.Unicode(512), default='', quote=True, name="session")
    # The token serial number
    serial = db.Column(db.Unicode(40), default='', index=True)
    timestamp = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
    expiration = db.Column(db.DateTime, index=True)
    received_count = db.Column(db.Integer(), default=0)
    otp_valid = db.Column(db.Boolean, default=False)

    @log_with(log)
    def __init__(self, serial, transaction_id=None,
                 challenge='', data='', session='', validitytime=120):

        self.transaction_id = transaction_id or self.create_transaction_id()
        self.challenge = challenge
        self.serial = serial
        self.data = data
        self.timestamp = datetime.utcnow()
        self.session = session
        self.received_count = 0
        self.otp_valid = False
        self.expiration = datetime.utcnow() + timedelta(seconds=validitytime)

    @staticmethod
    def create_transaction_id(length=20):
        return get_rand_digit_str(length)

    def is_valid(self):
        """
        Returns true, if the expiration time has not passed, yet.
        :return: True if valid
        :rtype: bool
        """
        ret = False
        c_now = datetime.utcnow()
        if self.timestamp <= c_now < self.expiration:
            ret = True
        return ret

    def set_data(self, data):
        """
        set the internal data of the challenge
        :param data: unicode data
        :type data: string, length 512
        """
        if type(data) in [dict, list]:
            self.data = dumps(data)
        else:
            self.data = convert_column_to_unicode(data)

    def get_data(self):
        data = {}
        try:
            data = loads(self.data)
        except:
            data = self.data
        return data

    def get_session(self):
        return self.session

    def set_session(self, session):
        self.session = convert_column_to_unicode(session)

    def set_challenge(self, challenge):
        self.challenge = convert_column_to_unicode(challenge)

    def get_challenge(self):
        return self.challenge

    def set_otp_status(self, valid=False):
        self.received_count += 1
        self.otp_valid = valid

    def get_otp_status(self):
        """
        This returns how many OTPs were already received for this challenge.
        and if a valid OTP was received.

        :return: tuple of count and True/False
        :rtype: tuple
        """
        return self.received_count, self.otp_valid

    def get_transaction_id(self):
        return self.transaction_id

    def get(self, timestamp=False):
        """
        return a dictionary of all vars in the challenge class

        :param timestamp: if true, the timestamp will given in a readable
                          format
                          2014-11-29 21:56:43.057293
        :type timestamp: bool
        :return: dict of vars
        """
        descr = {}
        descr['id'] = self.id
        descr['transaction_id'] = self.transaction_id
        descr['challenge'] = self.challenge
        descr['serial'] = self.serial
        descr['data'] = self.get_data()
        if timestamp is True:
            descr['timestamp'] = "{0!s}".format(self.timestamp)
        else:
            descr['timestamp'] = self.timestamp
        descr['otp_received'] = self.received_count > 0
        descr['received_count'] = self.received_count
        descr['otp_valid'] = self.otp_valid
        descr['expiration'] = self.expiration
        return descr

    def __str__(self):
        descr = self.get()
        return "{0!s}".format(descr)


def cleanup_challenges(serial):
    """
    Delete all challenges, that have expired.

    :return: None
    """
    c_now = datetime.utcnow()
    Challenge.query.filter(Challenge.expiration < c_now, Challenge.serial == serial).delete()
    db.session.commit()


# -----------------------------------------------------------------------------
#
# DESCRIPTION
#


class PolicyDescription(TimestampMethodsMixin, db.Model):
    """
    The description table is used to store the description of policy
    """
    __tablename__ = 'description'
    id = db.Column(db.Integer, Sequence("description_seq"), primary_key=True)
    object_id = db.Column(db.Integer, db.ForeignKey('policy.id'), nullable=False)
    object_type = db.Column(db.Unicode(64), unique=False, nullable=False)
    last_update = db.Column(db.DateTime, default=datetime.utcnow)
    description = db.Column(db.UnicodeText())

    __table_args__ = {'mysql_row_format': 'DYNAMIC'}

    def __init__(self, object_id, name="", object_type="", description=""):
        self.name = name
        self.object_type = object_type
        self.object_id = object_id
        self.last_update = datetime.now()
        self.description = description


# -----------------------------------------------------------------------------
#
# POLICY
#


class Policy(TimestampMethodsMixin, db.Model):
    """
    The policy table contains the policy definitions.

    The Policies control the behaviour in the scopes
     * enrollment
     * authentication
     * authorization
     * administration
     * user actions
     * webui
    """
    __tablename__ = "policy"
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("policy_seq"), primary_key=True)
    active = db.Column(db.Boolean, default=True)
    check_all_resolvers = db.Column(db.Boolean, default=False)
    name = db.Column(db.Unicode(64), unique=True, nullable=False)
    user_case_insensitive = db.Column(db.Boolean, default=False)
    scope = db.Column(db.Unicode(32), nullable=False)
    action = db.Column(db.Unicode(2000), default="")
    realm = db.Column(db.Unicode(256), default="")
    adminrealm = db.Column(db.Unicode(256), default="")
    adminuser = db.Column(db.Unicode(256), default="")
    resolver = db.Column(db.Unicode(256), default="")
    pinode = db.Column(db.Unicode(256), default="")
    user = db.Column(db.Unicode(256), default="")
    client = db.Column(db.Unicode(256), default="")
    time = db.Column(db.Unicode(64), default="")
    # If there are multiple matching policies, choose the one
    # with the lowest priority number. We choose 1 to be the default priority.
    priority = db.Column(db.Integer, default=1, nullable=False)
    conditions = db.relationship("PolicyCondition",
                                 lazy="joined",
                                 backref="policy",
                                 order_by="PolicyCondition.id",
                                 # With these cascade options, we ensure that whenever a Policy object is added
                                 # to a session, its conditions are also added to the session (save-update, merge).
                                 # Likewise, whenever a Policy object is deleted, its conditions are also
                                 # deleted (delete). Conditions without a policy are deleted (delete-orphan).
                                 cascade="save-update, merge, delete, delete-orphan")
    description = db.relationship('PolicyDescription', backref='policy',
                                  cascade="save-update, merge, delete, delete-orphan")

    def __init__(self, name,
                 active=True, scope="", action="", realm="", adminrealm="", adminuser="",
                 resolver="", user="", client="", time="", pinode="", priority=1,
                 check_all_resolvers=False, conditions=None, user_case_insensitive=False):
        if isinstance(active, str):
            active = is_true(active.lower())
        self.name = name

        self.user_case_insensitive = user_case_insensitive
        self.action = action
        self.scope = scope
        self.active = active
        self.realm = realm
        self.adminrealm = adminrealm
        self.adminuser = adminuser
        self.resolver = resolver
        self.pinode = pinode
        self.user = user
        self.client = client
        self.time = time
        self.priority = priority
        self.check_all_resolvers = check_all_resolvers
        if conditions is None:
            self.conditions = []
        else:
            self.set_conditions(conditions)

    def set_conditions(self, conditions):
        """
        Replace the list of conditions of this policy with a new list
        of conditions, i.e. a list of 5-tuples (section, key, comparator, value, active).
        """
        self.conditions = []
        for section, key, comparator, value, active in conditions:
            condition_object = PolicyCondition(
                section=section, Key=key, comparator=comparator, Value=value, active=active,
            )
            self.conditions.append(condition_object)

    def get_conditions_tuples(self):
        """
        :return: a list of 5-tuples (section, key, comparator, value, active).
        """
        return [condition.as_tuple() for condition in self.conditions]

    def get_policy_description(self):
        """

        """
        if self.description:
            ret = self.description[0].description
        else:
            ret = None
        return ret

    @staticmethod
    def _split_string(value):
        """
        Split the value at the "," and returns an array.
        If value is empty, it returns an empty array.
        The normal split would return an array with an empty string.

        :param value: The string to be splitted
        :type value: basestring
        :return: list
        """
        ret = [r.strip() for r in (value or "").split(",")]
        if ret == ['']:
            ret = []
        return ret

    def get(self, key=None):
        """
        Either returns the complete policy entry or a single value
        :param key: return the value for this key
        :type key: string
        :return: complete dict or single value
        :rytpe: dict or value
        """
        d = {"name": self.name,
             "user_case_insensitive": self.user_case_insensitive,
             "active": self.active,
             "scope": self.scope,
             "realm": self._split_string(self.realm),
             "adminrealm": self._split_string(self.adminrealm),
             "adminuser": self._split_string(self.adminuser),
             "resolver": self._split_string(self.resolver),
             "pinode": self._split_string(self.pinode),
             "check_all_resolvers": self.check_all_resolvers,
             "user": self._split_string(self.user),
             "client": self._split_string(self.client),
             "time": self.time,
             "conditions": self.get_conditions_tuples(),
             "priority": self.priority,
             "description": self.get_policy_description()}
        action_list = [x.strip().split("=", 1) for x in re.split(r'(?<!\\),', self.action or "")]
        action_dict = {}
        for a in action_list:
            if len(a) > 1:
                action_dict[a[0]] = a[1]
            else:
                action_dict[a[0]] = True
        d["action"] = action_dict
        if key:
            ret = d.get(key)
        else:
            ret = d
        return ret


class PolicyCondition(MethodsMixin, db.Model):
    __tablename__ = "policycondition"

    id = db.Column(db.Integer, Sequence("policycondition_seq"), primary_key=True)
    policy_id = db.Column(db.Integer, db.ForeignKey('policy.id'), nullable=False)
    section = db.Column(db.Unicode(255), nullable=False)
    # We use upper-case "Key" and "Value" to prevent conflicts with databases
    # that do not support "key" or "value" as column names
    Key = db.Column(db.Unicode(255), nullable=False)
    comparator = db.Column(db.Unicode(255), nullable=False, default='equals')
    Value = db.Column(db.Unicode(2000), nullable=False, default='')
    active = db.Column(db.Boolean, nullable=False, default=True)

    __table_args__ = {'mysql_row_format': 'DYNAMIC'}

    def as_tuple(self):
        """
        :return: the condition as a tuple (section, key, comparator, value, active)
        """
        return self.section, self.Key, self.comparator, self.Value, self.active


# ------------------------------------------------------------------
#
#  Machines
#

class MachineToken(MethodsMixin, db.Model):
    """
    The MachineToken assigns a Token and an application type to a
    machine.
    The Machine is represented as the tuple of machineresolver.id and the
    machine_id.
    The machine_id is defined by the machineresolver.

    This can be an n:m mapping.
    """
    __tablename__ = 'machinetoken'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer(), Sequence("machinetoken_seq"),
                   primary_key=True, nullable=False)
    token_id = db.Column(db.Integer(),
                         db.ForeignKey('token.id'))
    machineresolver_id = db.Column(db.Integer())
    machine_id = db.Column(db.Unicode(255))
    application = db.Column(db.Unicode(64))
    # This connects the machine with the token and makes the machines visible
    # in the token as "machine_list".
    token = db.relationship('Token',
                            lazy='joined',
                            backref='machine_list')

    @log_with(log)
    def __init__(self, machineresolver_id=None,
                 machineresolver=None, machine_id=None, token_id=None,
                 serial=None, application=None):

        if machineresolver_id:
            self.machineresolver_id = machineresolver_id
        elif machineresolver:
            # determine the machineresolver_id:
            self.machineresolver_id = MachineResolver.query.filter(
                MachineResolver.name == machineresolver).first().id
        if token_id:
            self.token_id = token_id
        elif serial:
            # determine token_id
            self.token_id = Token.query.filter_by(serial=serial).first().id
        self.machine_id = machine_id
        self.application = application

    def delete(self):
        ret = self.id
        db.session.query(MachineTokenOptions) \
            .filter(MachineTokenOptions.machinetoken_id == self.id) \
            .delete()
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret


"""
class MachineUser(db.Model):
    '''
    The MachineUser maps a user to a client and
    an application on this client

    The tuple of (machine, USER, application) is unique.

    This can be an n:m mapping.
    '''
    __tablename__ = "machineuser"
    id = db.Column(db.Integer(), primary_key=True, nullable=False)
    resolver = db.Column(db.Unicode(120), default=u'', index=True)
    resclass = db.Column(db.Unicode(120),  default=u'')
    user_id = db.Column(db.Unicode(120), default=u'', index=True)
    machine_id = db.Column(db.Integer(),
                           db.ForeignKey('clientmachine.id'))
    application = db.Column(db.Unicode(64))

    __table_args__ = (db.UniqueConstraint('resolver', 'resclass',
                                          'user_id', 'machine_id',
                                          'application', name='uixu_1'),
                      {})

    @log_with(log)
    def __init__(self, machine_id,
                 resolver,
                 resclass,
                 user_id,
                 application):
        log.debug("setting machine_id to %r" % machine_id)
        self.machine_id = machine_id
        self.resolver = resolver
        self.resclass = resclass
        self.user_id = user_id
        self.application = application

    @log_with(log)
    def store(self):
        db.session.add(self)
        db.session.commit()
        return True

    def to_json(self):
        machinename = ""
        ip = ""
        if self.machine:
            machinename = self.machine.cm_name
            ip = self.machine.cm_ip
        return {'id': self.id,
                'user_id': self.user_id,
                'resolver': self.resolver,
                'resclass': self.resclass,
                'machine_id': self.machine_id,
                'machinename': machinename,
                'ip': ip,
                'application': self.application}
"""


class MachineTokenOptions(db.Model):
    """
    This class holds an Option for the token assigned to
    a certain client machine.
    Each Token-Clientmachine-Combination can have several
    options.
    """
    __tablename__ = 'machinetokenoptions'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer(), Sequence("machtokenopt_seq"),
                   primary_key=True, nullable=False)
    machinetoken_id = db.Column(db.Integer(),
                                db.ForeignKey('machinetoken.id'))
    mt_key = db.Column(db.Unicode(64), nullable=False)
    mt_value = db.Column(db.Unicode(64), nullable=False)
    # This connects the MachineTokenOption with the MachineToken and makes the
    # options visible in the MachineToken as "option_list".
    machinetoken = db.relationship('MachineToken',
                                   lazy='joined',
                                   backref='option_list')

    def __init__(self, machinetoken_id, key, value):
        log.debug("setting {0!r} to {1!r} for MachineToken {2!s}".format(key,
                                                                         value,
                                                                         machinetoken_id))
        self.machinetoken_id = machinetoken_id
        self.mt_key = convert_column_to_unicode(key)
        self.mt_value = convert_column_to_unicode(value)

        # if the combination machinetoken_id / mt_key already exist,
        # we need to update
        c = MachineTokenOptions.query.filter_by(
            machinetoken_id=self.machinetoken_id,
            mt_key=self.mt_key).first()
        if c is None:
            # create a new one
            db.session.add(self)
        else:
            # update
            MachineTokenOptions.query.filter_by(
                machinetoken_id=self.machinetoken_id,
                mt_key=self.mt_key).update({'mt_value': self.mt_value})
        db.session.commit()


"""
class MachineUserOptions(db.Model):
    '''
    This class holds an Option for the Users assigned to
    a certain client machine.
    Each User-Clientmachine-Combination can have several
    options.
    '''
    __tablename__ = 'machineuseroptions'
    id = db.Column(db.Integer(), primary_key=True, nullable=False)
    machineuser_id = db.Column(db.Integer(), db.ForeignKey('machineuser.id'))
    mu_key = db.Column(db.Unicode(64), nullable=False)
    mu_value = db.Column(db.Unicode(64), nullable=False)

    def __init__(self, machineuser_id, key, value):
        log.debug("setting %r to %r for MachineUser %s" % (key,
                                                           value,
                                                           machineuser_id))
        self.machineuser_id = machineuser_id
        self.mu_key = key
        self.mu_value = value
        db.session.add(self)
        db.session.commit()

"""


class EventHandler(MethodsMixin, db.Model):
    """
    This model holds the list of defined events and actions to this events.
    A handler module can be bound to an event with the corresponding
    condition and action.
    """
    __tablename__ = 'eventhandler'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("eventhandler_seq"), primary_key=True,
                   nullable=False)
    # in fact the name is a description
    name = db.Column(db.Unicode(64), unique=False, nullable=True)
    active = db.Column(db.Boolean, default=True)
    ordering = db.Column(db.Integer, nullable=False, default=0)
    position = db.Column(db.Unicode(10), default="post")
    # This is the name of the event in the code
    event = db.Column(db.Unicode(255), nullable=False)
    # This is the identifier of an event handler module
    handlermodule = db.Column(db.Unicode(255), nullable=False)
    condition = db.Column(db.Unicode(1024), default="")
    action = db.Column(db.Unicode(1024), default="")
    # This creates an attribute "eventhandler" in the EventHandlerOption object
    options = db.relationship('EventHandlerOption',
                              lazy='dynamic',
                              backref='eventhandler')
    # This creates an attribute "eventhandler" in the EventHandlerCondition object
    conditions = db.relationship('EventHandlerCondition',
                                 lazy='dynamic',
                                 backref='eventhandler')

    def __init__(self, name, event, handlermodule, action, condition="",
                 ordering=0, options=None, id=None, conditions=None,
                 active=True, position="post"):
        self.name = name
        self.ordering = ordering
        self.event = event
        self.handlermodule = handlermodule
        self.condition = condition
        self.action = action
        self.active = active
        self.position = position
        if id == "":
            id = None
        self.id = id
        self.save()
        # add the options to the event handler
        options = options or {}
        for k, v in options.items():
            EventHandlerOption(eventhandler_id=self.id, Key=k, Value=v).save()
        conditions = conditions or {}
        for k, v in conditions.items():
            EventHandlerCondition(eventhandler_id=self.id, Key=k, Value=v).save()
        # Delete event handler conditions, that ar not used anymore.
        ev_conditions = EventHandlerCondition.query.filter_by(
            eventhandler_id=self.id).all()
        for cond in ev_conditions:
            if cond.Key not in conditions:
                EventHandlerCondition.query.filter_by(
                    eventhandler_id=self.id, Key=cond.Key).delete()
                db.session.commit()

    def save(self):
        if self.id is None:
            # create a new one
            db.session.add(self)
        else:
            # update
            EventHandler.query.filter_by(id=self.id).update({
                "ordering": self.ordering or 0,
                "position": self.position or "post",
                "event": self.event,
                "active": self.active,
                "name": self.name,
                "handlermodule": self.handlermodule,
                "condition": self.condition,
                "action": self.action
            })
        save_config_timestamp()
        db.session.commit()
        return self.id

    def delete(self):
        ret = self.id
        # delete all EventHandlerOptions
        db.session.query(EventHandlerOption) \
            .filter(EventHandlerOption.eventhandler_id == ret) \
            .delete()
        # delete all Conditions
        db.session.query(EventHandlerCondition) \
            .filter(EventHandlerCondition.eventhandler_id == ret) \
            .delete()
        # delete the event handler itself
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret

    def get(self):
        """
        Return the serialized eventhandler object including the options

        :return: complete dict
        :rytpe: dict
        """
        d = {"active": self.active,
             "name": self.name,
             "handlermodule": self.handlermodule,
             "id": self.id,
             "ordering": self.ordering,
             "position": self.position or "post",
             "action": self.action,
             "condition": self.condition}
        event_list = [x.strip() for x in self.event.split(",")]
        d["event"] = event_list
        option_dict = {}
        for option in self.options:
            option_dict[option.Key] = option.Value
        d["options"] = option_dict
        condition_dict = {}
        for cond in self.conditions:
            condition_dict[cond.Key] = cond.Value
        d["conditions"] = condition_dict
        return d


class EventHandlerCondition(db.Model):
    """
    Each EventHandler entry can have additional conditions according to the
    handler module
    """
    __tablename__ = "eventhandlercondition"
    id = db.Column(db.Integer, Sequence("eventhandlercond_seq"),
                   primary_key=True)
    eventhandler_id = db.Column(db.Integer,
                                db.ForeignKey('eventhandler.id'))
    Key = db.Column(db.Unicode(255), nullable=False)
    Value = db.Column(db.Unicode(2000), default='')
    comparator = db.Column(db.Unicode(255), default='equal')
    __table_args__ = (db.UniqueConstraint('eventhandler_id',
                                          'Key',
                                          name='ehcix_1'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, eventhandler_id, Key, Value, comparator="equal"):
        self.eventhandler_id = eventhandler_id
        self.Key = Key
        self.Value = convert_column_to_unicode(Value)
        self.comparator = comparator
        self.save()

    def save(self):
        ehc = EventHandlerCondition.query.filter_by(
            eventhandler_id=self.eventhandler_id, Key=self.Key).first()
        if ehc is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            EventHandlerCondition.query.filter_by(
                eventhandler_id=self.eventhandler_id, Key=self.Key) \
                .update({'Value': self.Value,
                         'comparator': self.comparator})
            ret = ehc.id
        db.session.commit()
        return ret


class EventHandlerOption(db.Model):
    """
    Each EventHandler entry can have additional options according to the
    handler module.
    """
    __tablename__ = 'eventhandleroption'
    id = db.Column(db.Integer, Sequence("eventhandleropt_seq"),
                   primary_key=True)
    eventhandler_id = db.Column(db.Integer,
                                db.ForeignKey('eventhandler.id'))
    Key = db.Column(db.Unicode(255), nullable=False)
    Value = db.Column(db.Unicode(2000), default='')
    Type = db.Column(db.Unicode(2000), default='')
    Description = db.Column(db.Unicode(2000), default='')
    __table_args__ = (db.UniqueConstraint('eventhandler_id',
                                          'Key',
                                          name='ehoix_1'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, eventhandler_id, Key, Value, Type="", Description=""):
        self.eventhandler_id = eventhandler_id
        self.Key = Key
        self.Value = convert_column_to_unicode(Value)
        self.Type = Type
        self.Description = Description
        self.save()

    def save(self):
        eho = EventHandlerOption.query.filter_by(
            eventhandler_id=self.eventhandler_id, Key=self.Key).first()
        if eho is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            EventHandlerOption.query.filter_by(
                eventhandler_id=self.eventhandler_id, Key=self.Key) \
                .update({'Value': self.Value,
                         'Type': self.Type,
                         'Description': self.Description})
            ret = eho.id
        db.session.commit()
        return ret


class MachineResolver(MethodsMixin, db.Model):
    """
    This model holds the definition to the machinestore.
    Machines could be located in flat files, LDAP directory or in puppet
    services or other...

    The usual MachineResolver just holds a name and a type and a reference to
    its config
    """
    __tablename__ = 'machineresolver'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("machineresolver_seq"),
                   primary_key=True, nullable=False)
    name = db.Column(db.Unicode(255), default="",
                     unique=True, nullable=False)
    rtype = db.Column(db.Unicode(255), default="",
                      nullable=False)
    rconfig = db.relationship('MachineResolverConfig',
                              lazy='dynamic',
                              backref='machineresolver')

    def __init__(self, name, rtype):
        self.name = name
        self.rtype = rtype

    def delete(self):
        ret = self.id
        # delete all MachineResolverConfig
        db.session.query(MachineResolverConfig) \
            .filter(MachineResolverConfig.resolver_id == ret) \
            .delete()
        # delete the MachineResolver itself
        db.session.delete(self)
        db.session.commit()
        return ret


class MachineResolverConfig(db.Model):
    """
    Each Machine Resolver can have multiple configuration entries.
    The config entries are referenced by the id of the machine resolver
    """
    __tablename__ = 'machineresolverconfig'
    id = db.Column(db.Integer, Sequence("machineresolverconf_seq"),
                   primary_key=True)
    resolver_id = db.Column(db.Integer,
                            db.ForeignKey('machineresolver.id'))
    Key = db.Column(db.Unicode(255), nullable=False)
    Value = db.Column(db.Unicode(2000), default='')
    Type = db.Column(db.Unicode(2000), default='')
    Description = db.Column(db.Unicode(2000), default='')
    __table_args__ = (db.UniqueConstraint('resolver_id',
                                          'Key',
                                          name='mrcix_2'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, resolver_id=None, Key=None, Value=None, resolver=None,
                 Type="", Description=""):
        if resolver_id:
            self.resolver_id = resolver_id
        elif resolver:
            self.resolver_id = MachineResolver.query \
                .filter_by(name=resolver) \
                .first() \
                .id
        self.Key = Key
        self.Value = convert_column_to_unicode(Value)
        self.Type = Type
        self.Description = Description

    def save(self):
        c = MachineResolverConfig.query.filter_by(
            resolver_id=self.resolver_id, Key=self.Key).first()
        if c is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            MachineResolverConfig.query.filter_by(
                resolver_id=self.resolver_id, Key=self.Key) \
                .update({'Value': self.Value,
                         'Type': self.Type,
                         'Description': self.Description})
            ret = c.id
        db.session.commit()
        return ret


def get_token_id(serial):
    """
    Return the database token ID for a given serial number
    :param serial:
    :return: token ID
    :rtpye: int
    """
    token = Token.query.filter(Token.serial == serial).first()
    return token.id


def get_machineresolver_id(resolvername):
    """
    Return the database ID of the machine resolver
    :param resolvername:
    :return:
    """
    mr = MachineResolver.query.filter(MachineResolver.name ==
                                      resolvername).first()
    return mr.id


def get_machinetoken_ids(machine_id, resolver_name, serial, application):
    """
    Returns a list of the ID in the machinetoken table

    :param machine_id: The resolverdependent machine_id
    :type machine_id: basestring
    :param resolver_name: The name of the resolver
    :type resolver_name: basestring
    :param serial: the serial number of the token
    :type serial: basestring
    :param application: The application type
    :type application: basestring
    :return: A list of IDs of the machinetoken entry
    :rtype: list of int
    """
    ret = []
    token_id = get_token_id(serial)
    if resolver_name:
        resolver = MachineResolver.query.filter(MachineResolver.name == resolver_name).first()
        resolver_id = resolver.id
    else:
        resolver_id = None

    mtokens = MachineToken.query.filter(and_(MachineToken.token_id == token_id,
                                             MachineToken.machineresolver_id == resolver_id,
                                             MachineToken.machine_id == machine_id,
                                             MachineToken.application == application)).all()
    if mtokens:
        for mt in mtokens:
            ret.append(mt.id)
    return ret


class SMSGateway(MethodsMixin, db.Model):
    """
    This table stores the SMS Gateway definitions.
    See
    https://github.com/privacyidea/privacyidea/wiki/concept:-Delivery-Gateway

    It saves the
    * unique name
    * a description
    * the SMS provider module

    All options and parameters are saved in other tables.
    """
    __tablename__ = 'smsgateway'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("smsgateway_seq"), primary_key=True)
    identifier = db.Column(db.Unicode(255), nullable=False, unique=True)
    description = db.Column(db.Unicode(1024), default="")
    providermodule = db.Column(db.Unicode(1024), nullable=False)
    options = db.relationship('SMSGatewayOption',
                              lazy='dynamic',
                              backref='smsgw')

    def __init__(self, identifier, providermodule, description=None,
                 options=None, headers=None):

        options = options or {}
        headers = headers or {}
        sql = SMSGateway.query.filter_by(identifier=identifier).first()
        if sql:
            self.id = sql.id
        self.identifier = identifier
        self.providermodule = providermodule
        self.description = description
        self.save()
        # delete non existing options in case of update
        opts = {"option": options, "header": headers}
        if sql:
            sql_opts = {"option": sql.option_dict, "header": sql.header_dict}
            for typ, vals in opts.items():
                for key in sql_opts[typ].keys():
                    # iterate through all existing options/headers
                    if key not in vals:
                        # if the option is not contained anymore
                        SMSGatewayOption.query.filter_by(gateway_id=self.id,
                                                         Key=key, Type=typ).delete()
        # add the options and headers to the SMS Gateway
        for typ, vals in opts.items():
            for k, v in vals.items():
                SMSGatewayOption(gateway_id=self.id, Key=k, Value=v, Type=typ).save()

    def save(self):
        if self.id is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
        else:
            # update
            SMSGateway.query.filter_by(id=self.id).update({
                "identifier": self.identifier,
                "providermodule": self.providermodule,
                "description": self.description
            })
            db.session.commit()
        return self.id

    def delete(self):
        """
        When deleting an SMS Gateway we also delete all the options.
        :return:
        """
        ret = self.id
        # delete all SMSGatewayOptions
        db.session.query(SMSGatewayOption) \
            .filter(SMSGatewayOption.gateway_id == ret) \
            .delete()
        # delete the SMSGateway itself
        db.session.delete(self)
        db.session.commit()
        return ret

    @property
    def option_dict(self):
        """
        Return all connected options as a dictionary

        :return: dict
        """
        res = {}
        for option in self.options:
            if option.Type == "option" or not option.Type:
                res[option.Key] = option.Value
        return res

    @property
    def header_dict(self):
        """
        Return all connected headers as a dictionary

        :return: dict
        """
        res = {}
        for option in self.options:
            if option.Type == "header":
                res[option.Key] = option.Value
        return res

    def as_dict(self):
        """
        Return the object as a dictionary

        :return: complete dict
        :rytpe: dict
        """
        d = {"id": self.id,
             "name": self.identifier,
             "providermodule": self.providermodule,
             "description": self.description,
             "options": self.option_dict,
             "headers": self.header_dict}

        return d


class SMSGatewayOption(MethodsMixin, db.Model):
    """
    This table stores the options, parameters and headers for an SMS Gateway definition.
    """
    __tablename__ = 'smsgatewayoption'
    id = db.Column(db.Integer, Sequence("smsgwoption_seq"), primary_key=True)
    Key = db.Column(db.Unicode(255), nullable=False)
    Value = db.Column(db.UnicodeText(), default='')
    Type = db.Column(db.Unicode(100), default='option')
    gateway_id = db.Column(db.Integer(),
                           db.ForeignKey('smsgateway.id'), index=True)
    __table_args__ = (db.UniqueConstraint('gateway_id',
                                          'Key', 'Type',
                                          name='sgix_1'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, gateway_id, Key, Value, Type=None):

        """
        Create a new gateway_option for the gateway_id
        """
        self.gateway_id = gateway_id
        self.Key = Key
        self.Value = convert_column_to_unicode(Value)
        self.Type = Type
        self.save()

    def save(self):
        # See, if there is this option or header for this this gateway
        # The first match takes precedence
        go = SMSGatewayOption.query.filter_by(gateway_id=self.gateway_id,
                                              Key=self.Key, Type=self.Type).first()
        if go is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            SMSGatewayOption.query.filter_by(gateway_id=self.gateway_id,
                                             Key=self.Key, Type=self.Type
                                             ).update({'Value': self.Value,
                                                       'Type': self.Type})
            ret = go.id
        db.session.commit()
        return ret


class PrivacyIDEAServer(MethodsMixin, db.Model):
    """
    This table can store remote privacyIDEA server definitions
    """
    __tablename__ = 'privacyideaserver'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("privacyideaserver_seq"),
                   primary_key=True)
    # This is a name to refer to
    identifier = db.Column(db.Unicode(255), nullable=False, unique=True)
    # This is the FQDN or the IP address
    url = db.Column(db.Unicode(255), nullable=False)
    tls = db.Column(db.Boolean, default=False)
    description = db.Column(db.Unicode(2000), default='')

    def save(self):
        pi = PrivacyIDEAServer.query.filter(PrivacyIDEAServer.identifier ==
                                            self.identifier).first()
        if pi is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            values = {"url": self.url}
            if self.tls is not None:
                values["tls"] = self.tls
            if self.description is not None:
                values["description"] = self.description
            PrivacyIDEAServer.query.filter(PrivacyIDEAServer.identifier ==
                                           self.identifier).update(values)
            ret = pi.id
        db.session.commit()
        return ret


class RADIUSServer(MethodsMixin, db.Model):
    """
    This table can store configurations of RADIUS servers.
    https://github.com/privacyidea/privacyidea/issues/321

    It saves
    * a unique name
    * a description
    * an IP address a
    * a Port
    * a secret
    * timeout in seconds (default 5)
    * retries (default 3)

    These RADIUS server definition can be used in RADIUS tokens or in a
    radius passthru policy.
    """
    __tablename__ = 'radiusserver'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("radiusserver_seq"), primary_key=True)
    # This is a name to refer to
    identifier = db.Column(db.Unicode(255), nullable=False, unique=True)
    # This is the FQDN or the IP address
    server = db.Column(db.Unicode(255), nullable=False)
    port = db.Column(db.Integer, default=25)
    secret = db.Column(db.Unicode(255), default="")
    dictionary = db.Column(db.Unicode(255),
                           default="/etc/privacyidea/dictionary")
    description = db.Column(db.Unicode(2000), default='')
    timeout = db.Column(db.Integer, default=5)
    retries = db.Column(db.Integer, default=3)

    def save(self):
        """
        If a RADIUS server with a given name is save, then the existing
        RADIUS server is updated.
        """
        radius = RADIUSServer.query.filter(RADIUSServer.identifier ==
                                           self.identifier).first()
        if radius is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            values = {"server": self.server}
            if self.port is not None:
                values["port"] = self.port
            if self.secret is not None:
                values["secret"] = self.secret
            if self.dictionary is not None:
                values["dictionary"] = self.dictionary
            if self.description is not None:
                values["description"] = self.description
            if self.timeout is not None:
                values["timeout"] = int(self.timeout)
            if self.retries is not None:
                values["retries"] = int(self.retries)
            RADIUSServer.query.filter(RADIUSServer.identifier ==
                                      self.identifier).update(values)
            ret = radius.id
        db.session.commit()
        return ret


class SMTPServer(MethodsMixin, db.Model):
    """
    This table can store configurations for SMTP servers.
    Each entry represents an SMTP server.
    EMail Token, SMS SMTP Gateways or Notifications like PIN handlers are
    supposed to use a reference to a server definition.
    Each Machine Resolver can have multiple configuration entries.
    The config entries are referenced by the id of the machine resolver
    """
    __tablename__ = 'smtpserver'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("smtpserver_seq"), primary_key=True)
    # This is a name to refer to
    identifier = db.Column(db.Unicode(255), nullable=False)
    # This is the FQDN or the IP address
    server = db.Column(db.Unicode(255), nullable=False)
    port = db.Column(db.Integer, default=25)
    username = db.Column(db.Unicode(255), default="")
    password = db.Column(db.Unicode(255), default="")
    sender = db.Column(db.Unicode(255), default="")
    tls = db.Column(db.Boolean, default=False)
    description = db.Column(db.Unicode(2000), default='')
    timeout = db.Column(db.Integer, default=10)
    enqueue_job = db.Column(db.Boolean, nullable=False, default=False)

    def get(self):
        """
        :return: the configuration as a dictionary
        """
        return {
            "id": self.id,
            "identifier": self.identifier,
            "server": self.server,
            "port": self.port,
            "username": self.username,
            "password": self.password,
            "sender": self.sender,
            "tls": self.tls,
            "description": self.description,
            "timeout": self.timeout,
            "enqueue_job": self.enqueue_job,
        }

    def save(self):
        smtp = SMTPServer.query.filter(SMTPServer.identifier ==
                                       self.identifier).first()
        if smtp is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            values = {"server": self.server}
            if self.port is not None:
                values["port"] = self.port
            if self.username is not None:
                values["username"] = self.username
            if self.password is not None:
                values["password"] = self.password
            if self.sender is not None:
                values["sender"] = self.sender
            if self.tls is not None:
                values["tls"] = self.tls
            if self.description is not None:
                values["description"] = self.description
            if self.timeout is not None:
                values["timeout"] = self.timeout
            if self.enqueue_job is not None:
                values["enqueue_job"] = self.enqueue_job
            SMTPServer.query.filter(SMTPServer.identifier ==
                                    self.identifier).update(values)
            ret = smtp.id
        db.session.commit()
        return ret


class ClientApplication(MethodsMixin, db.Model):
    """
    This table stores the clients, which sent an authentication request to
    privacyIDEA.
    This table is filled automatically by authentication requests.
    """
    __tablename__ = 'clientapplication'
    id = db.Column(db.Integer, Sequence("clientapp_seq"), primary_key=True)
    ip = db.Column(db.Unicode(255), nullable=False, index=True)
    hostname = db.Column(db.Unicode(255))
    clienttype = db.Column(db.Unicode(255), nullable=False, index=True)
    lastseen = db.Column(db.DateTime, index=True, default=datetime.utcnow())
    node = db.Column(db.Unicode(255), nullable=False)
    __table_args__ = (db.UniqueConstraint('ip',
                                          'clienttype',
                                          'node',
                                          name='caix'),
                      {'mysql_row_format': 'DYNAMIC'})

    def save(self):
        clientapp = ClientApplication.query.filter(
            ClientApplication.ip == self.ip,
            ClientApplication.clienttype == self.clienttype,
            ClientApplication.node == self.node).first()
        self.lastseen = datetime.now()
        if clientapp is None:
            # create a new one
            db.session.add(self)
        else:
            # update
            values = {"lastseen": self.lastseen}
            if self.hostname is not None:
                values["hostname"] = self.hostname
            ClientApplication.query.filter(ClientApplication.id == clientapp.id).update(values)
        try:
            db.session.commit()
        except IntegrityError as e:  # pragma: no cover
            log.info('Unable to write ClientApplication entry to db: {0!s}'.format(e))
            log.debug(traceback.format_exc())

    def __repr__(self):
        return "<ClientApplication [{0!s}][{1!s}:{2!s}] on {3!s}>".format(
            self.id, self.ip, self.clienttype, self.node)


class Subscription(MethodsMixin, db.Model):
    """
    This table stores the imported subscription files.
    """
    __tablename__ = 'subscription'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("subscription_seq"), primary_key=True)
    application = db.Column(db.Unicode(80), index=True)
    for_name = db.Column(db.Unicode(80), nullable=False)
    for_address = db.Column(db.Unicode(128))
    for_email = db.Column(db.Unicode(128), nullable=False)
    for_phone = db.Column(db.Unicode(50), nullable=False)
    for_url = db.Column(db.Unicode(80))
    for_comment = db.Column(db.Unicode(255))
    by_name = db.Column(db.Unicode(50), nullable=False)
    by_email = db.Column(db.Unicode(128), nullable=False)
    by_address = db.Column(db.Unicode(128))
    by_phone = db.Column(db.Unicode(50))
    by_url = db.Column(db.Unicode(80))
    date_from = db.Column(db.DateTime)
    date_till = db.Column(db.DateTime)
    num_users = db.Column(db.Integer)
    num_tokens = db.Column(db.Integer)
    num_clients = db.Column(db.Integer)
    level = db.Column(db.Unicode(80))
    signature = db.Column(db.Unicode(640))

    def save(self):
        subscription = Subscription.query.filter(
            Subscription.application == self.application).first()
        if subscription is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            ret = self.id
        else:
            # update
            values = self.get()
            Subscription.query.filter(
                Subscription.id == subscription.id).update(values)
            ret = subscription.id
        db.session.commit()
        return ret

    def __repr__(self):
        return "<Subscription [{0!s}][{1!s}:{2!s}:{3!s}]>".format(
            self.id, self.application, self.for_name, self.by_name)

    def get(self):
        """
        Return the database object as dict
        :return:
        """
        d = {}
        for attr in Subscription.__table__.columns.keys():
            if getattr(self, attr) is not None:
                d[attr] = getattr(self, attr)
        return d


class EventCounter(db.Model):
    """
    This table stores counters of the event handler "Counter".

    Note that an event counter name does *not* correspond to just one,
    but rather *several* table rows, because we store event counters
    for each privacyIDEA node separately.
    This is intended to improve the performance of replicated setups,
    because each privacyIDEA node then only writes to its own "private"
    table row. This way, we avoid locking issues that would occur
    if all nodes write to the same table row.
    """
    __tablename__ = 'eventcounter'
    id = db.Column(db.Integer, Sequence("eventcounter_seq"), primary_key=True)
    counter_name = db.Column(db.Unicode(80), nullable=False)
    counter_value = db.Column(db.Integer, default=0)
    node = db.Column(db.Unicode(255), nullable=False)
    __table_args__ = (db.UniqueConstraint('counter_name',
                                          'node',
                                          name='evctr_1'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, name, value=0, node=""):
        self.counter_value = value
        self.counter_name = name
        self.node = node
        self.save()

    def save(self):
        db.session.add(self)
        db.session.commit()

    def delete(self):
        ret = self.counter_name
        db.session.delete(self)
        db.session.commit()
        return ret

    def increase(self):
        """
        Increase the value of a counter
        :return:
        """
        self.counter_value = self.counter_value + 1
        self.save()

    def decrease(self):
        """
        Decrease the value of a counter.
        :return:
        """
        self.counter_value = self.counter_value - 1
        self.save()


### Audit

audit_column_length = {"signature": 620,
                       "action": 50,
                       "serial": 40,
                       "token_type": 12,
                       "user": 20,
                       "realm": 20,
                       "resolver": 50,
                       "administrator": 20,
                       "action_detail": 50,
                       "info": 50,
                       "privacyidea_server": 255,
                       "client": 50,
                       "loglevel": 12,
                       "clearance_level": 12,
                       "thread_id": 20,
                       "authentication": 12,
                       "user_agent": 20,
                       "user_agent_version": 20,
                       "policies": 255,
                       "container_serial": 20,
                       "container_type": 20}
AUDIT_TABLE_NAME = 'pidea_audit'


class Audit(MethodsMixin, db.Model):
    """
    This class stores the Audit entries
    """
    __tablename__ = AUDIT_TABLE_NAME
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("audit_seq"), primary_key=True)
    date = db.Column(db.DateTime, index=True)
    startdate = db.Column(db.DateTime)
    duration = db.Column(db.Interval(second_precision=6))
    signature = db.Column(db.Unicode(audit_column_length.get("signature")))
    action = db.Column(db.Unicode(audit_column_length.get("action")))
    success = db.Column(db.Integer)
    authentication = db.Column(db.Unicode(audit_column_length.get("authentication")))
    serial = db.Column(db.Unicode(audit_column_length.get("serial")))
    token_type = db.Column(db.Unicode(audit_column_length.get("token_type")))
    container_serial = db.Column(db.Unicode(audit_column_length.get("container_serial")))
    container_type = db.Column(db.Unicode(audit_column_length.get("container_type")))
    user = db.Column(db.Unicode(audit_column_length.get("user")), index=True)
    realm = db.Column(db.Unicode(audit_column_length.get("realm")))
    resolver = db.Column(db.Unicode(audit_column_length.get("resolver")))
    administrator = db.Column(
        db.Unicode(audit_column_length.get("administrator")))
    action_detail = db.Column(
        db.Unicode(audit_column_length.get("action_detail")))
    info = db.Column(db.Unicode(audit_column_length.get("info")))
    privacyidea_server = db.Column(
        db.Unicode(audit_column_length.get("privacyidea_server")))
    client = db.Column(db.Unicode(audit_column_length.get("client")))
    user_agent = db.Column(db.Unicode(audit_column_length.get("user_agent")))
    user_agent_version = db.Column(db.Unicode(audit_column_length.get("user_agent_version")))
    loglevel = db.Column(db.Unicode(audit_column_length.get("loglevel")))
    clearance_level = db.Column(db.Unicode(audit_column_length.get(
        "clearance_level")))
    thread_id = db.Column(db.Unicode(audit_column_length.get("thread_id")))
    policies = db.Column(db.Unicode(audit_column_length.get("policies")))

    def __init__(self,
                 action="",
                 success=0,
                 authentication="",
                 serial="",
                 token_type="",
                 container_serial="",
                 container_type="",
                 user="",
                 realm="",
                 resolver="",
                 administrator="",
                 action_detail="",
                 info="",
                 privacyidea_server="",
                 client="",
                 user_agent="",
                 user_agent_version="",
                 loglevel="default",
                 clearance_level="default",
                 thread_id="0",
                 policies="",
                 startdate=None,
                 duration=None
                 ):
        self.signature = ""
        self.date = datetime.now()
        self.startdate = startdate
        self.duration = duration
        self.action = convert_column_to_unicode(action)
        self.success = success
        self.authentication = convert_column_to_unicode(authentication)
        self.serial = convert_column_to_unicode(serial)
        self.token_type = convert_column_to_unicode(token_type)
        self.container_serial = convert_column_to_unicode(container_serial)
        self.container_type = convert_column_to_unicode(container_type)
        self.user = convert_column_to_unicode(user)
        self.realm = convert_column_to_unicode(realm)
        self.resolver = convert_column_to_unicode(resolver)
        self.administrator = convert_column_to_unicode(administrator)
        self.action_detail = convert_column_to_unicode(action_detail)
        self.info = convert_column_to_unicode(info)
        self.privacyidea_server = convert_column_to_unicode(privacyidea_server)
        self.client = convert_column_to_unicode(client)
        self.loglevel = convert_column_to_unicode(loglevel)
        self.clearance_level = convert_column_to_unicode(clearance_level)
        self.thread_id = convert_column_to_unicode(thread_id)
        self.policies = convert_column_to_unicode(policies)
        self.user_agent = convert_column_to_unicode(user_agent)
        self.user_agent_version = convert_column_to_unicode(user_agent_version)


### User Cache

class UserCache(MethodsMixin, db.Model):
    __tablename__ = 'usercache'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("usercache_seq"), primary_key=True)
    username = db.Column(db.Unicode(64), default="", index=True)
    used_login = db.Column(db.Unicode(64), default="", index=True)
    resolver = db.Column(db.Unicode(120), default='')
    user_id = db.Column(db.Unicode(320), default='', index=True)
    timestamp = db.Column(db.DateTime, index=True)

    def __init__(self, username, used_login, resolver, user_id, timestamp):
        self.username = username
        self.used_login = used_login
        self.resolver = resolver
        self.user_id = user_id
        self.timestamp = timestamp


class AuthCache(MethodsMixin, db.Model):
    __tablename__ = 'authcache'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("authcache_seq"), primary_key=True)
    first_auth = db.Column(db.DateTime, index=True)
    last_auth = db.Column(db.DateTime, index=True)
    username = db.Column(db.Unicode(64), default="", index=True)
    resolver = db.Column(db.Unicode(120), default='', index=True)
    realm = db.Column(db.Unicode(120), default='', index=True)
    client_ip = db.Column(db.Unicode(40), default="")
    user_agent = db.Column(db.Unicode(120), default="")
    auth_count = db.Column(db.Integer, default=0)
    # We can hash the password like this:
    # binascii.hexlify(hashlib.sha256("secret123456").digest())
    authentication = db.Column(db.Unicode(255), default="")

    def __init__(self, username, realm, resolver, authentication,
                 first_auth=None, last_auth=None):
        self.username = username
        self.realm = realm
        self.resolver = resolver
        self.authentication = authentication
        self.first_auth = first_auth if first_auth else datetime.utcnow()
        self.last_auth = last_auth if last_auth else self.first_auth


### Periodic Tasks

class PeriodicTask(MethodsMixin, db.Model):
    """
    This class stores tasks that should be run periodically.
    """
    __tablename__ = 'periodictask'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("periodictask_seq"), primary_key=True)
    name = db.Column(db.Unicode(64), unique=True, nullable=False)
    active = db.Column(db.Boolean, default=True, nullable=False)
    retry_if_failed = db.Column(db.Boolean, default=True, nullable=False)
    interval = db.Column(db.Unicode(256), nullable=False)
    nodes = db.Column(db.Unicode(256), nullable=False)
    taskmodule = db.Column(db.Unicode(256), nullable=False)
    ordering = db.Column(db.Integer, nullable=False, default=0)
    last_update = db.Column(db.DateTime(False), nullable=False)
    options = db.relationship('PeriodicTaskOption',
                              lazy='dynamic',
                              backref='periodictask')
    last_runs = db.relationship('PeriodicTaskLastRun',
                                lazy='dynamic',
                                backref='periodictask')

    def __init__(self, name, active, interval, node_list, taskmodule, ordering, options=None, id=None,
                 retry_if_failed=True):
        """
        :param name: Unique name of the periodic task as unicode
        :param active: a boolean
        :param retry_if_failed: a boalean
        :param interval: a unicode specifying the periodicity of the task
        :param node_list: a list of unicodes, denoting the node names that should execute that task.
                          If we update an existing PeriodicTask entry, PeriodicTaskLastRun entries
                          referring to nodes that are not present in ``node_list`` any more will be deleted.
        :param taskmodule: a unicode
        :param ordering: an integer. Lower tasks are executed first.
        :param options: a dictionary of options, mapping unicode keys to values. Values will be converted to unicode.
                        If we update an existing PeriodicTask entry, all options that have been set previously
                        but are not present in ``options`` will be deleted.
        :param id: the ID of an existing entry, if any
        """
        self.id = id
        self.name = name
        self.active = active
        self.retry_if_failed = retry_if_failed
        self.interval = interval
        self.nodes = ", ".join(node_list)
        self.taskmodule = taskmodule
        self.ordering = ordering
        self.save()
        # add the options to the periodic task
        options = options or {}
        for k, v in options.items():
            PeriodicTaskOption(periodictask_id=self.id, key=k, value=v)
        # remove all leftover options
        all_options = PeriodicTaskOption.query.filter_by(periodictask_id=self.id).all()
        for option in all_options:
            if option.key not in options:
                PeriodicTaskOption.query.filter_by(id=option.id).delete()
        # remove all leftover last_runs
        all_last_runs = PeriodicTaskLastRun.query.filter_by(periodictask_id=self.id).all()
        for last_run in all_last_runs:
            if last_run.node not in node_list:
                PeriodicTaskLastRun.query.filter_by(id=last_run.id).delete()
        db.session.commit()

    @property
    def aware_last_update(self):
        """
        Return self.last_update with attached UTC tzinfo
        """
        return self.last_update.replace(tzinfo=tzutc())

    def get(self):
        """
        Return the serialized periodic task object including the options and last runs.
        The last runs are returned as timezone-aware UTC datetimes.

        :return: complete dict
        """
        return {"id": self.id,
                "name": self.name,
                "active": self.active,
                "interval": self.interval,
                "nodes": [node.strip() for node in self.nodes.split(",")],
                "taskmodule": self.taskmodule,
                "retry_if_failed": self.retry_if_failed,
                "last_update": self.aware_last_update,
                "ordering": self.ordering,
                "options": dict((option.key, option.value) for option in self.options),
                "last_runs": dict((last_run.node, last_run.aware_timestamp) for last_run in self.last_runs)}

    def save(self):
        """
        If the entry has an ID set, update the entry. If not, create one.
        Set ``last_update`` to the current time.
        :return: the entry ID
        """
        self.last_update = datetime.utcnow()
        if self.id is None:
            # create a new one
            db.session.add(self)
        else:
            # update
            PeriodicTask.query.filter_by(id=self.id).update({
                "name": self.name,
                "active": self.active,
                "interval": self.interval,
                "nodes": self.nodes,
                "taskmodule": self.taskmodule,
                "ordering": self.ordering,
                "retry_if_failed": self.retry_if_failed,
                "last_update": self.last_update,
            })
        db.session.commit()
        return self.id

    def delete(self):
        ret = self.id
        # delete all PeriodicTaskOptions and PeriodicTaskLastRuns before deleting myself
        db.session.query(PeriodicTaskOption).filter_by(periodictask_id=ret).delete()
        db.session.query(PeriodicTaskLastRun).filter_by(periodictask_id=ret).delete()
        db.session.delete(self)
        db.session.commit()
        return ret

    def set_last_run(self, node, timestamp):
        """
        Store the information that the last run of the periodic job occurred on ``node`` at ``timestamp``.
        :param node: Node name as a string
        :param timestamp: Timestamp as UTC datetime (without timezone information)
        :return:
        """
        PeriodicTaskLastRun(self.id, node, timestamp)


class PeriodicTaskOption(db.Model):
    """
    Each PeriodicTask entry can have additional options according to the
    task module.
    """
    __tablename__ = 'periodictaskoption'
    id = db.Column(db.Integer, Sequence("periodictaskopt_seq"),
                   primary_key=True)
    periodictask_id = db.Column(db.Integer, db.ForeignKey('periodictask.id'))
    key = db.Column(db.Unicode(255), nullable=False)
    value = db.Column(db.Unicode(2000), default='')

    __table_args__ = (db.UniqueConstraint('periodictask_id',
                                          'key',
                                          name='ptoix_1'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, periodictask_id, key, value):
        self.periodictask_id = periodictask_id
        self.key = key
        self.value = convert_column_to_unicode(value)
        self.save()

    def save(self):
        """
        Create or update a PeriodicTaskOption entry, depending on the value of ``self.id``
        :return: the entry ID
        """
        option = PeriodicTaskOption.query.filter_by(
            periodictask_id=self.periodictask_id, key=self.key
        ).first()
        if option is None:
            # create a new one
            db.session.add(self)
            ret = self.id
        else:
            # update
            PeriodicTaskOption.query.filter_by(periodictask_id=self.periodictask_id, key=self.key).update({
                'value': self.value,
            })
            ret = option.id
        db.session.commit()
        return ret


class PeriodicTaskLastRun(db.Model):
    """
    Each PeriodicTask entry stores, for each node, the timestamp of the last successful run.
    """
    __tablename__ = 'periodictasklastrun'
    id = db.Column(db.Integer, Sequence("periodictasklastrun_seq"),
                   primary_key=True)
    periodictask_id = db.Column(db.Integer, db.ForeignKey('periodictask.id'))
    node = db.Column(db.Unicode(255), nullable=False)
    timestamp = db.Column(db.DateTime(False), nullable=False)

    __table_args__ = (db.UniqueConstraint('periodictask_id',
                                          'node',
                                          name='ptlrix_1'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, periodictask_id, node, timestamp):
        """
        :param periodictask_id: ID of the periodic task we are referring to
        :param node: Node name as unicode
        :param timestamp: Time of the last run as a datetime. A timezone must not be set!
                          We require the time to be given in UTC.
        """
        self.periodictask_id = periodictask_id
        self.node = node
        self.timestamp = timestamp
        self.save()

    @property
    def aware_timestamp(self):
        """
        Return self.timestamp with attached UTC tzinfo
        """
        return self.timestamp.replace(tzinfo=tzutc())

    def save(self):
        """
        Create or update a PeriodicTaskLastRun entry, depending on the value of ``self.id``.
        :return: the entry id
        """
        last_run = PeriodicTaskLastRun.query.filter_by(
            periodictask_id=self.periodictask_id, node=self.node,
        ).first()
        if last_run is None:
            # create a new one
            db.session.add(self)
            ret = self.id
        else:
            # update
            PeriodicTaskLastRun.query.filter_by(periodictask_id=self.periodictask_id, node=self.node).update({
                'timestamp': self.timestamp,
            })
            ret = last_run.id
        db.session.commit()
        return ret


class MonitoringStats(MethodsMixin, db.Model):
    """
    This is the table that stores measured, arbitrary statistic points in time.

    This could be used to store time series but also to store current values,
    by simply fetching the last value from the database.
    """
    __tablename__ = 'monitoringstats'
    id = db.Column(db.Integer, Sequence("monitoringstats_seq"),
                   primary_key=True)
    # We store this as a naive datetime in UTC
    timestamp = db.Column(db.DateTime(False), nullable=False, index=True)
    stats_key = db.Column(db.Unicode(128), nullable=False)
    stats_value = db.Column(db.Integer, nullable=False, default=0)

    __table_args__ = (db.UniqueConstraint('timestamp',
                                          'stats_key',
                                          name='msix_1'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, timestamp, key, value):
        """
        Create a new database entry in the monitoring stats table
        :param timestamp: The time of the measurement point
        :type timestamp: timezone-naive datetime
        :param key: The key of the measurement
        :type key: basestring
        :param value: The value of the measurement
        :type value: Int
        """
        self.timestamp = timestamp
        self.stats_key = key
        self.stats_value = value
        # self.save()


class Serviceid(TimestampMethodsMixin, db.Model):
    """
    The serviceid table contains the defined service IDs. These service ID
    describe services like "webservers" or "dbservers" which e.g. request SSH keys
    from the privacyIDEA system.
    """
    __tablename__ = 'serviceid'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("serviceid_seq"), primary_key=True,
                   nullable=False)
    name = db.Column(db.Unicode(255), default='',
                     unique=True, nullable=False)
    Description = db.Column(db.Unicode(2000), default='')

    @log_with(log)
    def __init__(self, servicename, description=None):
        self.name = servicename
        self.Description = description

    def save(self):
        si = Serviceid.query.filter_by(name=self.name).first()
        if si is None:
            return TimestampMethodsMixin.save(self)
        else:
            # update
            Serviceid.query.filter_by(id=si.id).update({'Description': self.Description})
            ret = si.id
            db.session.commit()
        return ret


class Tokengroup(TimestampMethodsMixin, db.Model):
    """
    The tokengroup table contains the definition of available token groups.
    A token can then be assigned to several of these tokengroups.
    """
    __tablename__ = 'tokengroup'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column(db.Integer, Sequence("tokengroup_seq"), primary_key=True,
                   nullable=False)
    name = db.Column(db.Unicode(255), default='',
                     unique=True, nullable=False)
    Description = db.Column(db.Unicode(2000), default='')

    @log_with(log)
    def __init__(self, groupname, description=None):
        self.name = groupname
        self.Description = description

    def delete(self):
        ret = self.id
        # delete all TokenTokenGroup
        db.session.query(TokenTokengroup) \
            .filter(TokenTokengroup.tokengroup_id == ret) \
            .delete()
        # delete the tokengroup
        db.session.delete(self)
        save_config_timestamp()
        db.session.commit()
        return ret

    def save(self):
        ti_func = Tokengroup.query.filter_by(name=self.name).first
        ti = ti_func()
        if ti is None:
            return TimestampMethodsMixin.save(self)
        else:
            # update
            Tokengroup.query.filter_by(id=ti.id).update({'Description': self.Description})
            ret = ti.id
            db.session.commit()
        return ret


class TokenTokengroup(TimestampMethodsMixin, db.Model):
    """
    This table stores the assignment of tokens to tokengroups.
    A token can be assigned to several different token groups.
    """
    __tablename__ = 'tokentokengroup'
    __table_args__ = (db.UniqueConstraint('token_id',
                                          'tokengroup_id',
                                          name='ttgix_2'),
                      {'mysql_row_format': 'DYNAMIC'})
    id = db.Column(db.Integer(), Sequence("tokentokengroup_seq"), primary_key=True)
    token_id = db.Column(db.Integer(),
                         db.ForeignKey('token.id'))
    tokengroup_id = db.Column(db.Integer(),
                              db.ForeignKey('tokengroup.id'))
    # This creates an attribute "tokengroup_list" in the Token object
    token = db.relationship('Token',
                            lazy='joined',
                            backref='tokengroup_list')
    # This creates an attribute "token_list" in the Tokengroup object
    tokengroup = db.relationship('Tokengroup',
                                 lazy='joined',
                                 backref='token_list')

    def __init__(self, tokengroup_id=0, token_id=0, tokengroupname=None):
        """
        Create a new TokenTokengroup assignment
        :param tokengroup_id: The id of the token group
        :param tokengroupname: the name of the tokengroup
        :param token_id: The id of the token
        """
        if tokengroupname:
            r = Tokengroup.query.filter_by(name=tokengroupname).first()
            if not r:
                raise Exception("tokengroup does not exist")
            self.tokengroup_id = r.id
        if tokengroup_id:
            self.tokengroup_id = tokengroup_id
        self.token_id = token_id

    def save(self):
        """
        We only save this, if it does not exist, yet.
        """
        tr_func = TokenTokengroup.query.filter_by(tokengroup_id=self.tokengroup_id,
                                                  token_id=self.token_id).first
        tr = tr_func()
        if tr is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            if get_app_config_value(SAFE_STORE, False):
                tr = tr_func()
                ret = tr.id
            else:
                ret = self.id
        else:
            ret = self.id
        return ret


class TokenContainer(MethodsMixin, db.Model):
    """
    The "Tokencontainer" table contains the containers and their associated tokens.
    """

    __tablename__ = 'tokencontainer'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column("id", db.Integer, db.Identity(), primary_key=True)
    type = db.Column(db.Unicode(100), default='Generic', nullable=False)
    description = db.Column(db.Unicode(1024), default='')
    tokens = db.relationship('Token', secondary='tokencontainertoken', back_populates='container')
    serial = db.Column(db.Unicode(40), default='', unique=True, nullable=False, index=True)
    owners = db.relationship('TokenContainerOwner', lazy='dynamic', back_populates='container',
                             cascade="all, delete-orphan")
    last_seen = db.Column(db.DateTime, default=None)
    last_updated = db.Column(db.DateTime, default=None)
    states = db.relationship('TokenContainerStates', lazy='dynamic', back_populates='container',
                             cascade="all, delete-orphan")
    info_list = db.relationship('TokenContainerInfo', lazy='select', back_populates='container',
                                cascade="all, delete-orphan")
    realms = db.relationship('Realm', secondary='tokencontainerrealm', back_populates='container')
    template_id = db.Column(db.ForeignKey('tokencontainertemplate.id', name="tokencontainertemplate_id"))
    template = db.relationship('TokenContainerTemplate', back_populates='containers')

    def __init__(self, serial, container_type="Generic", tokens=None, description="", states=None):
        self.serial = serial
        self.type = container_type
        self.description = description
        if tokens:
            self.tokens = [t.token for t in tokens]
        if states:
            self.states = states

    def set_info(self, info):
        """
        Set the additional container info for this container

        Entries that end with ".type" are used as type for the keys.
        I.e. two entries sshkey="XYZ" and sshkey.type="password" will store
        the key sshkey as type "password".

        :param info: The key-values to set for this container
        :type info: dict
        """
        if not self.id:
            # If there is no ID to reference the container, we need to save the
            # container
            self.save()
        types = {}
        for k, v in info.items():
            if k and k.endswith(".type"):
                types[".".join(k.split(".")[:-1])] = v
        for k, v in info.items():
            if k and not k.endswith(".type"):
                TokenContainerInfo(self.id, k, v,
                                   type=types.get(k)).save(persistent=False)
        db.session.commit()


class TokenContainerOwner(MethodsMixin, db.Model):
    __tablename__ = 'tokencontainerowner'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column("id", db.Integer, db.Identity(), primary_key=True)
    container_id = db.Column(db.Integer(), db.ForeignKey("tokencontainer.id"))
    container = db.relationship('TokenContainer', back_populates='owners')
    resolver = db.Column(db.Unicode(120), default='', index=True)
    user_id = db.Column(db.Unicode(320), default='', index=True)
    realm_id = db.Column(db.Integer(), db.ForeignKey('realm.id'))
    realm = db.relationship('Realm', lazy='joined', backref='tokencontainerowners')

    def __init__(self, container_id=None, container_serial=None, resolver=None, user_id=None, realm_id=None,
                 realm_name=None):
        """
        Create a new TokenContainerOwner assignment.

        :param container_id:
        :param container_serial: alternative to container_id
        :param resolver:
        :param user_id:
        :param realm_id:
        :param realm_name: alternative to realm_id
        """
        if realm_id is not None:
            self.realm_id = realm_id
        elif realm_name:
            realm = Realm.query.filter_by(name=realm_name).first()
            self.realm_id = realm.id
        if container_id is not None:
            self.container_id = container_id
        elif container_serial:
            container = TokenContainer.query.filter_by(serial=container_serial).first()
            self.container_id = container.id
        self.resolver = resolver
        self.user_id = user_id

    def save(self, persistent=True):
        to_func = TokenContainerOwner.query.filter_by(container_id=self.container_id,
                                                      user_id=self.user_id,
                                                      realm_id=self.realm_id,
                                                      resolver=self.resolver).first
        to = to_func()
        if to is None:
            # This very assignment does not exist, yet:
            db.session.add(self)
            db.session.commit()
            if get_app_config_value(SAFE_STORE, False):
                to = to_func()
                ret = to.id
            else:
                ret = self.id
        else:
            ret = to.id
            # There is nothing to update

        if persistent:
            db.session.commit()
        return ret


class TokenContainerStates(MethodsMixin, db.Model):
    __tablename__ = 'tokencontainerstates'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column("id", db.Integer, db.Identity(), primary_key=True)
    container_id = db.Column(db.Integer(), db.ForeignKey("tokencontainer.id"))
    container = db.relationship("TokenContainer", back_populates="states")
    state = db.Column(db.Unicode(100), default='active', nullable=False)

    """
    The table "tokencontainerstates" is used to store the states of the container. A container can be in several states.
    """

    def __init__(self, container_id=None, state="active"):
        self.container_id = container_id
        self.state = state


class TokenContainerInfo(MethodsMixin, db.Model):
    """
    The table "tokencontainerinfo" is used to store additional, long information that
    is specific to the containertype.

    The tokencontainerinfo is reference by the foreign key to the "tokencontainer" table.
    """
    __tablename__ = 'tokencontainerinfo'
    id = db.Column(db.Integer, db.Identity(), primary_key=True)
    key = db.Column(db.Unicode(255), nullable=False)
    value = db.Column(db.UnicodeText(), default='')
    type = db.Column(db.Unicode(100), default='')
    description = db.Column(db.Unicode(2000), default='')
    container_id = db.Column(db.Integer(), db.ForeignKey('tokencontainer.id'), index=True)
    container = db.relationship('TokenContainer', back_populates='info_list')
    __table_args__ = (db.UniqueConstraint('container_id', 'key', name='container_id_constraint'),
                      {'mysql_row_format': 'DYNAMIC'})

    def __init__(self, container_id, key, value,
                 type=None,
                 description=None):
        """
        Create a new tokencontainerinfo for a given token_id
        """
        self.container_id = container_id
        self.key = key
        self.value = convert_column_to_unicode(value)
        self.type = type
        self.description = description

    def save(self, persistent=True):
        ti_func = TokenContainerInfo.query.filter_by(container_id=self.container_id,
                                                     key=self.key).first
        ti = ti_func()
        if ti is None:
            # create a new one
            db.session.add(self)
            db.session.commit()
            if get_app_config_value(SAFE_STORE, False):
                ti = ti_func()
                ret = ti.id
            else:
                ret = self.id
        else:
            # update
            TokenContainerInfo.query.filter_by(container_id=self.container_id,
                                               key=self.key).update({'value': self.value,
                                                                     'description': self.description,
                                                                     'type': self.type})
            ret = ti.id
        if persistent:
            db.session.commit()
        return ret


class TokenContainerRealm(MethodsMixin, db.Model):
    """
    This table stores to which realms a container is assigned. A container is in the
    realm of the user it is assigned to. But a container can also be put into
    many additional realms.
    """
    __tablename__ = 'tokencontainerrealm'
    container_id = db.Column(db.Integer(), db.ForeignKey("tokencontainer.id"), primary_key=True)
    realm_id = db.Column(db.Integer(), db.ForeignKey('realm.id'), primary_key=True)

    __table_args__ = ({'mysql_row_format': 'DYNAMIC'})


class TokenContainerTemplate(MethodsMixin, db.Model):
    __tablename__ = 'tokencontainertemplate'
    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
    id = db.Column("id", db.Integer, db.Identity(), primary_key=True)
    options = db.Column(db.Unicode(2000), default='')
    name = db.Column(db.Unicode(200), default='')
    container_type = db.Column(db.Unicode(100), default='generic', nullable=False)
    default = db.Column(db.Boolean, default=False, nullable=False)
    containers = db.relationship('TokenContainer', back_populates='template')

    def __init__(self, name, container_type="generic", options='', default=False):
        self.name = name
        self.container_type = container_type
        self.options = options
        self.default = default


class TokenCredentialIdHash(MethodsMixin, db.Model):
    __tablename__ = "tokencredentialidhash"
    id = db.Column("id", db.Integer, db.Identity(), primary_key=True)
    credential_id_hash = db.Column(db.String(256), nullable=False)
    token_id = db.Column(db.Integer(), db.ForeignKey("token.id"), nullable=False)
    __table_args__ = (db.Index('ix_tokencredentialidhash_credentialidhash',
                               'credential_id_hash', unique=True),)

    def __init__(self, credential_id_hash, token_id):
        self.credential_id_hash = credential_id_hash
        self.token_id = token_id
