from io import IOBase as FileType
import copy
import codecs
import json
import math
import os
import pickle
import platform
import re
import sys
import time
import types
import xml.etree.ElementTree as ET
import pandas
from Cryptodome.Cipher import AES
from dateutil import tz
from flask import session, url_for as base_url_for
from flask_login import current_user
from flask_wtf.csrf import generate_csrf
from sqlalchemy import or_, and_, select, delete
import ruamel.yaml
import tzlocal
from docassemble.base.error import DAException
from docassemble.webapp.da_flask_mail import Message  # noqa: F401 # pylint: disable=unused-import
from docassemble.base.functions import pickleable_objects, filename_invalid
from docassemble.base.config import daconfig, hostname, DEBUG_BOOT, START_TIME, boot_log
from docassemble.base.generate_key import random_bytes, random_alphanumeric
from docassemble.base.logger import logmessage
import docassemble.base.functions
import docassemble.base.parse
from docassemble.webapp.app_object import app
from docassemble.webapp.core.models import Uploads, UploadsUserAuth, UploadsRoleAuth, SpeakList, ObjectStorage, Shortener, MachineLearning, GlobalObjectStorage, Email, EmailAttachment
from docassemble.webapp.db_object import db
from docassemble.webapp.file_access import get_info_from_file_number, get_info_from_file_reference, url_if_exists  # noqa: F401 # pylint: disable=unused-import
from docassemble.webapp.file_number import get_new_file_number
from docassemble.webapp.files import SavedFile, get_ext_and_mimetype
from docassemble.webapp.fixpickle import fix_pickle_obj, fix_pickle_dict
from docassemble.webapp.mailgun_mail import Mail as MailgunMail
from docassemble.webapp.da_flask_mail import FlaskMail
from docassemble.webapp.packages.models import PackageAuth
from docassemble.webapp.screenreader import to_text
from docassemble.webapp.sendgrid_mail import Mail as SendgridMail
from docassemble.webapp.users.models import UserModel, Role, ChatLog, UserDict, UserDictKeys, UserAuthModel, UserRoles
import docassemble.webapp.cloud
import docassemble.webapp.database
import docassemble.webapp.machinelearning
import docassemble.webapp.setup
import docassemble.webapp.user_database
import docassemble.webapp.worker
if platform.machine() == 'x86_64':
    import docassemble.webapp.google_api

if DEBUG_BOOT:
    boot_log("backend: starting")
TypeType = type(type(None))
NoneType = type(None)
DEBUG = daconfig.get('debug', False)

safeyaml = ruamel.yaml.YAML(typ='safe')

# def elapsed(name_of_function):
#     def elapse_decorator(func):
#         def time_func(*pargs, **kwargs):
#             time_start = time.time()
#             result = func(*pargs, **kwargs)
#             logmessage(name_of_function + ': ' + str(time.time() - time_start))
#             return result
#         return time_func
#     return elapse_decorator


def write_record(key, data):
    new_record = ObjectStorage(key=key, value=pack_object(data))
    db.session.add(new_record)
    db.session.commit()
    return new_record.id


def read_records(key):
    results = {}
    for record in db.session.execute(select(ObjectStorage).filter_by(key=key).order_by(ObjectStorage.id)).scalars():
        results[record.id] = unpack_object(record.value)
    return results


def delete_record(key, the_id):
    db.session.execute(delete(ObjectStorage).filter_by(key=key, id=the_id))
    db.session.commit()


# @elapsed('save_numbered_file')
def save_numbered_file(filename, orig_path, yaml_file_name=None, uid=None):
    if uid is None:
        try:
            uid = docassemble.base.functions.get_uid()
            assert uid is not None
        except:
            uid = unattached_uid()
    if uid is None:
        raise DAException("save_numbered_file: uid not defined")
    file_number = get_new_file_number(uid, filename, yaml_file_name=yaml_file_name)
    extension, mimetype = get_ext_and_mimetype(filename)
    new_file = SavedFile(file_number, extension=extension, fix=True, should_not_exist=True)
    new_file.copy_from(orig_path)
    new_file.save(finalize=True)
    return (file_number, extension, mimetype)


def fix_ml_files(playground_number, current_project):
    playground = SavedFile(playground_number, section='playgroundsources', fix=False)
    changed = False
    for filename in playground.list_of_files():
        if re.match(r'^ml-.*\.json', filename):
            playground.fix()
            try:
                if write_ml_source(playground, playground_number, current_project, filename, finalize=False):
                    changed = True
            except:
                logmessage("Error writing machine learning source file " + str(filename))
    if changed:
        playground.finalize()


def is_package_ml(parts):
    if len(parts) == 3 and parts[0].startswith('docassemble.') and re.match(r'data/sources/.*\.json', parts[1]):
        return True
    return False


def project_name(name):
    return '' if name == 'default' else name


def add_project(filename, current_project):
    if current_project == 'default':
        return filename
    return os.path.join(current_project, filename)


def directory_for(area, current_project):
    if current_project == 'default':
        return area.directory
    return os.path.join(area.directory, current_project)


def write_ml_source(playground, playground_number, current_project, filename, finalize=True):
    if re.match(r'ml-.*\.json', filename):
        output = {}
        prefix = 'docassemble.playground' + str(playground_number) + project_name(current_project) + ':data/sources/' + str(filename)
        for record in db.session.execute(select(MachineLearning.group_id, MachineLearning.independent, MachineLearning.dependent, MachineLearning.key).where(MachineLearning.group_id.like(prefix + ':%'))).all():
            parts = record.group_id.split(':')
            if not is_package_ml(parts):
                continue
            if parts[2] not in output:
                output[parts[2]] = []
            the_independent = record.independent
            if the_independent is not None:
                the_independent = fix_pickle_obj(codecs.decode(bytearray(the_independent, encoding='utf-8'), 'base64'))
            the_dependent = record.dependent
            if the_dependent is not None:
                the_dependent = fix_pickle_obj(codecs.decode(bytearray(the_dependent, encoding='utf-8'), 'base64'))
            the_entry = {'independent': the_independent, 'dependent': the_dependent}
            if record.key is not None:
                the_entry['key'] = record.key
            output[parts[2]].append(the_entry)
        if len(output) > 0:
            playground.write_as_json(output, filename=os.path.join(directory_for(playground, current_project), filename))
            if finalize:
                playground.finalize()
            return True
    return False


def user_is_developer(user_id):
    try:
        for user in db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(id=int(user_id))).unique().scalars():
            for role in user.roles:
                if role.name in ('developer', 'admin'):
                    return True
    except:
        pass
    return False


def absolute_filename(the_file):
    match = re.match(r'^docassemble.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*):(.*)', the_file)
    # logmessage("absolute_filename call: " + the_file)
    if match:
        if not user_is_developer(match.group(1)):
            return None
        filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', match.group(3))
        if filename_invalid(filename):
            return None
        # logmessage("absolute_filename: filename is " + filename + " and subdir is " + match.group(2))
        playground = SavedFile(match.group(1), section='playground', fix=True, filename=filename, subdir=match.group(2), must_exist=True)
        return playground
    match = re.match(r'^/playgroundtemplate/([0-9]+)/([A-Za-z0-9]+)/(.*)', the_file)
    if match:
        if not user_is_developer(match.group(1)):
            return None
        filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', match.group(3))
        if filename_invalid(filename):
            return None
        playground = SavedFile(match.group(1), section='playgroundtemplate', fix=True, filename=filename, subdir=match.group(2), must_exist=True)
        return playground
    match = re.match(r'^/playgroundstatic/([0-9]+)/([A-Za-z0-9]+)/(.*)', the_file)
    if match:
        if not user_is_developer(match.group(1)):
            return None
        filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', match.group(3))
        if filename_invalid(filename):
            return None
        playground = SavedFile(match.group(1), section='playgroundstatic', fix=True, filename=filename, subdir=match.group(2), must_exist=True)
        return playground
    match = re.match(r'^/playgroundsources/([0-9]+)/([A-Za-z0-9]+)/(.*)', the_file)
    if match:
        if not user_is_developer(match.group(1)):
            return None
        filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', match.group(3))
        if filename_invalid(filename):
            return None
        playground = SavedFile(match.group(1), section='playgroundsources', fix=True, filename=filename, subdir=match.group(2), must_exist=True)
        write_ml_source(playground, match.group(1), match.group(2), filename)
        return playground
    return None


def get_mail_config():
    the_mail_configs = {}
    default_config = None
    for mail_config in daconfig['mail']:
        if mail_config.get('name', None) == 'default':
            default_config = mail_config
    if default_config is None and len(daconfig['mail']) > 0:
        default_config = daconfig['mail'][0]
    if default_config is None:
        default_config = {'username': None, 'password': None, 'default sender': None, 'server': 'localhost', 'port': 25, 'use ssl': False, 'use tls': True}
    app.config['MAIL_USERNAME'] = default_config.get('username', None)
    app.config['MAIL_PASSWORD'] = default_config.get('password', None)
    app.config['MAIL_DEFAULT_SENDER'] = default_config.get('default sender', None)
    app.config['MAIL_SERVER'] = default_config.get('server', 'localhost')
    app.config['MAIL_PORT'] = default_config.get('port', 25)
    app.config['MAIL_USE_SSL'] = default_config.get('use ssl', False)
    app.config['MAIL_USE_TLS'] = default_config.get('use tls', True)
    count = 0
    for mail_config in daconfig['mail']:
        the_config = copy.deepcopy(mail_config)
        if the_config.get('mailgun api key', None):
            try:
                the_config['mailgun api url'] = mail_config.get('mailgun api url', 'https://api.mailgun.net/v3/%s/messages.mime') % mail_config.get('mailgun domain', 'NOT_USING_MAILGUN')
            except:
                the_config['mailgun api url'] = 'https://api.mailgun.net/v3/%s/messages.mime' % (mail_config.get('mailgun domain', 'NOT_USING_MAILGUN'),)
        if not the_config.get('name', None):
            the_config['name'] = 'config' + str(count)
        the_mail_configs[the_config['name']] = the_config
        if 'default' not in the_mail_configs and mail_config is default_config:
            the_mail_configs['default'] = the_config
        count += 1
    if 'default' not in the_mail_configs:
        the_mail_configs['default'] = default_config
    for config_name, mail_config in the_mail_configs.items():
        if mail_config.get('mailgun domain', None) and mail_config.get('mailgun api key', None):
            mail_class = MailgunMail
            config = {
                'MAILGUN_API_URL': mail_config['mailgun api url'],
                'MAILGUN_API_KEY': mail_config['mailgun api key'],
                'MAIL_DEFAULT_SENDER': mail_config.get('default sender', None),
                'MAIL_DEBUG': app.config.get('MAIL_DEBUG', False),
                'MAIL_MAX_EMAILS': app.config.get('MAIL_MAX_EMAILS'),
                'MAIL_SUPPRESS_SEND': app.config.get('MAIL_SUPPRESS_SEND', False),
                'MAIL_ASCII_ATTACHMENTS': app.config.get('MAIL_ASCII_ATTACHMENTS', False)
            }
        elif 'sendgrid api key' in mail_config and mail_config['sendgrid api key']:
            mail_class = SendgridMail
            config = {
                'SENDGRID_API_KEY': mail_config['sendgrid api key'],
                'MAIL_DEFAULT_SENDER': mail_config.get('default sender', None),
                'MAIL_DEBUG': app.config.get('MAIL_DEBUG', False),
                'MAIL_MAX_EMAILS': app.config.get('MAIL_MAX_EMAILS'),
                'MAIL_SUPPRESS_SEND': app.config.get('MAIL_SUPPRESS_SEND', False),
                'MAIL_ASCII_ATTACHMENTS': app.config.get('MAIL_ASCII_ATTACHMENTS', False)
            }
        else:
            mail_class = FlaskMail
            config = {
                'MAIL_SERVER': mail_config.get('server', 'localhost'),
                'MAIL_USERNAME': mail_config.get('username', None),
                'MAIL_PASSWORD': mail_config.get('password', None),
                'MAIL_PORT': mail_config.get('port', 25),
                'MAIL_USE_TLS': mail_config.get('use tls', True),
                'MAIL_USE_SSL': mail_config.get('use ssl', False),
                'MAIL_DEFAULT_SENDER': mail_config.get('default sender', None),
                'MAIL_DEBUG': app.config.get('MAIL_DEBUG', False),
                'MAIL_MAX_EMAILS': app.config.get('MAIL_MAX_EMAILS'),
                'MAIL_SUPPRESS_SEND': app.config.get('MAIL_SUPPRESS_SEND', False),
                'MAIL_ASCII_ATTACHMENTS': app.config.get('MAIL_ASCII_ATTACHMENTS', False)
            }
        if config_name == 'default':
            mail_config['mail'] = mail_class(app=app, config=config)
        else:
            mail_config['mail'] = mail_class(config=config)
    return the_mail_configs

if DEBUG_BOOT:
    boot_log("backend: getting email configuration")

mail_configs = get_mail_config()

if DEBUG_BOOT:
    boot_log("backend: finished getting email configuration")


def da_send_mail(the_message, config='default'):
    if config not in mail_configs:
        logmessage("invalid mail configuration " + config)
        config = 'default'
    mail_configs[config]['mail'].send(the_message)

DEFAULT_LANGUAGE = daconfig.get('language', 'en')
DEFAULT_LOCALE = daconfig.get('locale', 'en_US.utf8')
DEFAULT_DIALECT = daconfig.get('dialect', 'us')
DEFAULT_VOICE = daconfig.get('voice', None)
if 'timezone' in daconfig and daconfig['timezone'] is not None:
    DEFAULT_TIMEZONE = daconfig['timezone']
else:
    try:
        DEFAULT_TIMEZONE = tzlocal.get_localzone_name()
    except:
        DEFAULT_TIMEZONE = 'America/New_York'

COOKIELESS_SESSIONS = daconfig.get('cookieless sessions', False)


def url_for(*pargs, **kwargs):
    if 'jsembed' in docassemble.base.functions.this_thread.misc:
        kwargs['_external'] = True
        if pargs[0] == 'index':
            kwargs['js_target'] = docassemble.base.functions.this_thread.misc['jsembed']
    if COOKIELESS_SESSIONS:
        if pargs[0] == 'index':
            pargs = list(pargs)
            pargs[0] = 'html_index'
        kwargs['_external'] = True
    return base_url_for(*pargs, **kwargs)


def sql_get(key, secret=None):
    for record in db.session.execute(select(GlobalObjectStorage).filter_by(key=key)).scalars():
        if record.encrypted:
            try:
                result = decrypt_object(record.value, secret)
            except:
                raise DAException("Unable to decrypt stored object.")
        else:
            try:
                result = unpack_object(record.value)
            except:
                raise DAException("Unable to unpack stored object.")
        return result
    return None


def sql_defined(key):
    record = db.session.execute(select(GlobalObjectStorage.id).filter_by(key=key)).first()
    if record is None:
        return False
    return True


def sql_set(key, val, encrypted=True, secret=None, the_user_id=None):
    user_id, temp_user_id = parse_the_user_id(the_user_id)
    updated = False
    for record in db.session.execute(select(GlobalObjectStorage).filter_by(key=key).with_for_update()).scalars():
        record.user_id = user_id
        record.temp_user_id = temp_user_id
        record.encrypted = encrypted
        if encrypted:
            record.value = encrypt_object(val, secret)
        else:
            record.value = pack_object(val)
        updated = True
    if not updated:
        if encrypted:
            record = GlobalObjectStorage(key=key, value=encrypt_object(val, secret), encrypted=True, user_id=user_id, temp_user_id=temp_user_id)
        else:
            record = GlobalObjectStorage(key=key, value=pack_object(val), encrypted=False, user_id=user_id, temp_user_id=temp_user_id)
        db.session.add(record)
    db.session.commit()


def sql_delete(key):
    db.session.execute(delete(GlobalObjectStorage).filter_by(key=key))
    db.session.commit()


def sql_keys(prefix):
    n = len(prefix)
    stmt = select(GlobalObjectStorage.key).where(GlobalObjectStorage.key.like(prefix + '%'))
    return list(set(y.key[n:] for y in db.session.execute(stmt)))


def get_info_from_file_reference_with_uids(*pargs, **kwargs):
    if 'uids' not in kwargs:
        kwargs['uids'] = get_session_uids()
    return get_info_from_file_reference(*pargs, **kwargs)


def get_info_from_file_number_with_uids(*pargs, **kwargs):
    if 'uids' not in kwargs:
        kwargs['uids'] = get_session_uids()
    return get_info_from_file_number(*pargs, **kwargs)

if DEBUG_BOOT:
    boot_log("backend: configuring common functions")

classes = daconfig['table css class'].split(',')
DEFAULT_TABLE_CLASS = json.dumps(classes[0].strip())
if len(classes) > 1:
    DEFAULT_THEAD_CLASS = json.dumps(classes[1].strip())
else:
    DEFAULT_THEAD_CLASS = None
del classes

DEFAULT_COUNTRY = daconfig.get('country', None) or re.sub(r'^.*_', '', re.sub(r'\..*', r'', DEFAULT_LOCALE))


docassemble.base.functions.update_server(default_language=DEFAULT_LANGUAGE,
                                         default_locale=DEFAULT_LOCALE,
                                         default_dialect=DEFAULT_DIALECT,
                                         default_voice=DEFAULT_VOICE,
                                         default_timezone=DEFAULT_TIMEZONE,
                                         default_country=DEFAULT_COUNTRY,
                                         daconfig=daconfig,
                                         hostname=hostname,
                                         debug_status=DEBUG,
                                         save_numbered_file=save_numbered_file,
                                         send_mail=da_send_mail,
                                         absolute_filename=absolute_filename,
                                         write_record=write_record,
                                         read_records=read_records,
                                         delete_record=delete_record,
                                         generate_csrf=generate_csrf,
                                         url_for=url_for,
                                         get_new_file_number=get_new_file_number,
                                         get_ext_and_mimetype=get_ext_and_mimetype,
                                         file_finder=get_info_from_file_reference_with_uids,
                                         file_number_finder=get_info_from_file_number_with_uids,
                                         server_sql_get=sql_get,
                                         server_sql_defined=sql_defined,
                                         server_sql_set=sql_set,
                                         server_sql_delete=sql_delete,
                                         server_sql_keys=sql_keys,
                                         alchemy_url=docassemble.webapp.user_database.alchemy_url,
                                         connect_args=docassemble.webapp.user_database.connect_args,
                                         default_table_class=DEFAULT_TABLE_CLASS,
                                         default_thead_class=DEFAULT_THEAD_CLASS,
                                         to_text=to_text)
docassemble.base.functions.set_language(DEFAULT_LANGUAGE, dialect=DEFAULT_DIALECT, voice=DEFAULT_VOICE)
docassemble.base.functions.set_locale(DEFAULT_LOCALE)
docassemble.base.functions.update_locale()

if DEBUG_BOOT:
    boot_log("backend: finished configuring common functions")


def fix_words():
    word_file_list = daconfig.get('words', [])
    if not isinstance(word_file_list, list):
        word_file_list = [word_file_list]
    for word_file in word_file_list:
        # logmessage("Reading from " + str(word_file))
        if not isinstance(word_file, str):
            logmessage("Error reading words: file references must be plain text.")
            continue
        filename = docassemble.base.functions.static_filename_path(word_file)
        if filename is None:
            logmessage("Error reading " + str(word_file) + ": file not found.")
            continue
        if os.path.isfile(filename):
            if filename.lower().endswith('.yaml') or filename.lower().endswith('.yml'):
                with open(filename, 'r', encoding='utf-8') as stream:
                    try:
                        for document in safeyaml.load_all(stream):
                            if document and isinstance(document, dict):
                                for lang, words in document.items():
                                    if isinstance(words, dict):
                                        docassemble.base.functions.update_word_collection(lang, words)
                                    else:
                                        logmessage("Error reading " + str(word_file) + ": words not in dictionary form.")
                            else:
                                logmessage("Error reading " + str(word_file) + ": yaml file not in dictionary form.")
                    except:
                        logmessage("Error reading " + str(word_file) + ": yaml could not be processed.")
            elif filename.lower().endswith('.xlsx'):
                try:
                    df = pandas.read_excel(filename, na_values=['#NA', '#N/A'], keep_default_na=False)
                    invalid = False
                    for column_name in ('orig_lang', 'tr_lang', 'orig_text', 'tr_text'):
                        if column_name not in df.columns:
                            invalid = True
                            break
                    if invalid:
                        logmessage("Error reading " + str(word_file) + ": xlsx did not have the correct columns.")
                        continue
                    translations = {}
                    problems = []
                    for indexno in df.index:
                        try:
                            assert df['orig_lang'][indexno]
                            assert df['tr_lang'][indexno]
                            assert df['orig_text'][indexno] != ''
                            assert df['tr_text'][indexno] != ''
                            if isinstance(df['orig_text'][indexno], float):
                                assert not math.isnan(df['orig_text'][indexno])
                            if isinstance(df['tr_text'][indexno], float):
                                assert not math.isnan(df['tr_text'][indexno])
                        except:
                            problems.append(str(indexno + 2))
                            continue
                        if df['tr_lang'][indexno] not in translations:
                            translations[df['tr_lang'][indexno]] = {}
                        translations[df['tr_lang'][indexno]][str(df['orig_text'][indexno])] = str(df['tr_text'][indexno])
                    for lang, the_dict in translations.items():
                        try:
                            docassemble.base.functions.update_word_collection(lang, the_dict)
                        except:
                            logmessage("Error reading " + str(word_file) + ": xlsx for language " + lang + " could not be processed.")
                    if len(problems) > 0:
                        logmessage("Error reading " + str(word_file) + ": could not read lines " + ", ".join(problems) + ".")
                except Exception as err:
                    logmessage("Error reading " + str(word_file) + ": xlsx processing raised exception " + err.__class__.__name__ + ": " + str(err))
            elif filename.lower().endswith('.xlf') or filename.lower().endswith('.xliff'):
                try:
                    tree = ET.parse(filename)
                    root = tree.getroot()
                    translations = {}
                    if root.attrib['version'] == "1.2":
                        for the_file in root.iter('{urn:oasis:names:tc:xliff:document:1.2}file'):
                            target_lang = the_file.attrib.get('target-language', 'en')
                            if target_lang not in translations:
                                translations[target_lang] = {}
                            for transunit in the_file.iter('{urn:oasis:names:tc:xliff:document:1.2}trans-unit'):
                                orig_text = ''
                                tr_text = ''
                                for source in transunit.iter('{urn:oasis:names:tc:xliff:document:1.2}source'):
                                    if source.text:
                                        orig_text += source.text
                                    for mrk in source:
                                        orig_text += mrk.text
                                        if mrk.tail:
                                            orig_text += mrk.tail
                                for target in transunit.iter('{urn:oasis:names:tc:xliff:document:1.2}target'):
                                    if target.text:
                                        tr_text += target.text
                                    for mrk in target:
                                        tr_text += mrk.text
                                        if mrk.tail:
                                            tr_text += mrk.tail
                                if orig_text == '' or tr_text == '':
                                    continue
                                translations[target_lang][orig_text] = tr_text
                    elif root.attrib['version'] == "2.0":
                        target_lang = root.attrib['trgLang']
                        if target_lang not in translations:
                            translations[target_lang] = {}
                        for segment in root.iter('{urn:oasis:names:tc:xliff:document:2.0}segment'):
                            orig_text = ''
                            tr_text = ''
                            for source in segment.iter('{urn:oasis:names:tc:xliff:document:2.0}source'):
                                if source.text:
                                    orig_text += source.text
                                for mrk in source:
                                    orig_text += mrk.text
                                    if mrk.tail:
                                        orig_text += mrk.tail
                            for target in segment.iter('{urn:oasis:names:tc:xliff:document:2.0}target'):
                                if target.text:
                                    tr_text += target.text
                                for mrk in target:
                                    tr_text += mrk.text
                                    if mrk.tail:
                                        tr_text += mrk.tail
                            if orig_text == '' or tr_text == '':
                                continue
                            translations[target_lang][orig_text] = tr_text
                    else:
                        logmessage("Error reading " + str(word_file) + ": invalid XLIFF version.")
                    for lang, the_dict in translations.items():
                        try:
                            docassemble.base.functions.update_word_collection(lang, the_dict)
                        except:
                            logmessage("Error reading " + str(word_file) + ": xlf for language " + lang + " could not be processed.")
                except Exception as err:
                    logmessage("Error reading " + str(word_file) + ": xlf processing raised exception " + err.__class__.__name__ + ": " + str(err))
            else:
                logmessage("filename " + filename + " had an unknown type")
        else:
            logmessage("filename " + filename + " did not exist")

if DEBUG_BOOT:
    boot_log("backend: processing translations")

if 'currency symbol' in daconfig:
    docassemble.base.functions.update_language_function('*', 'currency_symbol', lambda: daconfig['currency symbol'])

if DEBUG_BOOT:
    boot_log("backend: finished processing translations")
    boot_log("backend: obtaining cloud object")

cloud = docassemble.webapp.cloud.get_cloud()

cloud_cache = {}


def cloud_custom(provider, config):
    config_id = str(provider) + str(config)
    if config_id in cloud_cache:
        return cloud_cache[config_id]
    the_config = daconfig.get(config, None)
    if the_config is None or not isinstance(the_config, dict):
        logmessage("cloud_custom: invalid cloud configuration")
        return None
    cloud_cache[config_id] = docassemble.webapp.cloud.get_custom_cloud(provider, the_config)
    return cloud_cache[config_id]

docassemble.base.functions.update_server(cloud=cloud,
                                         cloud_custom=cloud_custom)

if platform.machine() == 'x86_64':
    docassemble.base.functions.update_server(google_api=docassemble.webapp.google_api)

if DEBUG_BOOT:
    boot_log("backend: finished obtaining cloud object")

initial_dict = {'_internal': {'session_local': {}, 'device_local': {}, 'user_local': {}, 'dirty': {}, 'progress': 0, 'tracker': 0, 'docvar': {}, 'doc_cache': {}, 'steps': 1, 'steps_offset': 0, 'secret': None, 'informed': {}, 'livehelp': {'availability': 'unavailable', 'mode': 'help', 'roles': [], 'partner_roles': []}, 'answered': set(), 'answers': {}, 'objselections': {}, 'starttime': None, 'modtime': None, 'accesstime': {}, 'tasks': {}, 'gather': [], 'event_stack': {}, 'misc': {}}, 'url_args': {}, 'nav': docassemble.base.functions.DANav()}
# else:
#    initial_dict = {'_internal': {'tracker': 0, 'steps_offset': 0, 'answered': set(), 'answers': {}, 'objselections': {}}, 'url_args': {}}
if 'initial_dict' in daconfig:
    initial_dict.update(daconfig['initial dict'])
docassemble.base.parse.set_initial_dict(initial_dict)

# def absolute_validator(the_file):
#     # logmessage("Running my validator")
#     if the_file.startswith(os.path.join(UPLOAD_DIRECTORY, 'playground')) and current_user.is_authenticated and not current_user.is_anonymous and current_user.has_role('admin', 'developer') and os.path.dirname(the_file) == os.path.join(UPLOAD_DIRECTORY, 'playground', str(current_user.id)):
#         return True
#     return False

# docassemble.base.parse.set_absolute_filename(absolute_filename)
# logmessage("Server started")


# @elapsed('can_access_file_number')
def can_access_file_number(file_number, uids=None):
    upload = db.session.execute(select(Uploads).where(Uploads.indexno == file_number)).scalar()
    if upload is None:
        return False
    if current_user and current_user.is_authenticated and current_user.has_role('admin', 'developer', 'advocate', 'trainer'):
        return True
    if not upload.private:
        return True
    if uids is None or len(uids) == 0:
        new_uid = docassemble.base.functions.get_uid()
        if new_uid is not None:
            uids = [new_uid]
        else:
            uids = []
    if upload.key in uids:
        return True
    if current_user and current_user.is_authenticated:
        if db.session.execute(select(UserDictKeys).filter_by(key=upload.key, user_id=current_user.id)).first() or db.session.execute(select(UploadsUserAuth).filter_by(uploads_indexno=file_number, user_id=current_user.id)).first() or db.session.execute(select(UploadsRoleAuth).join(UserRoles, and_(UserRoles.user_id == current_user.id, UploadsRoleAuth.role_id == UserRoles.role_id)).where(UploadsRoleAuth.uploads_indexno == file_number)).first():
            return True
    elif session and 'tempuser' in session:
        temp_user_id = int(session['tempuser'])
        if db.session.execute(select(UserDictKeys).filter_by(key=upload.key, temp_user_id=temp_user_id)).first() or db.session.execute(select(UploadsUserAuth).filter_by(uploads_indexno=file_number, temp_user_id=temp_user_id)).first():
            return True
    return False

# if in_celery:
#     LOGFILE = daconfig.get('celery flask log', '/tmp/celery-flask.log')
# else:
#     LOGFILE = daconfig.get('flask log', '/tmp/flask.log')

# if not os.path.exists(LOGFILE):
#     with open(LOGFILE, 'a', encoding='utf-8'):
#         os.utime(LOGFILE, None)

# error_file_handler = logging.FileHandler(filename=LOGFILE)
# error_file_handler.setLevel(logging.DEBUG)
# app.logger.addHandler(error_file_handler)

# sys.stderr.write("__name__ is " + str(__name__) + " and __package__ is " + str(__package__) + "\n")

# def flask_logger(message):
#     # app.logger.warning(message)
#     sys.stderr.write(str(message) + "\n")


def pad(the_string):
    return the_string + bytearray((16 - len(the_string) % 16) * chr(16 - len(the_string) % 16), encoding='utf-8')


def unpad(the_string):
    if isinstance(the_string[-1], int):
        return the_string[0:-the_string[-1]]
    return the_string[0:-ord(the_string[-1])]


def encrypt_phrase(phrase, secret):
    iv = random_bytes(16)
    encrypter = AES.new(bytearray(secret, encoding='utf-8'), AES.MODE_CBC, iv)
    if isinstance(phrase, str):
        phrase = bytearray(phrase, 'utf-8')
    return (iv + codecs.encode(encrypter.encrypt(pad(phrase)), 'base64')).decode('utf-8')


def pack_phrase(phrase):
    phrase = bytearray(phrase, encoding='utf-8')
    return codecs.encode(phrase, 'base64').decode('utf-8')


def decrypt_phrase(phrase_string, secret):
    phrase_string = bytearray(phrase_string, encoding='utf-8')
    decrypter = AES.new(bytearray(secret, encoding='utf-8'), AES.MODE_CBC, phrase_string[:16])
    return unpad(decrypter.decrypt(codecs.decode(phrase_string[16:], 'base64'))).decode('utf-8')


def unpack_phrase(phrase_string):
    return codecs.decode(bytearray(phrase_string, encoding='utf-8'), 'base64').decode('utf-8')


def encrypt_dictionary(the_dict, secret):
    iv = random_bytes(16)
    encrypter = AES.new(bytearray(secret, encoding='utf-8'), AES.MODE_CBC, iv)
    return (iv + codecs.encode(encrypter.encrypt(pad(pickle.dumps(pickleable_objects(the_dict)))), 'base64')).decode()


def pack_object(the_object):
    return codecs.encode(pickle.dumps(safe_pickle(the_object)), 'base64').decode()


def unpack_object(the_string):
    the_string = bytearray(the_string, encoding='utf-8')
    return fix_pickle_dict(codecs.decode(the_string, 'base64'))


def encrypt_object(obj, secret):
    iv = random_bytes(16)
    encrypter = AES.new(bytearray(secret, encoding='utf-8'), AES.MODE_CBC, iv)
    return (iv + codecs.encode(encrypter.encrypt(pad(pickle.dumps(safe_pickle(obj)))), 'base64')).decode()


def decrypt_object(obj_string, secret):
    obj_string = bytearray(obj_string, encoding='utf-8')
    decrypter = AES.new(bytearray(secret, encoding='utf-8'), AES.MODE_CBC, obj_string[:16])
    return fix_pickle_obj(unpad(decrypter.decrypt(codecs.decode(obj_string[16:], 'base64'))))


def parse_the_user_id(the_user_id):
    if the_user_id is None:
        return (None, None)
    m = re.match(r'(t?)([0-9]+)', str(the_user_id))
    if m:
        if m.group(1) == 't':
            return None, int(m.group(2))
        return int(m.group(2)), None
    raise DAException("Invalid user ID")


def safe_pickle(the_object):
    if isinstance(the_object, list):
        return [safe_pickle(x) for x in the_object]
    if isinstance(the_object, dict):
        new_dict = {}
        for key, value in the_object.items():
            new_dict[key] = safe_pickle(value)
        return new_dict
    if isinstance(the_object, set):
        new_set = set()
        for sub_object in the_object:
            new_set.add(safe_pickle(sub_object))
        return new_set
    if isinstance(the_object, (types.ModuleType, types.FunctionType, TypeType, types.BuiltinFunctionType, types.BuiltinMethodType, types.MethodType, FileType)):
        return None
    return the_object


def pack_dictionary(the_dict):
    retval = codecs.encode(pickle.dumps(pickleable_objects(the_dict)), 'base64').decode()
    return retval


def decrypt_dictionary(dict_string, secret):
    dict_string = bytearray(dict_string, encoding='utf-8')
    decrypter = AES.new(bytearray(secret, encoding='utf-8'), AES.MODE_CBC, dict_string[:16])
    return fix_pickle_dict(unpad(decrypter.decrypt(codecs.decode(dict_string[16:], 'base64'))))


def unpack_dictionary(dict_string):
    dict_string = codecs.decode(bytearray(dict_string, encoding='utf-8'), 'base64')
    return fix_pickle_dict(dict_string)


def nice_date_from_utc(timestamp, timezone=tz.tzlocal()):
    return timestamp.replace(tzinfo=tz.tzutc()).astimezone(timezone).strftime('%x %X')


def nice_utc_date(timestamp):
    return timestamp.strftime('%F %T')


# @elapsed('fetch_user_dict')
def fetch_user_dict(user_code, filename, secret=None):
    # logmessage("fetch_user_dict: user_code is " + str(user_code) + " and filename is " + str(filename))
    user_dict = None
    steps = 1
    encrypted = True
    subq = select(db.func.max(UserDict.indexno).label('indexno'), db.func.count(UserDict.indexno).label('cnt')).where(and_(UserDict.key == user_code, UserDict.filename == filename)).subquery()
    stmt = select(UserDict.indexno, UserDict.dictionary, UserDict.encrypted, subq.c.cnt).join(subq, subq.c.indexno == UserDict.indexno)
    result = db.session.execute(stmt)
    for d in list(result):
        # logmessage("fetch_user_dict: indexno is " + str(d.indexno))
        if d.dictionary and isinstance(d.dictionary, str):
            if d.encrypted:
                # logmessage("fetch_user_dict: entry was encrypted")
                user_dict = decrypt_dictionary(d.dictionary, secret)
                # logmessage("fetch_user_dict: decrypted dictionary")
            else:
                # logmessage("fetch_user_dict: entry was not encrypted")
                user_dict = unpack_dictionary(d.dictionary)
                # logmessage("fetch_user_dict: unpacked dictionary")
                encrypted = False
        if d.cnt:
            steps = d.cnt
        break
    return steps, user_dict, encrypted


# @elapsed('user_dict_exists')
def user_dict_exists(user_code, filename):
    result = db.session.execute(select(UserDict).where(and_(UserDict.key == user_code, UserDict.filename == filename))).first()
    if result:
        return True
    return False


# @elapsed('fetch_previous_user_dict')
def fetch_previous_user_dict(user_code, filename, secret):
    max_indexno = db.session.execute(select(db.func.max(UserDict.indexno)).where(and_(UserDict.key == user_code, UserDict.filename == filename))).scalar()
    if max_indexno is not None:
        db.session.execute(delete(UserDict).where(UserDict.indexno == max_indexno))
        db.session.commit()
    return fetch_user_dict(user_code, filename, secret=secret)


def advance_progress(user_dict, interview):
    if user_dict['_internal']['progress'] is None:
        return
    if hasattr(interview, 'progress_bar_multiplier'):
        multiplier = interview.progress_bar_multiplier
    else:
        multiplier = 0.05
    if hasattr(interview, 'progress_bar_method') and interview.progress_bar_method == 'stepped':
        next_part = 100.0
        for value in sorted(interview.progress_points):
            if value > user_dict['_internal']['progress']:
                next_part = value
                break
        user_dict['_internal']['progress'] += multiplier*(next_part-user_dict['_internal']['progress'])
    else:
        user_dict['_internal']['progress'] += multiplier*(100-user_dict['_internal']['progress'])


def delete_temp_user_data(temp_user_id, r):
    db.session.execute(delete(UserDictKeys).where(UserDictKeys.temp_user_id == temp_user_id))
    db.session.commit()
    db.session.execute(delete(UploadsUserAuth).where(UploadsUserAuth.temp_user_id == temp_user_id))
    db.session.commit()
    db.session.execute(delete(ChatLog).where(ChatLog.temp_owner_id == temp_user_id))
    db.session.commit()
    db.session.execute(delete(ChatLog).where(ChatLog.temp_user_id == temp_user_id))
    db.session.commit()
    db.session.execute(delete(GlobalObjectStorage).where(GlobalObjectStorage.temp_user_id == temp_user_id))
    db.session.commit()
    db.session.execute(delete(GlobalObjectStorage).where(or_(GlobalObjectStorage.key.like('da:userid:t' + str(temp_user_id) + ':%'), GlobalObjectStorage.key.like('da:daglobal:userid:t' + str(temp_user_id) + ':%'))).execution_options(synchronize_session=False))
    db.session.commit()
    files_to_delete = []
    for short_code_item in db.session.execute(select(Shortener).filter_by(temp_user_id=temp_user_id)).scalars():
        for email in db.session.execute(select(Email).filter_by(short=short_code_item.short)).scalars():
            for attachment in db.session.execute(select(EmailAttachment).filter_by(email_id=email.id)).scalars():
                files_to_delete.append(attachment.upload)
    for file_number in files_to_delete:
        the_file = SavedFile(file_number)
        the_file.delete()
    db.session.execute(delete(Shortener).where(Shortener.temp_user_id == temp_user_id))
    db.session.commit()
    keys_to_delete = set()
    for key in r.keys('*userid:t' + str(temp_user_id)):
        keys_to_delete.add(key)
    for key in r.keys('*userid:t' + str(temp_user_id) + ':*'):
        keys_to_delete.add(key)
    for key in keys_to_delete:
        r.delete(key)


def delete_user_data(user_id, r, r_user):
    db.session.execute(delete(UserDict).where(UserDict.user_id == user_id))
    db.session.commit()
    db.session.execute(delete(UserDictKeys).where(UserDictKeys.user_id == user_id))
    db.session.commit()
    db.session.execute(delete(UploadsUserAuth).where(UploadsUserAuth.user_id == user_id))
    db.session.commit()
    db.session.execute(delete(ChatLog).where(ChatLog.owner_id == user_id))
    db.session.commit()
    db.session.execute(delete(ChatLog).where(ChatLog.user_id == user_id))
    db.session.commit()
    db.session.execute(delete(GlobalObjectStorage).where(GlobalObjectStorage.user_id == user_id))
    db.session.commit()
    db.session.execute(delete(GlobalObjectStorage).where(or_(GlobalObjectStorage.key.like('da:userid:' + str(user_id) + ':%'), GlobalObjectStorage.key.like('da:daglobal:userid:' + str(user_id) + ':%'))).execution_options(synchronize_session=False))
    db.session.commit()
    for package_auth in db.session.execute(select(PackageAuth).filter_by(user_id=user_id)).scalars():
        package_auth.user_id = 1
    db.session.commit()
    files_to_delete = []
    for short_code_item in db.session.execute(select(Shortener).filter_by(user_id=user_id)).scalars():
        for email in db.session.execute(select(Email).filter_by(short=short_code_item.short)).scalars():
            for attachment in db.session.execute(select(EmailAttachment).filter_by(email_id=email.id)).scalars():
                files_to_delete.append(attachment.upload)
    for file_number in files_to_delete:
        the_file = SavedFile(file_number)
        the_file.delete()
    db.session.execute(delete(Shortener).where(Shortener.user_id == user_id))
    db.session.commit()
    db.session.execute(delete(UserRoles).where(UserRoles.user_id == user_id))
    db.session.commit()
    for user_auth in db.session.execute(select(UserAuthModel).filter_by(user_id=user_id).with_for_update()).scalars():
        user_auth.password = ''
        user_auth.reset_password_token = ''
    db.session.commit()
    for section in ('playground', 'playgroundmodules', 'playgroundpackages', 'playgroundsources', 'playgroundstatic', 'playgroundtemplate'):
        the_section = SavedFile(user_id, section=section)
        the_section.delete()
    old_email = None
    for user_object in db.session.execute(select(UserModel).filter_by(id=user_id)).scalars():
        old_email = user_object.email
        user_object.active = False
        user_object.first_name = ''
        user_object.last_name = ''
        user_object.nickname = ''
        user_object.email = None
        user_object.country = ''
        user_object.subdivisionfirst = ''
        user_object.subdivisionsecond = ''
        user_object.subdivisionthird = ''
        user_object.organization = ''
        user_object.timezone = None
        user_object.language = None
        user_object.pypi_username = None
        user_object.pypi_password = None
        user_object.otp_secret = None
        user_object.confirmed_at = None
        user_object.last_login = None
        user_object.social_id = 'disabled$' + str(user_id)
    db.session.commit()
    keys_to_delete = set()
    for key in r.keys('*userid:' + str(user_id)):
        keys_to_delete.add(key)
    for key in r.keys('*userid:' + str(user_id) + ':*'):
        keys_to_delete.add(key)
    for key in keys_to_delete:
        r.delete(key)
    keys_to_delete = set()
    for key in r_user.keys('*:user:' + str(old_email)):
        keys_to_delete.add(key)
    for key in keys_to_delete:
        r_user.delete(key)


# @elapsed('reset_user_dict')
def reset_user_dict(user_code, filename, user_id=None, temp_user_id=None, force=False):
    # logmessage("reset_user_dict called with " + str(user_code) + " and " + str(filename) + " and " + str(user_id) + " and " + str(temp_user_id) + " and " + str(force))
    if force:
        the_user_id = None
    else:
        if user_id is None and temp_user_id is None:
            if current_user.is_authenticated:
                user_type = 'user'
                the_user_id = current_user.id
            else:
                user_type = 'tempuser'
                the_user_id = session.get('tempuser', None)
        elif user_id is not None:
            user_type = 'user'
            the_user_id = user_id
        else:
            user_type = 'tempuser'
            the_user_id = temp_user_id
    if the_user_id is None:
        db.session.execute(delete(UserDictKeys).filter_by(key=user_code, filename=filename))
        db.session.commit()
        do_delete = True
    else:
        if user_type == 'user':
            db.session.execute(delete(UserDictKeys).filter_by(key=user_code, filename=filename, user_id=the_user_id))
        else:
            db.session.execute(delete(UserDictKeys).filter_by(key=user_code, filename=filename, temp_user_id=the_user_id))
        db.session.commit()
        existing_user_dict_key = db.session.execute(select(UserDictKeys).filter_by(key=user_code, filename=filename)).scalar()
        do_delete = not bool(existing_user_dict_key)
    if not force:
        files_to_save = []
        for upload in db.session.execute(select(Uploads).filter_by(key=user_code, yamlfile=filename, persistent=True)).scalars():
            files_to_save.append(upload.indexno)
        if len(files_to_save) > 0:
            something_added = False
            if user_type == 'user':
                for uploads_indexno in files_to_save:
                    existing_auth = db.session.execute(select(UploadsUserAuth).filter_by(user_id=the_user_id, uploads_indexno=uploads_indexno)).scalar()
                    if not existing_auth:
                        new_auth_record = UploadsUserAuth(user_id=the_user_id, uploads_indexno=uploads_indexno)
                        db.session.add(new_auth_record)
                        something_added = True
            else:
                for uploads_indexno in files_to_save:
                    existing_auth = db.session.execute(select(UploadsUserAuth).filter_by(temp_user_id=the_user_id, uploads_indexno=uploads_indexno)).scalar()
                    if not existing_auth:
                        new_auth_record = UploadsUserAuth(temp_user_id=the_user_id, uploads_indexno=uploads_indexno)
                        db.session.add(new_auth_record)
                        something_added = True
            if something_added:
                db.session.commit()
    if do_delete:
        db.session.execute(delete(UserDict).filter_by(key=user_code, filename=filename))
        db.session.commit()
        files_to_delete = []
        for speaklist in db.session.execute(select(SpeakList).filter_by(key=user_code, filename=filename)).scalars():
            if speaklist.upload is not None:
                files_to_delete.append(speaklist.upload)
        db.session.execute(delete(SpeakList).filter_by(key=user_code, filename=filename))
        db.session.commit()
        for upload in db.session.execute(select(Uploads).filter_by(key=user_code, yamlfile=filename, persistent=False)).scalars():
            files_to_delete.append(upload.indexno)
        db.session.execute(delete(Uploads).filter_by(key=user_code, yamlfile=filename, persistent=False))
        db.session.commit()
        db.session.execute(delete(GlobalObjectStorage).where(or_(GlobalObjectStorage.key.like('da:uid:' + user_code + ':i:' + filename + ':%'), GlobalObjectStorage.key.like('da:daglobal:uid:' + user_code + ':i:' + filename + ':%'))).execution_options(synchronize_session=False))
        db.session.commit()
        db.session.execute(delete(ChatLog).filter_by(key=user_code, filename=filename))
        db.session.commit()
        for short_code_item in db.session.execute(select(Shortener).filter_by(uid=user_code, filename=filename)).scalars():
            for email in db.session.execute(select(Email).filter_by(short=short_code_item.short)).scalars():
                for attachment in db.session.execute(select(EmailAttachment).filter_by(email_id=email.id)).scalars():
                    files_to_delete.append(attachment.upload)
        db.session.execute(delete(Shortener).filter_by(uid=user_code, filename=filename))
        db.session.commit()
        # docassemble.base.functions.server.delete_answer_json(user_code, filename, delete_all=True)
        for file_number in files_to_delete:
            the_file = SavedFile(file_number)
            the_file.delete()


# @elapsed('get_person')
def get_person(user_id, cache):
    if user_id in cache:
        return cache[user_id]
    for record in db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(id=user_id)).unique().scalars():
        cache[record.id] = record
        return record
    return None


# @elapsed('get_chat_log')
def get_chat_log(chat_mode, yaml_filename, session_id, user_id, temp_user_id, secret, self_user_id, self_temp_id):
    messages = []
    people = {}
    if user_id is not None:
        if get_person(user_id, people) is None:
            return []
        chat_person_type = 'auth'
        chat_person_id = user_id
    else:
        chat_person_type = 'anon'
        chat_person_id = temp_user_id
    if self_user_id is not None:
        if get_person(self_user_id, people) is None:
            return []
        self_person_type = 'auth'
        self_person_id = self_user_id
    else:
        self_person_type = 'anon'
        self_person_id = self_temp_id
    if chat_person_type == 'auth':
        if chat_mode in ('peer', 'peerhelp'):
            records = db.session.execute(select(ChatLog).where(and_(ChatLog.filename == yaml_filename, ChatLog.key == session_id, or_(ChatLog.open_to_peer == True, ChatLog.owner_id == chat_person_id))).order_by(ChatLog.id)).scalars().all()  # noqa: E712 # pylint: disable=singleton-comparison
        else:
            records = db.session.execute(select(ChatLog).where(and_(ChatLog.filename == yaml_filename, ChatLog.key == session_id, ChatLog.owner_id == chat_person_id)).order_by(ChatLog.id)).scalars().all()
        for record in records:
            if record.encrypted:
                try:
                    message = decrypt_phrase(record.message, secret)
                except:
                    logmessage("Could not decrypt phrase with secret " + secret)
                    continue
            else:
                message = unpack_phrase(record.message)
            modtime = nice_utc_date(record.modtime)
            if self_person_type == 'auth':
                is_self = bool(self_person_id == record.user_id)
            else:
                is_self = bool(self_person_id == record.temp_user_id)
            if record.user_id is not None:
                person = get_person(record.user_id, people)
                if person is None:
                    logmessage("Person " + str(record.user_id) + " did not exist")
                    continue
                role_list = [role.name for role in person.roles]
                if len(role_list) == 0:
                    role_list = ['user']
                messages.append({'id': record.id, 'is_self': is_self, 'temp_owner_id': record.temp_owner_id, 'temp_user_id': record.temp_user_id, 'owner_id': record.owner_id, 'user_id': record.user_id, 'first_name': person.first_name, 'last_name': person.last_name, 'email': person.email, 'modtime': modtime, 'message': message, 'roles': role_list})
            else:
                messages.append({'id': record.id, 'is_self': is_self, 'temp_owner_id': record.temp_owner_id, 'temp_user_id': record.temp_user_id, 'owner_id': record.owner_id, 'user_id': record.user_id, 'modtime': modtime, 'message': message, 'roles': ['user']})
    else:
        if chat_mode in ['peer', 'peerhelp']:
            records = db.session.execute(select(ChatLog).where(and_(ChatLog.filename == yaml_filename, ChatLog.key == session_id, or_(ChatLog.open_to_peer == True, ChatLog.temp_owner_id == chat_person_id))).order_by(ChatLog.id)).scalars().all()  # noqa: E712 # pylint: disable=singleton-comparison
        else:
            records = db.session.execute(select(ChatLog).where(and_(ChatLog.filename == yaml_filename, ChatLog.key == session_id, ChatLog.temp_owner_id == chat_person_id)).order_by(ChatLog.id)).scalars().all()
        for record in records:
            if record.encrypted:
                try:
                    message = decrypt_phrase(record.message, secret)
                except:
                    logmessage("Could not decrypt phrase with secret " + secret)
                    continue
            else:
                message = unpack_phrase(record.message)
            modtime = nice_utc_date(record.modtime)
            if self_person_type == 'auth':
                is_self = bool(self_person_id == record.user_id)
            else:
                # logmessage("self person id is " + str(self_person_id) + " and record user id is " + str(record.temp_user_id))
                is_self = bool(self_person_id == record.temp_user_id)
            if record.user_id is not None:
                person = get_person(record.user_id, people)
                if person is None:
                    logmessage("Person " + str(record.user_id) + " did not exist")
                    continue
                role_list = [role.name for role in person.roles]
                if len(role_list) == 0:
                    role_list = ['user']
                messages.append({'id': record.id, 'is_self': is_self, 'temp_owner_id': record.temp_owner_id, 'temp_user_id': record.temp_user_id, 'owner_id': record.owner_id, 'user_id': record.user_id, 'first_name': person.first_name, 'last_name': person.last_name, 'email': person.email, 'modtime': modtime, 'message': message, 'roles': role_list})
            else:
                messages.append({'id': record.id, 'is_self': is_self, 'temp_owner_id': record.temp_owner_id, 'temp_user_id': record.temp_user_id, 'owner_id': record.owner_id, 'user_id': record.user_id, 'modtime': modtime, 'message': message, 'roles': ['user']})
    return messages


# @elapsed('file_set_attributes')
def file_set_attributes(file_number, **kwargs):
    upload = db.session.execute(select(Uploads).filter_by(indexno=file_number).with_for_update()).scalar()
    if upload is None:
        db.session.commit()
        raise DAException("file_set_attributes: file number " + str(file_number) + " not found.")
    if 'private' in kwargs and kwargs['private'] in [True, False] and upload.private != kwargs['private']:
        upload.private = kwargs['private']
    if 'persistent' in kwargs and kwargs['persistent'] in [True, False] and upload.persistent != kwargs['persistent']:
        upload.persistent = kwargs['persistent']
    if 'session' in kwargs and isinstance(kwargs['session'], str):
        upload.key = kwargs['session']
    if 'filename' in kwargs and isinstance(kwargs['filename'], str):
        upload.filename = kwargs['filename']
    db.session.commit()


def file_user_access(file_number, allow_user_id=None, allow_email=None, disallow_user_id=None, disallow_email=None, disallow_all=False):
    something_added = False
    if allow_user_id:
        for user_id in set(allow_user_id):
            existing_user = db.session.execute(select(UserModel).filter_by(id=user_id)).first()
            if not existing_user:
                logmessage("file_user_access: invalid user ID " + repr(user_id))
                continue
            if db.session.execute(select(UploadsUserAuth).filter_by(uploads_indexno=file_number, user_id=user_id)).first():
                continue
            new_auth_record = UploadsUserAuth(uploads_indexno=file_number, user_id=user_id)
            db.session.add(new_auth_record)
            something_added = True
    if something_added:
        db.session.commit()
    something_added = False
    if allow_email:
        for email in set(allow_email):
            existing_user = db.session.execute(select(UserModel).filter_by(email=email)).first()
            if not existing_user:
                logmessage("file_user_access: invalid email " + repr(email))
                continue
            if db.session.execute(select(UploadsUserAuth).filter_by(uploads_indexno=file_number, user_id=existing_user.id)).first():
                continue
            new_auth_record = UploadsUserAuth(uploads_indexno=file_number, user_id=existing_user.id)
            db.session.add(new_auth_record)
            something_added = True
    if something_added:
        db.session.commit()
    if disallow_user_id:
        for user_id in set(disallow_user_id):
            db.session.execute(delete(UploadsUserAuth).filter_by(uploads_indexno=file_number, user_id=user_id))
        db.session.commit()
    if disallow_email:
        for email in set(disallow_email):
            existing_user = db.session.execute(select(UserModel).filter_by(email=email)).scalar()
            if not existing_user:
                logmessage("file_user_access: invalid email " + repr(email))
                continue
            db.session.execute(delete(UploadsUserAuth).filter_by(uploads_indexno=file_number, user_id=existing_user.id))
        db.session.commit()
    if disallow_all:
        db.session.execute(delete(UploadsUserAuth).filter_by(uploads_indexno=file_number))
    if not (allow_user_id or allow_email or disallow_user_id or disallow_email or disallow_all):
        result = {'user_ids': [], 'emails': [], 'temp_user_ids': []}
        for auth in db.session.execute(select(UploadsUserAuth.user_id, UploadsUserAuth.temp_user_id, UserModel.email).outerjoin(UserModel, UploadsUserAuth.user_id == UserModel.id).where(UploadsUserAuth.uploads_indexno == file_number)).all():
            if auth.user_id is not None:
                result['user_ids'].append(auth.user_id)
            if auth.temp_user_id is not None:
                result['temp_user_ids'].append(auth.temp_user_id)
            if auth.email:
                result['emails'].append(auth.email)
        return result
    return None


def file_privilege_access(file_number, allow=None, disallow=None, disallow_all=False):
    something_added = False
    if allow:
        for privilege in set(allow):
            existing_role = db.session.execute(select(Role).filter_by(name=privilege)).scalar_one()
            if not existing_role:
                logmessage("file_privilege_access: invalid privilege " + repr(privilege))
                continue
            if db.session.execute(select(UploadsRoleAuth).filter_by(uploads_indexno=file_number, role_id=existing_role.id)).first():
                continue
            new_auth_record = UploadsRoleAuth(uploads_indexno=file_number, role_id=existing_role.id)
            db.session.add(new_auth_record)
            something_added = True
    if something_added:
        db.session.commit()
    if disallow:
        for privilege in set(disallow):
            existing_role = db.session.execute(select(Role).filter_by(name=privilege)).scalar_one()
            if not existing_role:
                logmessage("file_privilege_access: invalid privilege " + repr(privilege))
                continue
            db.session.execute(delete(UploadsRoleAuth).filter_by(uploads_indexno=file_number, role_id=existing_role.id))
        db.session.commit()
    if disallow_all:
        db.session.execute(delete(UploadsRoleAuth).filter_by(uploads_indexno=file_number))
    if not (allow or disallow or disallow_all):
        result = []
        for auth in db.session.execute(select(UploadsRoleAuth.id, Role.name).join(Role, UploadsRoleAuth.role_id == Role.id).where(UploadsRoleAuth.uploads_indexno == file_number)).all():
            result.append(auth.name)
        return result
    return None


def clear_session(i):
    if 'sessions' in session and i in session['sessions']:
        del session['sessions'][i]
    session.modified = True


def clear_specific_session(i, uid):
    if 'sessions' in session and i in session['sessions']:
        if session['sessions'][i]['uid'] == uid:
            del session['sessions'][i]
    session.modified = True


def guess_yaml_filename():
    yaml_filename = None
    if 'i' in session and 'uid' in session:  # TEMPORARY
        yaml_filename = session['i']
    if 'sessions' in session:
        for item in session['sessions']:
            yaml_filename = item
            break
    return yaml_filename


def delete_obsolete():
    for name in ('i', 'uid', 'key_logged', 'encrypted', 'chatstatus'):
        if name in session:
            del session[name]
    session.modified = True


def get_session(i):
    if 'sessions' not in session:
        session['sessions'] = {}
    if i in session['sessions']:
        return session['sessions'][i]
    if 'i' in session and 'uid' in session:  # TEMPORARY
        session['sessions'][session['i']] = {'uid': session['uid'], 'encrypted': session.get('encrypted', True), 'key_logged': session.get('key_logged', False), 'chatstatus': session.get('chatstatus', 'off')}
        if i == session['i']:
            delete_obsolete()
            return session['sessions'][i]
        delete_obsolete()
    return None


def unattached_uid():
    while True:
        newname = random_alphanumeric(32)
        existing_key = db.session.execute(select(UserDict).filter_by(key=newname)).first()
        if existing_key:
            continue
        return newname


def get_uid_for_filename(i):
    if 'sessions' not in session:
        session['sessions'] = {}
    if i not in session['sessions']:
        return None
    return session['sessions'][i]['uid']


def update_session(i, uid=None, encrypted=None, key_logged=None, chatstatus=None):
    if 'sessions' not in session:
        session['sessions'] = {}
    if i not in session['sessions'] or uid is not None:
        if uid is None:
            raise DAException("update_session: cannot create new session without a uid")
        if encrypted is None:
            encrypted = True
        if key_logged is None:
            key_logged = False
        if chatstatus is None:
            chatstatus = 'off'
        session['sessions'][i] = {'uid': uid, 'encrypted': encrypted, 'key_logged': key_logged, 'chatstatus': chatstatus}
    else:
        if uid is not None:
            session['sessions'][i]['uid'] = uid
        if encrypted is not None:
            session['sessions'][i]['encrypted'] = encrypted
        if key_logged is not None:
            session['sessions'][i]['key_logged'] = key_logged
        if chatstatus is not None:
            session['sessions'][i]['chatstatus'] = chatstatus
    session.modified = True
    return session['sessions'][i]


def get_session_uids():
    if 'i' in session:  # TEMPORARY
        get_session(session['i'])
    if 'sessions' in session:
        return [item['uid'] for item in session['sessions'].values()]
    return []

if DEBUG_BOOT:
    boot_log("backend: completed")
