# coding: utf-8

import json
import logging
import os
import time
import UEVaultManager.tkgui.modules.globals as gui_g  # using the shortest variable name for globals for convenience
from base64 import b64decode
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta
from hashlib import sha1
from locale import getlocale, LC_CTYPE
from platform import system
from threading import current_thread, enumerate as thread_enumerate
from urllib.parse import urlparse

from requests import session, __version__
from requests.exceptions import HTTPError, ConnectionError

from UEVaultManager.api.egs import EPCAPI, GrabResult
from UEVaultManager.api.lgd import LGDAPI
from UEVaultManager.lfs.egl import EPCLFS
from UEVaultManager.lfs.lgndry import LGDLFS
from UEVaultManager.models.app import *
from UEVaultManager.models.exceptions import *
from UEVaultManager.models.json_manifest import JSONManifest
from UEVaultManager.models.manifest import Manifest
from UEVaultManager.utils.cli import check_and_create_path
from UEVaultManager.utils.egl_crypt import decrypt_epic_data
from UEVaultManager.utils.env import is_windows_mac_or_pyi

# The heading dict contains the title of each column and a boolean value to know if its contents must be preserved if it already exists in the output file (To Avoid overwriting data changed by the user in the file)
CSV_headings = {
    'Asset_id': False,  # ! important: Do not Rename => this field is used as main key for each asset
    'App name': False,
    'App title': False,
    'Category': False,
    'Image': False,
    'Url': True,  # could be kept if a better url that can be used to download the asset is found
    'UE Version': False,
    'Compatible Versions': False,
    'Review': False,
    'Developer': False,
    'Description': False,
    'Uid': False,
    'Creation Date': False,
    'Update Date': False,
    'Status': False,
    # Modified Fields when added into the file (mainly from extras data)
    'Date Added': True,
    'Price': False,  # ! important: Rename Wisely => this field is searched by text in the next lines
    'Old Price': False,  # ! important: always place it after the Price field in the list
    'On Sale': False,  # ! important: always place it after the Old Price field in the list
    'Purchased': False,
    'Obsolete': True,
    # Extracted from page, can be compared with value in metadata. Coud be used to if check data grabbing if OK
    'Supported Versions': False,
    'Page title': False,
    'Grab result': False,
    # User Fields
    'Comment': True,
    'Stars': True,
    'Asset Folder': True,
    'Must Buy': True,
    'Test result': True,
    'Installed Folder': True,
    'Alternative': True
}

# ToDo: instead of true/false return values for success/failure actually raise an exception that the CLI/GUI
#  can handle to give the user more details. (Not required yet since there's no GUI so log output is fine)


class AppCore:
    """
    AppCore handles most of the lower level interaction with
    the downloader, lfs, and api components to make writing CLI/GUI
    code easier and cleaner and avoid duplication.
    """
    _egl_version = '11.0.1-14907503+++Portal+Release-Live'

    def __init__(self, override_config=None, timeout=10.0):
        self.log = logging.getLogger('Core')
        self.egs = EPCAPI(timeout=timeout)
        self.lgd = LGDLFS(config_file=override_config)
        self.egl = EPCLFS()
        self.lgd_api = LGDAPI()

        # on non-Windows load the programdata path from config
        if os.name != 'nt':
            self.egl.programdata_path = self.lgd.config.get('UEVaultManager', 'egl_programdata', fallback=None)
            if self.egl.programdata_path and not os.path.exists(self.egl.programdata_path):
                self.log.error(f'Config EGL path ("{self.egl.programdata_path}") is invalid! Disabling sync...')
                self.egl.programdata_path = None
                self.lgd.config.remove_option('UEVaultManager', 'egl_programdata')
                self.lgd.save_config()

        self.local_timezone = datetime.now().astimezone().tzinfo
        self.language_code, self.country_code = ('en', 'US')

        if locale := self.lgd.config.get('UEVaultManager', 'locale', fallback=getlocale(LC_CTYPE)[0]):
            try:
                self.language_code, self.country_code = locale.split('-' if '-' in locale else '_')
                self.log.debug(f'Set locale to {self.language_code}-{self.country_code}')
                # adjust egs api language as well
                self.egs.language_code, self.egs.country_code = self.language_code, self.country_code
            except Exception as error:
                self.log.warning(f'Getting locale failed: {error!r}, falling back to using en-US.')
        elif system() != 'Darwin':  # macOS doesn't have a default locale we can query
            self.log.warning('Could not determine locale, falling back to en-US')

        self.update_available = False
        self.force_show_update = False
        self.webview_killswitch = False
        self.logged_in = False

        self.default_datetime_format = '%y-%m-%d %H:%M:%S'
        # UE assets metadata cache properties
        self.ue_assets_count = 0
        self.cache_is_invalidate = False
        # Delay (in seconds) when UE assets metadata cache will be invalidated. Default value is 15 days
        self.ue_assets_max_cache_duration = 15 * 24 * 3600
        # set to True to add print more information during long operations
        self.verbose_mode = False
        # Create a backup of the output file (when using the --output option) suffixed by a timestamp before creating a new file
        self.create_output_backup = True
        # Set the file name (and path) for logging when an asset is ignored or filtered when running the --list command
        self.ignored_assets_filename_log = ''
        # Set the file name (and path) for logging when an asset is not found on the marketplace when running the --list command
        self.notfound_assets_filename_log = ''
        # Set the file name (and path) for logging when an asset has metadata and extras data are incoherent when running the --list command
        self.bad_data_assets_filename_log = ''
        # Create a backup of the log files that store asset analysis suffixed by a timestamp before creating a new file
        self.create_log_backup = True
        # new file loggers
        self.ignored_logger = None
        self.notfound_logger = None
        self.bad_data_logger = None
        # store time to process metadata and extras update
        self.process_time_average = {'time': 0.0, 'count': 0}
        self.use_threads = False
        self.engine_version_for_obsolete_assets = '4.26'

    def setup_assets_logging(self):
        formatter = logging.Formatter('%(message)s')
        message = f"-----\n{datetime.now().strftime(self.default_datetime_format)} Log Started\n-----\n"

        if self.ignored_assets_filename_log != '':
            ignored_assets_filename_log = self.ignored_assets_filename_log.replace('~/.config', self.lgd.path)
            if check_and_create_path(ignored_assets_filename_log):
                ignored_assets_handler = logging.FileHandler(ignored_assets_filename_log, mode='w')
                ignored_assets_handler.setFormatter(formatter)
                self.ignored_logger = logging.Logger('IgnoredAssets', 'INFO')
                self.ignored_logger.addHandler(ignored_assets_handler)
                self.ignored_logger.info(message)

        if self.notfound_assets_filename_log != '':
            notfound_assets_filename_log = self.notfound_assets_filename_log.replace('~/.config', self.lgd.path)
            if check_and_create_path(notfound_assets_filename_log):
                notfound_assets_handler = logging.FileHandler(notfound_assets_filename_log, mode='w')
                notfound_assets_handler.setFormatter(formatter)
                self.notfound_logger = logging.Logger('NotFoundAssets', 'INFO')
                self.notfound_logger.addHandler(notfound_assets_handler)
                self.notfound_logger.info(message)

        if self.bad_data_assets_filename_log != '':
            bad_data_assets_filename_log = self.bad_data_assets_filename_log.replace('~/.config', self.lgd.path)
            if check_and_create_path(bad_data_assets_filename_log):
                bad_data_assets_handler = logging.FileHandler(bad_data_assets_filename_log, mode='w')
                bad_data_assets_handler.setFormatter(formatter)
                self.bad_data_logger = logging.Logger('BadDataAssets', 'INFO')
                self.bad_data_logger.addHandler(bad_data_assets_handler)
                self.bad_data_logger.info(message)

    def auth_sid(self, sid) -> str:
        """
        Handles getting an exchange code from an id
        :param sid: session id
        :return: exchange code
        """
        s = session()
        s.headers.update(
            {
                'X-Epic-Event-Action':
                'login',
                'X-Epic-Event-Category':
                'login',
                'X-Epic-Strategy-Flags':
                '',
                'X-Requested-With':
                'XMLHttpRequest',
                'User-Agent':
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                'AppleWebKit/537.36 (KHTML, like Gecko) '
                f'EpicGamesLauncher/{self._egl_version} '
                'UnrealEngine/4.23.0-14907503+++Portal+Release-Live '
                'Chrome/84.0.4147.38 Safari/537.36'
            }
        )
        s.cookies['EPIC_COUNTRY'] = self.country_code.upper()

        # get first set of cookies (EPIC_BEARER_TOKEN etc.)
        _ = s.get('https://www.epicgames.com/id/api/set-sid', params=dict(sid=sid))
        # get XSRF-TOKEN and EPIC_SESSION_AP cookie
        _ = s.get('https://www.epicgames.com/id/api/csrf')
        # finally, get the exchange code
        r = s.post('https://www.epicgames.com/id/api/exchange/generate', headers={'X-XSRF-TOKEN': s.cookies['XSRF-TOKEN']})

        if r.status_code == 200:
            return r.json()['code']

        self.log.error(f'Getting exchange code failed: {r.json()}')
        return ''

    def auth_code(self, code) -> bool:
        """
        Handles authentication via authorization code (either retrieved manually or automatically)
        """
        try:
            self.lgd.userdata = self.egs.start_session(authorization_code=code)
            return True
        except Exception as error:
            self.log.error(f'Logging in failed with {error!r}, please try again.')
            return False

    def auth_ex_token(self, code) -> bool:
        """
        Handles authentication via exchange token (either retrieved manually or automatically)
        """
        try:
            self.lgd.userdata = self.egs.start_session(exchange_token=code)
            return True
        except Exception as error:
            self.log.error(f'Logging in failed with {error!r}, please try again.')
            return False

    def auth_import(self) -> bool:
        """Import refresh token from EGL installation and use it for logging in"""
        self.egl.read_config()
        remember_me_data = self.egl.config.get('RememberMe', 'Data')
        raw_data = b64decode(remember_me_data)
        # data is encrypted
        if raw_data[0] != '{':
            for data_key in self.egl.data_keys:
                try:
                    decrypted_data = decrypt_epic_data(data_key, raw_data)
                    re_data = json.loads(decrypted_data)[0]
                    break
                except Exception as error:
                    self.log.debug(f'Decryption with key {data_key} failed with {error!r}')
            else:
                raise ValueError('Decryption of EPIC launcher user information failed.')
        else:
            re_data = json.loads(raw_data)[0]

        if 'Token' not in re_data:
            raise ValueError('No login session in config')
        refresh_token = re_data['Token']
        try:
            self.lgd.userdata = self.egs.start_session(refresh_token=refresh_token)
            return True
        except Exception as error:
            self.log.error(f'Logging in failed with {error!r}, please try again.')
            return False

    def login(self, force_refresh=False) -> bool:
        """
        Attempts logging in with existing credentials.

        raises ValueError if no existing credentials or InvalidCredentialsError if the API return an error
        """
        if not self.lgd.userdata:
            raise ValueError('No saved credentials')
        elif self.logged_in and self.lgd.userdata['expires_at']:
            dt_exp = datetime.fromisoformat(self.lgd.userdata['expires_at'][:-1])
            dt_now = datetime.utcnow()
            td = dt_now - dt_exp

            # if session still has at least 10 minutes left we can re-use it.
            if dt_exp > dt_now and abs(td.total_seconds()) > 600:
                return True
            else:
                self.logged_in = False

        # run update check
        if self.update_check_enabled():
            try:
                self.check_for_updates()
            except Exception as error:
                self.log.warning(f'Checking for UEVaultManager updates failed: {error!r}')
        else:
            self.apply_lgd_config()

        if self.lgd.userdata['expires_at'] and not force_refresh:
            dt_exp = datetime.fromisoformat(self.lgd.userdata['expires_at'][:-1])
            dt_now = datetime.utcnow()
            td = dt_now - dt_exp

            # if session still has at least 10 minutes left we can re-use it.
            if dt_exp > dt_now and abs(td.total_seconds()) > 600:
                self.log.info('Trying to re-use existing login session...')
                try:
                    self.egs.resume_session(self.lgd.userdata)
                    self.logged_in = True
                    return True
                except InvalidCredentialsError as error:
                    self.log.warning(f'Resuming failed due to invalid credentials: {error!r}')
                except Exception as error:
                    self.log.warning(f'Resuming failed for unknown reason: {error!r}')
                # If verify fails just continue the normal authentication process
                self.log.info('Falling back to using refresh token...')

        try:
            self.log.info('Logging in...')
            userdata = self.egs.start_session(self.lgd.userdata['refresh_token'])
        except InvalidCredentialsError:
            self.log.error('Stored credentials are no longer valid! Please login again.')
            self.lgd.invalidate_userdata()
            return False
        except (HTTPError, ConnectionError) as error:
            self.log.error(f'HTTP request for login failed: {error!r}, please try again later.')
            return False

        self.lgd.userdata = userdata
        self.logged_in = True
        return True

    def update_check_enabled(self):
        return not self.lgd.config.getboolean('UEVaultManager', 'disable_update_check', fallback=False)

    def update_notice_enabled(self):
        if self.force_show_update:
            return True
        return not self.lgd.config.getboolean('UEVaultManager', 'disable_update_notice', fallback=not is_windows_mac_or_pyi())

    def check_for_updates(self, force=False):

        def version_tuple(v):
            return tuple(map(int, (v.split('.'))))

        cached = self.lgd.get_cached_version()
        version_info = cached['data']
        if force or not version_info or (datetime.now().timestamp() - cached['last_update']) > 24 * 3600:
            version_info = self.lgd_api.get_version_information()
            self.lgd.set_cached_version(version_info)

        web_version = version_info['release_info']['version']

        self.update_available = version_tuple(web_version) > version_tuple(__version__)
        # version check is disabled because it's checking for Legendary
        # TODO: check UEVaultManager updates instead
        self.update_available = False

        self.apply_lgd_config(version_info)

    def apply_lgd_config(self, version_info=None):
        """Applies configuration options returned by update API"""
        if not version_info:
            version_info = self.lgd.get_cached_version()['data']
        # if cached data is invalid
        if not version_info:
            self.log.debug('No cached UEVaultManager config to apply.')
            return

        if 'egl_config' in version_info:
            self.egs.update_egs_params(version_info['egl_config'])
            self._egl_version = version_info['egl_config'].get('version', self._egl_version)
            for data_key in version_info['egl_config'].get('data_keys', []):
                if data_key not in self.egl.data_keys:
                    self.egl.data_keys.append(data_key)

        if lgd_config := version_info.get('legendary_config'):
            self.webview_killswitch = lgd_config.get('webview_killswitch', False)

    def get_update_info(self):
        return self.lgd.get_cached_version()['data'].get('release_info')

    def update_aliases(self, force=False):
        _aliases_enabled = not self.lgd.config.getboolean('UEVaultManager', 'disable_auto_aliasing', fallback=False)
        if _aliases_enabled and (force or not self.lgd.aliases):
            self.lgd.generate_aliases()

    def get_assets(self, update_assets=False, platform='Windows') -> List[AppAsset]:
        # do not save and always fetch list when platform is overridden
        if not self.lgd.assets or update_assets or platform not in self.lgd.assets:
            # if not logged in, return empty list
            if not self.egs.user:
                return []

            assets = self.lgd.assets.copy() if self.lgd.assets else dict()

            assets.update({platform: [AppAsset.from_egs_json(a) for a in self.egs.get_item_assets(platform=platform)]})

            # only save (and write to disk) if there were changes
            if self.lgd.assets != assets:
                self.lgd.assets = assets

        return self.lgd.assets[platform]

    def get_asset(self, app_name, platform='Windows', update=False) -> AppAsset:
        if update or platform not in self.lgd.assets:
            self.get_assets(update_assets=True, platform=platform)

        try:
            return next(i for i in self.lgd.assets[platform] if i.app_name == app_name)
        except StopIteration:
            raise ValueError

    def asset_available(self, item: App, platform='Windows') -> bool:
        # Just say yes for Origin titles
        if item.third_party_store:
            return True

        try:
            asset = self.get_asset(item.app_name, platform=platform)
            return asset is not None
        except ValueError:
            return False

    def get_item(self, app_name, update_meta=False, platform='Windows') -> App:
        if update_meta:
            self.get_asset_list(True, platform=platform)
        return self.lgd.get_item_meta(app_name)

    def get_asset_list(self, update_assets=True, platform='Windows', filter_category='', force_refresh=False) -> (List[App], Dict[str, List[App]]):

        def fetch_asset_meta(name: str) -> bool:
            if (name in currently_fetching or not fetch_list.get(name)) and ('Asset_Fetcher' in thread_enumerate()):
                return False

            thread_data = ''
            if self.use_threads:
                thread = current_thread()
                thread_data = f' ==> By Thread name={thread.name}'

            self.log.debug(f'--- START fetching data {name}{thread_data}')

            currently_fetching[name] = True
            start_time = datetime.now()
            name, namespace, catalog_item_id, _process_meta, _process_extras = fetch_list[name]

            if _process_meta:
                eg_meta = self.egs.get_item_info(namespace, catalog_item_id, timeout=10.0)
                app = App(app_name=name, app_title=eg_meta['title'], metadata=eg_meta, asset_infos=assets[name])
                self.lgd.set_item_meta(app.app_name, app)
                apps[name] = app

            if _process_extras:
                # we use title because it's less ambiguous than a name when searching an asset
                eg_extras = self.egs.get_assets_extras(asset_name=name, asset_title=apps[name].app_title, verbose_mode=self.verbose_mode)
                self.lgd.set_item_extras(app_name=name, extras=eg_extras, update_global_dict=True)

                # log the asset if the title in metadata and the title in the marketplace grabbed page are not identical
                if eg_extras['page_title'] != '' and eg_extras['page_title'] != apps[name].app_title:
                    self.log.warning(f'{name} has incoherent data. It has been added to the bad_data_logger file')
                    eg_extras['grab_result'] = GrabResult.INCONSISTANT_DATA.name
                    if self.bad_data_logger:
                        self.bad_data_logger.info(name)

            # compute process time and average in s
            end_time = datetime.now()
            process_time = (end_time - start_time).total_seconds()
            self.process_time_average['time'] += process_time
            self.process_time_average['count'] += 1

            if fetch_list.get(name):
                del fetch_list[name]
                if self.verbose_mode:
                    self.log.info(f'Removed {name} from the metadata update')

            if currently_fetching.get(name):
                del currently_fetching[name]

            if not self.use_threads:
                process_average = self.process_time_average['time'] / self.process_time_average['count']
                self.log.info(f'===Time Average={process_average:.3f} s # ({(len(fetch_list) * process_average):.3f} s time left)')

            self.log.info(
                f'--- END fetching data in {name}{thread_data}. Time For Processing={process_time:.3f}s # Still {len(fetch_list)} assets to process'
            )
            return True

        _ret = []
        meta_updated = False

        # fetch asset information for Windows, all installed platforms, and the specified one
        platforms = {'Windows'}
        platforms |= {platform}
        if gui_g.progress_window_ref is not None:
            gui_g.progress_window_ref.reset(new_value=0, new_text="Fetching platforms...", new_max_value=len(platforms))
        for _platform in platforms:
            self.get_assets(update_assets=update_assets, platform=_platform)
            if gui_g.progress_window_ref is not None and not gui_g.progress_window_ref.update_and_continue(increment=1):
                return []

        if not self.lgd.assets:
            return _ret

        assets = {}
        if gui_g.progress_window_ref is not None:
            gui_g.progress_window_ref.reset(new_value=0, new_text="Fetching assets...", new_max_value=len(self.lgd.assets.items()))
        for _platform, _assets in self.lgd.assets.items():
            for _asset in _assets:
                if gui_g.progress_window_ref is not None and not gui_g.progress_window_ref.update_and_continue(increment=1):
                    return []
                if _asset.app_name in assets:
                    assets[_asset.app_name][_platform] = _asset
                else:
                    assets[_asset.app_name] = {_platform: _asset}

        fetch_list = {}
        assets_bypassed = {}
        apps = {}

        # loop through assets items to check for if they are for ue or not
        valid_items = []
        bypass_count = 0
        self.log.info(f'======\nSTARTING phase 1: asset indexing (ue or not)\n')
        if gui_g.progress_window_ref is not None:
            gui_g.progress_window_ref.reset(new_value=0, new_text="Indexing assets...", new_max_value=len(assets.items()))
        # note: we sort by reverse, as it the most recent version of an asset will be listed first
        for app_name, app_assets in sorted(assets.items(), reverse=True):
            if gui_g.progress_window_ref is not None and not gui_g.progress_window_ref.update_and_continue(increment=1):
                return []
            # notes:
            #   asset_id is not unique because somme assets can have the same asset_id but with several UE versions
            #   app_name is unique because it includes the unreal version
            # asset_id = app_assets['Windows'].asset_id
            assets_bypassed[app_name] = False
            if app_assets['Windows'].namespace != 'ue':
                self.log.debug(f'{app_name} has been bypassed (namespace != "ue") in phase 1')
                bypass_count += 1
                assets_bypassed[app_name] = True
                continue

            item = {'name': app_name, 'asset': app_assets}
            valid_items.append(item)

        self.ue_assets_count = len(valid_items)

        self.log.info(f'A total of {bypass_count} on {len(valid_items)} assets have been bypassed in phase 1')

        # check if we must refresh ue asset metadata cache
        self.check_for_ue_assets_updates(self.ue_assets_count, force_refresh)
        force_refresh = self.cache_is_invalidate
        if force_refresh:
            self.log.info(f'!! Assets metadata will be updated !!\n')
        else:
            self.log.info(f"Asset metadata won't be updated\n")

        self.log.info(f'======\nSTARTING phase 2:asset filtering and metadata updating\n')
        if gui_g.progress_window_ref is not None:
            gui_g.progress_window_ref.reset(new_value=0, new_text="Updating metadata...", new_max_value=len(valid_items))
        # loop through valid items to check for update and filtering
        bypass_count = 0
        filtered_items = []
        currently_fetching = {}
        i = 0
        while i < len(valid_items):
            if gui_g.progress_window_ref is not None and not gui_g.progress_window_ref.update_and_continue(increment=1):
                return []
            item = valid_items[i]
            app_name = item['name']
            app_assets = item['asset']
            if self.verbose_mode:
                self.log.info(f'Checking {app_name}....')

            item_metadata = self.lgd.get_item_meta(app_name)
            asset_updated = False

            if not item_metadata:
                self.log.info(f'Metadata for {app_name} are missing. It Will be ADDED to the FETCH list')
            else:
                category = str(item_metadata.metadata['categories'][0]['path']).lower()
                if filter_category and filter_category.lower() not in category:
                    self.log.info(
                        f'{app_name} has been FILTERED by category ("{filter_category}" text not found in "{category}").It has been added to the ignored_logger file'
                    )
                    if self.ignored_logger:
                        self.ignored_logger.info(app_name)
                    assets_bypassed[app_name] = True
                    bypass_count += 1
                    i += 1
                    continue
                asset_updated = any(item_metadata.app_version(_p) != app_assets[_p].build_version for _p in app_assets.keys())
                apps[app_name] = item_metadata
                self.log.debug(f'{app_name} has been ADDED to the apps list with asset_updated={asset_updated}')

            # get extras data only in not filtered
            if force_refresh or asset_updated:
                process_extras = True
            else:
                # will read the extras data from file if necessary and put in the global dict
                process_extras = self.lgd.get_item_extras(app_name) is None

            process_meta = not item_metadata or force_refresh or asset_updated

            if update_assets and (process_extras or process_meta):
                self.log.debug(f'Scheduling metadata and extras update for {app_name}')
                # namespace/catalog item are the same for all platforms, so we can just use the first one
                _ga = next(iter(app_assets.values()))
                fetch_list[app_name] = (app_name, _ga.namespace, _ga.catalog_item_id, process_meta, process_extras)
                meta_updated = True
            i += 1
            filtered_items.append(item)
            # end while i < len(valid_items):

        # setup and teardown of thread pool takes some time, so only do it when it makes sense.
        self.use_threads = len(fetch_list) > 5
        # self.use_threads = False  # test only
        if fetch_list:
            if gui_g.progress_window_ref is not None:
                gui_g.progress_window_ref.reset(new_value=0, new_text="Fetching missing metadata...\nIt Could take a while !", new_max_value=0)
                gui_g.progress_window_ref.hide_progress_bar()
                gui_g.progress_window_ref.hide_stop_button()

            self.log.info(f'Fetching metadata for {len(fetch_list)} app(s).')
            if self.use_threads:
                # note:  unreal engine API limits the number of connection to 16. So no more than 16 threads !
                with ThreadPoolExecutor(max_workers=min(16, os.cpu_count() + 2), thread_name_prefix="Asset_Fetcher") as executor:
                    executor.map(fetch_asset_meta, fetch_list.keys(), timeout=30.0)

        self.log.info(f'A total of {bypass_count} on {len(valid_items)} assets have been bypassed in phase 2')
        self.log.info(f'======\nSTARTING phase 3: emptying the List of assets to be fetched \n')
        if gui_g.progress_window_ref is not None:
            gui_g.progress_window_ref.show_progress_bar()  # show progress bar, must be before reset
            gui_g.progress_window_ref.show_stop_button()
            gui_g.progress_window_ref.reset(new_value=0, new_text="Checking assets data...", new_max_value=len(filtered_items))
        # loop through valid and filtered items
        meta_updated = (bypass_count == 0) and meta_updated  # to avoid deleting metadata files or assets that have been filtered
        while len(filtered_items) > 0:
            if gui_g.progress_window_ref is not None and not gui_g.progress_window_ref.update_and_continue(increment=1):
                return []
            item = filtered_items.pop()
            app_name = item['name']
            app_assets = item['asset']
            if self.verbose_mode:
                self.log.info(f'Checking {app_name}. Still {len(filtered_items)} assets to check')
            try:
                app_item = apps.get(app_name)
            except (KeyError, IndexError):
                self.log.debug(f'{app_name} has not been found int the app list. Bypassing')
                # item not found in app, ignore and pass to next one
                continue

            # retry if the asset is still in fetch list (with active fetcher treads)
            if fetch_list.get(app_name) and (not currently_fetching.get(app_name) or 'Asset_Fetcher' not in thread_enumerate()):
                self.log.info(f'Fetching metadata for {app_name} is still no done, retrying')
                if currently_fetching.get(app_name):
                    del currently_fetching[app_name]
                fetch_asset_meta(app_name)

            try:
                is_bypassed = (app_name in assets_bypassed) and (assets_bypassed[app_name])
                is_a_mod = any(i['path'] == 'mods' for i in app_item.metadata.get('categories', []))
            except (KeyError, IndexError, AttributeError):
                self.log.debug(f'{app_name} has no metadata. Adding to the fetch list (again)')
                try:
                    fetch_list[app_name] = (app_name, item.namespace, item.catalog_item_id, True, True)
                    _ret.append(app_item)
                except (KeyError, IndexError, AttributeError):
                    self.log.debug(f'{app_name} has an invalid format. Could not been added to the fetch list')
                continue

            has_valid_platform = platform in app_assets
            is_still_fetching = (app_name in fetch_list) or (app_name in currently_fetching)

            if is_still_fetching:
                # put again the asset in the list waiting when it will be fetched
                filtered_items.append(item)
                time.sleep(3)  # Sleep for 3 seconds to let the fetch process progress or end

            # check if the asset will be added to the final list
            if not is_bypassed and not is_still_fetching and not is_a_mod and has_valid_platform:
                _ret.append(app_item)

        self.log.info(f'A total of {len(_ret)} assets have been analysed and kept in phase 3')

        self.update_aliases(force=meta_updated)

        if meta_updated:
            # delete old files
            self._prune_metadata()
            self._save_metadata(_ret)
        #  meta_updated = True  # test only
        if meta_updated:
            # save new ones
            self._prune_extras_data(update_global_dict=False)
            self._save_extras_data(self.lgd.assets_extras_data, update_global_dict=False)
        return _ret

    # end def get_asset_list(self, update_assets=True, platform='Windows', filter_category='') -> (List[App], Dict[str, List[App]]):

    def _prune_metadata(self):
        # compile list of assets without assets, then delete their metadata
        available_assets = set()
        available_assets |= {i.app_name for i in self.get_assets(platform='Windows')}

        for app_name in self.lgd.get_item_app_names():
            self.log.debug(f'Removing old/unused metadata for "{app_name}"')
            self.lgd.delete_item_meta(app_name)

    def _prune_extras_data(self, update_global_dict: True):
        # compile list of assets without assets, then delete their extras data
        available_assets = set()
        available_assets |= {i.app_name for i in self.get_assets(platform='Windows')}

        for app_name in self.lgd.get_item_app_names():
            self.log.debug(f'Removing old/unused extras data for "{app_name}"')
            self.lgd.delete_item_extras(app_name, update_global_dict=update_global_dict)

    def _save_metadata(self, assets):
        for app in assets:
            self.lgd.set_item_meta(app.app_name, app)

    def _save_extras_data(self, extras: dict, update_global_dict: True):
        for app_name, eg_extras in extras.items():
            self.lgd.set_item_extras(app_name=app_name, extras=eg_extras, update_global_dict=update_global_dict)

    def get_non_asset_library_items(self, force_refresh=False, skip_ue=True) -> (List[App], Dict[str, List[App]]):
        """
        Gets a list of Items without assets for installation, for instance Items delivered via
        third-party stores that do not have assets for installation

        :param force_refresh: Force a metadata refresh
        :param skip_ue: Ignore Unreal Marketplace entries
        :return: List of Items that do not have assets
        """
        _ret = []
        # get all the app names we have to ignore
        ignore = set(i.app_name for i in self.get_assets())

        for lib_item in self.egs.get_library_items():
            if lib_item['namespace'] == 'ue' and skip_ue:
                continue
            if lib_item['appName'] in ignore:
                continue

            item = self.lgd.get_item_meta(lib_item['appName'])
            if not item or force_refresh:
                eg_meta = self.egs.get_item_info(lib_item['namespace'], lib_item['catalogItemId'])
                item = App(app_name=lib_item['appName'], app_title=eg_meta['title'], metadata=eg_meta)
                self.lgd.set_item_meta(item.app_name, item)

            if not any(i['path'] == 'mods' for i in item.metadata.get('categories', [])):
                _ret.append(item)

        # Force refresh to make sure these titles are included in aliasing
        self.update_aliases(force=True)
        return _ret

    def get_app_environment(self, app_name) -> dict:
        # get environment overrides from config
        env = dict()
        if 'default.env' in self.lgd.config:
            env |= {k: v for k, v in self.lgd.config['default.env'].items() if v and not k.startswith(';')}
        if f'{app_name}.env' in self.lgd.config:
            env |= {k: v for k, v in self.lgd.config[f'{app_name}.env'].items() if v and not k.startswith(';')}

        return env

    @staticmethod
    def load_manifest(data: bytes) -> Manifest:
        if data[0:1] == b'{':
            return JSONManifest.read_all(data)
        else:
            return Manifest.read_all(data)

    def get_cdn_urls(self, item, platform='Windows'):
        m_api_r = self.egs.get_item_manifest(item.namespace, item.catalog_item_id, item.app_name, platform)

        # never seen this outside the launcher itself, but if it happens: PANIC!
        if len(m_api_r['elements']) > 1:
            raise ValueError('Manifest response has more than one element!')

        manifest_hash = m_api_r['elements'][0]['hash']
        base_urls = []
        manifest_urls = []
        for manifest in m_api_r['elements'][0]['manifests']:
            base_url = manifest['uri'].rpartition('/')[0]
            if base_url not in base_urls:
                base_urls.append(base_url)

            if 'queryParams' in manifest:
                params = '&'.join(f'{p["name"]}={p["value"]}' for p in manifest['queryParams'])
                manifest_urls.append(f'{manifest["uri"]}?{params}')
            else:
                manifest_urls.append(manifest['uri'])

        return manifest_urls, base_urls, manifest_hash

    def get_cdn_manifest(self, item, platform='Windows', disable_https=False):
        manifest_urls, base_urls, manifest_hash = self.get_cdn_urls(item, platform)
        if not manifest_urls:
            raise ValueError('No manifest URLs returned by API')

        if disable_https:
            manifest_urls = [url.replace('https://', 'http://') for url in manifest_urls]

        r = {}
        for url in manifest_urls:
            self.log.debug(f'Trying to download manifest from "{url}"...')
            try:
                r = self.egs.unauth_session.get(url, timeout=10.0)
            except Exception as error:
                self.log.warning(f'Unable to download manifest from "{urlparse(url).netloc}" '
                                 f'(Exception: {error!r}), trying next URL...')
                continue

            if r.status_code == 200:
                manifest_bytes = r.content
                break
            else:
                self.log.warning(f'Unable to download manifest from "{urlparse(url).netloc}" '
                                 f'(status: {r.status_code}), trying next URL...')
        else:
            raise ValueError(f'Unable to get manifest from any CDN URL, last result: {r.status_code} ({r.reason})')

        if sha1(manifest_bytes).hexdigest() != manifest_hash:
            raise ValueError('Manifest sha hash mismatch!')

        return manifest_bytes, base_urls

    def get_uri_manifest(self, uri):
        if uri.startswith('http'):
            r = self.egs.unauth_session.get(uri)
            r.raise_for_status()
            new_manifest_data = r.content
            base_urls = [r.url.rpartition('/')[0]]
        else:
            base_urls = []
            with open(uri, 'rb') as f:
                new_manifest_data = f.read()

        return new_manifest_data, base_urls

    def get_delta_manifest(self, base_url, old_build_id, new_build_id):
        """Get optimized delta manifest (doesn't seem to exist for most games)"""
        if old_build_id == new_build_id:
            return None

        r = self.egs.unauth_session.get(f'{base_url}/Deltas/{new_build_id}/{old_build_id}.delta')
        return r.content if r.status_code == 200 else None

    # Check if the UE assets metadata cache must be updated
    def check_for_ue_assets_updates(self, assets_count: int, force_refresh=False):
        self.cache_is_invalidate = False
        cached = self.lgd.get_ue_assets_cache_data()
        cached_assets_count = cached['ue_assets_count']

        date_now = datetime.now().timestamp()
        date_diff = date_now - cached['last_update']

        if not cached_assets_count or cached_assets_count != assets_count:
            self.log.info(f'New assets are available. {assets_count} available VS {cached_assets_count} in cache')
            self.lgd.set_ue_assets_cache_data(last_update=cached['last_update'], ue_assets_count=assets_count)

        if force_refresh or date_diff > self.ue_assets_max_cache_duration:
            self.cache_is_invalidate = True
            self.lgd.set_ue_assets_cache_data(last_update=date_now, ue_assets_count=assets_count)
            if not force_refresh:
                self.log.info(f'Data cache is outdated. Cache age is {date_diff:.1f} s OR {str(timedelta(seconds=date_diff))}')
        else:
            self.log.info(f'Data cache is still valid. Cache age is {str(timedelta(seconds=date_diff))}')

    def clean_exit(self, code=0):
        """ Do cleanup, config saving, and exit. """
        self.lgd.save_config()
        logging.shutdown()
        exit(code)
