# -*- coding: utf-8 -*-
#
# Copyright 2016 - 2018  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

from __future__ import absolute_import, division, print_function

import fcntl
import json
import os
import shutil
from itertools import count, groupby, product
from logging import getLogger
from uuid import uuid4

import sqlalchemy as sqla
from flask import current_app as app
from pkg_resources import resource_filename

from marv_node.run import run_nodes
from marv_store import Store
from . import utils
from .collection import esc
from .collection import Collections
from .config import Config
from .model import STATUS_MISSING, STATUS_OUTDATED
from .model import Comment, Dataset, File, Group, Tag, User, dataset_tag, db


DEFAULT_NODES = """
marv_nodes:dataset
marv_nodes:meta_table
marv_nodes:summary_keyval
"""

DEFAULT_FILTERS = """
name       | Name       | substring         | string   | (get "dataset.name")
setid      | Set Id     | startswith        | string   | (get "dataset.id")
size       | Size       | lt le eq ne ge gt | filesize | (sum (get "dataset.files[:].size"))
status     | Status     | any all           | subset   | (status )
tags       | Tags       | any all           | subset   | (tags )
comments   | Comments   | substring         | string   | (comments )
files      | File paths | substring_any     | string[] | (get "dataset.files[:].path")
time_added | Added      | lt le eq ne ge gt | datetime | (get "dataset.time_added")
"""

DEFAULT_LISTING_COLUMNS = """
name       | Name   | route    | (detail_route (get "dataset.id") (get "dataset.name"))
size       | Size   | filesize | (sum (get "dataset.files[:].size"))
status     | Status | icon[]   | (status )
tags       | Tags   | pill[]   | (tags )
time_added | Added  | datetime | (get "dataset.time_added")
"""

DEFAULT_LISTING_SUMMARY = """
datasets | datasets | int       | (len (rows ))
size     | size     | filesize  | (sum (rows "size" 0))
"""

DEFAULT_DETAIL_SUMMARY_WIDGETS = """
summary_keyval
meta_table
"""

log = getLogger(__name__)


class UnknownNode(Exception):
    pass


def make_config(siteconf):
    sitedir = os.path.dirname(siteconf)
    defaults = {
        'marv': {
            'acl': 'marv_webapi.acls:authenticated',
            'dburi': 'sqlite:///' + os.path.join(sitedir, 'db', 'db.sqlite'),
            'frontenddir': os.path.join(sitedir, 'frontend'),
            'oauth': '',
            'reverse_proxy': None,
            'sessionkey_file': os.path.join(sitedir, 'sessionkey'),
            'staticdir': os.path.join(resource_filename('marv_ludwig', 'static')),
            'storedir': os.path.join(sitedir, 'store'),
            'window_title': '',
        },
        'collection': {
            'compare': None,
            'detail_summary_widgets': DEFAULT_DETAIL_SUMMARY_WIDGETS,
            'detail_sections': '',
            'detail_title': '(get "dataset.name")',
            'filters': DEFAULT_FILTERS,
            'listing_columns': DEFAULT_LISTING_COLUMNS,
            'listing_sort': '| ascending',
            'listing_summary': DEFAULT_LISTING_SUMMARY,
            'nodes': DEFAULT_NODES,
        }
    }
    required = {
        'marv': [
            'collections',
        ],
        'collection': [
            'scanner',
            'scanroots',
        ],
    }
    schema = {
        'marv': {
            'acl': 'find_obj',
            'collections': 'space_separated_list',
            'oauth': 'lines',
            'staticdir': 'path',
            'storedir': 'path',
        },
        'collection': {
            'compare': 'find_obj',
            'detail_title': 'function',
            'detail_summary_widgets': 'lines',
            'detail_sections': 'lines',
            'filters': 'lines',
            'listing_columns': 'lines',
            'listing_sort': 'pipe_separated_list',
            'listing_summary': 'lines',
            'nodes': 'lines',
            'scanroots': 'path_lines',
        },
    }
    return Config.from_file(siteconf, defaults=defaults,
                            required=required, schema=schema)


class Site(object):
    def __init__(self, siteconf):
        self.config = make_config(siteconf)
        # TODO: maybe ordereddict for meta, or generate multiple keys
        self.config.marv.oauth = dict(
            (i.split(' ')[0], [s.strip() for s in i.split('|')]
        ) for i in self.config.marv.oauth)
        self.collections = Collections(config=self.config)

    def load_for_web(self):
        _ = [getattr(x, y) and None for x, y, in product(
            self.collections.values(),
            ('compare', 'filter_specs', 'listing_columns', 'model',
             'sortcolumn', 'sortorder', 'summary_items'))]

    def init(self):
        try:
            fd = os.open(self.config.marv.sessionkey_file,
                         os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
        except OSError as e:
            if e.errno != 17:
                raise
        else:
            with os.fdopen(fd, 'w') as f:
                f.write(str(uuid4()))
            log.verbose('Generated %s', self.config.marv.sessionkey_file)

        try:
            os.mkdir(self.config.marv.storedir)
            log.verbose('Created %s', self.config.marv.storedir)
        except OSError as e:
            if e.errno != 17:
                raise

        try:
            dbdir = os.path.dirname(self.config.marv.dburi.replace('sqlite:///', ''))
            os.mkdir(dbdir)
            log.verbose('Created %s', dbdir)
        except OSError as e:
            if e.errno != 17:
                raise

        # TODO: maybe this should be an explicit method...
        # load models
        _ = [x.model for x in self.collections.values()]

        prefixes = ['listing_{}'.format(col) for col in self.collections.keys()]
        result = db.session.execute('SELECT name FROM sqlite_master WHERE type="table";')
        tables = {name for name in [x[0] for x in result]
                  if any(name.startswith(prefix) for prefix in prefixes)}
        for table in [x for x in tables if x.endswith('_sec')]:
            db.session.execute('DROP TABLE {};'.format(table))
            tables.discard(table)
        for table in [x for x in tables if x not in prefixes]:
            db.session.execute('DROP TABLE {};'.format(table))
            tables.discard(table)
        for table in tables:
            db.session.execute('DROP TABLE {};'.format(table))

        db.create_all()
        for name in ('admin', ):
            if not Group.query.filter(Group.name==name).count():
                db.session.add(Group(name=name))
                db.session.commit()

        try:
            with open('users', 'r') as usersfile:
                users = json.load(usersfile)
                for name, password in users.items():
                    if not User.query.filter(User.name==name).count():
                        db.session.add(User(name=name, password=password))
                        db.session.commit()
            os.rename('users', 'users.imported')
        except IOError:
            pass

        log.verbose('Initialized database %s', app.config['SQLALCHEMY_DATABASE_URI'])
        for col in self.collections.keys():
            collection = self.collections[col]
            loop = count()
            batchsize = 50
            # TODO: increase verbosity and probably introduce --reinit
            while True:
                batch = db.session.query(Dataset)\
                                  .filter(Dataset.collection == col)\
                                  .options(db.joinedload(Dataset.files))\
                                  .limit(batchsize)\
                                  .offset(batchsize*loop.next())\
                                  .all()
                for dataset in batch:
                    collection.render_detail(dataset)
                if batch:
                    collection.update_listings(batch)
                if len(batch) < batchsize:
                    break
        log.info('Initialized from %s', self.config.filename)

    def cleanup_discarded(self):
        collections = self.collections
        query = db.session.query(Dataset.collection, Dataset.id)\
                          .filter(Dataset.discarded.is_(True))\
                          .order_by(Dataset.collection)
        datasets = [(col, [x[1] for x in ids])
                    for col, ids in groupby(query, lambda x: x[0])]
        if not datasets:
            return

        for col, ids in datasets:
            collection = collections[col]
            model = collection.model
            listing = model.Listing.__table__
            for secondary in model.secondaries.values():
                stmt = secondary.delete()\
                                .where(secondary.c.listing_id.in_(ids))
                db.session.execute(stmt)

            stmt = listing.delete()\
                          .where(listing.c.id.in_(ids))
            db.session.execute(stmt)

            stmt = dataset_tag.delete()\
                              .where(dataset_tag.c.dataset_id.in_(ids))
            db.session.execute(stmt)

            db.session.query(Comment)\
                      .filter(Comment.dataset_id.in_(ids))\
                      .delete(synchronize_session=False)

            db.session.query(File)\
                      .filter(File.dataset_id.in_(ids))\
                      .delete(synchronize_session=False)

            db.session.query(Dataset)\
                      .filter(Dataset.id.in_(ids))\
                      .delete(synchronize_session=False)

        db.session.commit()
        # TODO: Cleanup corresponding store paths

    def cleanup_relations(self):
        """Cleanup listing relations"""
        collections = self.collections
        for relation in [x for col in collections.values()
                         for x in col.model.relations.values()]:
            db.session.query(relation)\
                       .filter(~relation.listing.any())\
                       .delete(synchronize_session=False)
        db.session.commit()

    def cleanup_tags(self):
        db.session.query(Tag)\
                  .filter(~Tag.datasets.any())\
                  .delete(synchronize_session=False)
        db.session.commit()

    def restore_database(self, datasets=None, users=None):
        for user in users or []:
            user.setdefault('realm', 'marv')
            user.setdefault('realmuid', '')
            groups = user.pop('groups', [])
            app.um.user_add(_restore=True, **user)
            for grp in groups:
                try:
                    app.um.group_adduser(grp, user['name'])
                except ValueError:
                    app.um.group_add(grp)
                    app.um.group_adduser(grp, user['name'])

        for key, sets in (datasets or {}).items():
            key = self.collections.keys()[0] if key == 'DEFAULT_COLLECTION' else key
            self.collections[key].restore_datasets(sets)

    def listtags(self, collections=None):
        query = db.session.query(Tag.value)
        if collections:
            query = query.filter(Tag.collection.in_(collections))
        query = query.group_by(Tag.value)\
                     .order_by(Tag.value)
        tags = [x[0] for x in query]
        return tags

    def query(self, collections=None, discarded=None, outdated=None,
              path=None, tags=None, abbrev=None, missing=None):
        abbrev = 10 if abbrev is True else abbrev
        discarded = bool(discarded)
        query = db.session.query(Dataset.setid)

        if collections:
            query = query.filter(Dataset.collection.in_(collections))

        if outdated:
            query = query.filter(Dataset.status.op('&')(STATUS_OUTDATED) == STATUS_OUTDATED)

        query = query.filter(Dataset.discarded.is_(discarded))

        if path:
            relquery = db.session.query(File.dataset_id)\
                                 .filter(File.path.like('{}%'.format(esc(path)), escape='$'))\
                                 .group_by(File.dataset_id)
            query = query.filter(Dataset.id.in_(relquery.subquery()))

        if missing:
            relquery = db.session.query(File.dataset_id)\
                                 .filter(File.missing.is_(True))\
                                 .group_by(File.dataset_id)
            query = query.filter(Dataset.id.in_(relquery.subquery()))

        if tags:
            relquery = db.session.query(dataset_tag.c.dataset_id)\
                                 .join(Tag)\
                                 .filter(Tag.value.in_(tags))
            query = query.filter(Dataset.id.in_(relquery.subquery()))

        setids = [x[0][:abbrev] if abbrev else x[0]
                  for x in query.order_by(Dataset.setid)]
        return setids

    def run(self, setid, selected_nodes=None, deps=None, force=None, keep=None,
            force_dependent=None, update_detail=None, update_listing=None,
            excluded_nodes=None, cachesize=None):
        assert not force_dependent or selected_nodes

        excluded_nodes = set(excluded_nodes or [])
        dataset = Dataset.query.filter(Dataset.setid == str(setid))\
                               .options(db.joinedload(Dataset.files))\
                               .one()
        collection = self.collections[dataset.collection]
        selected_nodes = set(selected_nodes or [])
        if not (selected_nodes or update_listing or update_detail):
            selected_nodes.update(collection.listing_deps)
            selected_nodes.update(collection.detail_deps)
        persistent = collection.nodes
        try:
            nodes = {persistent[name] if not ':' in name else utils.find_obj(name)
                     for name in selected_nodes
                     if name not in excluded_nodes
                     if name != 'dataset'}
        except KeyError as e:
            raise UnknownNode(dataset.collection, e.args[0])

        if force_dependent:
            nodes.update(x for name in selected_nodes
                         for x in persistent[name].dependent)
        nodes = sorted(nodes)

        storedir = app.site.config.marv.storedir
        store = Store(storedir, persistent)

        changed = False
        try:
            if nodes:
                changed = run_nodes(dataset, nodes, store, force=force,
                                    persistent=persistent,
                                    deps=deps, cachesize=cachesize)
        except:
            raise
        else:
            if changed or update_detail:
                collection.render_detail(dataset)
                log.verbose('%s detail rendered', setid)
            if changed or update_listing:
                collection.update_listings([dataset])
                log.verbose('%s listing rendered', setid)
        finally:
            if not keep:
                for tmpdir, tmpdir_fd in store.pending.values():
                    store.logdebug('Cleaning up %r', tmpdir)
                    shutil.rmtree(tmpdir)
                    fcntl.flock(tmpdir_fd, fcntl.LOCK_UN)
                    os.close(tmpdir_fd)
                store.pending.clear()

        return changed

    def scan(self, dry_run=None):
        for collection in self.collections.values():
            for scanroot in collection.scanroots:
                collection.scan(scanroot, dry_run)

    def comment(self, username, message, ids):
        now = int(utils.now() * 1000)
        comments = [Comment(dataset_id=id, author=username, time_added=now,
                            text=message)
                    for id in ids]
        db.session.add_all(comments)
        db.session.commit()

    def tag(self, setids, add=None, remove=None):
        assert setids
        assert add or remove, (add, remove)
        add = sorted(add or [])
        remove = remove or []
        addremove = set(add) | set(remove)
        assert len(addremove) == len(add) + len(remove), (add, remove)

        query = db.session.query(Dataset.collection, Dataset.id, Dataset.setid)\
                          .filter(Dataset.setid.in_(setids))\
                          .order_by(Dataset.collection)
        for collection, group in groupby(query, key=lambda x: x[0]):
            setidmap = {setid: id for _, id, setid in group}
            dataset_ids = setidmap.values()

            if add:
                stmt = Tag.__table__.insert().prefix_with('OR IGNORE')
                db.session.execute(stmt, [{'collection': collection,
                                           'value': x} for x in add])

            tags = {value: id for id, value in (
                db.session.query(Tag.id, Tag.value)
                .filter(Tag.collection == collection)
                .filter(Tag.value.in_(addremove)))}

            if add:
                stmt = dataset_tag.insert().prefix_with('OR IGNORE')
                db.session.execute(stmt,
                                   [{'dataset_id': x, 'tag_id': y} for x, y in
                                    product(dataset_ids, (tags[x] for x in add))])

            if remove:
                where = (dataset_tag.c.tag_id.in_(tags[x] for x in remove) &
                         dataset_tag.c.dataset_id.in_(dataset_ids))
                stmt = dataset_tag.delete().where(where)
                db.session.execute(stmt)

        db.session.commit()


def dump_database(dburi):
    """Dump database.

    The structure of the database is reflected and therefore also
    older databases, not matching the current version of marv, can be
    dumped.
    """

    engine = sqla.create_engine(dburi)
    meta = sqla.MetaData(engine)
    meta.reflect()
    con = engine.connect()
    tables = {k: v for k, v in meta.tables.items() if not k.startswith('listing_')}
    select = sqla.sql.select

    def rows2dcts(stmt):
        result = con.execute(stmt)
        keys = result.keys()
        return [{k: v for k, v in zip(keys, row)} for row in result]

    comments = {}
    comment_t = tables.pop('comment')
    stmt = select([comment_t]).order_by(comment_t.c.dataset_id, comment_t.c.id)
    for did, grp in groupby(rows2dcts(stmt), key=lambda x: x['dataset_id']):
        assert did not in comments
        comments[did] = lst = []
        for comment in grp:
            # Everything except these fields is included in the dump
            del comment['dataset_id']
            del comment['id']
            lst.append(comment)

    files = {}
    file_t = tables.pop('file')
    stmt = select([file_t]).order_by(file_t.c.dataset_id, file_t.c.idx)
    for did, grp in groupby(rows2dcts(stmt), key=lambda x: x['dataset_id']):
        assert did not in files
        files[did] = lst = []
        for file in grp:
            # Everything except these fields is included in the dump
            del file['dataset_id']
            del file['idx']
            lst.append(file)

    tags = {}
    dataset_tag_t = tables.pop('dataset_tag')
    tag_t = tables.pop('tag')
    stmt = select([dataset_tag_t, tag_t]).where(dataset_tag_t.c.tag_id == tag_t.c.id)\
                                         .order_by(dataset_tag_t.c.dataset_id, tag_t.c.value)
    for did, grp in groupby(rows2dcts(stmt), key=lambda x: x['dataset_id']):
        assert did not in tags
        tags[did] = lst = []
        for tag in grp:
            del tag['dataset_id']
            del tag['collection']
            del tag['id']
            del tag['tag_id']
            lst.append(tag.pop('value'))
            assert not tag

    dump = {}
    dump['datasets'] = collections = {}
    dataset_t = tables.pop('dataset')
    stmt = select([dataset_t]).order_by(dataset_t.c.setid)
    for dataset in rows2dcts(stmt):
        did = dataset.pop('id')  # Everything except this is included in the dump
        dataset['comments'] = comments.pop(did, [])
        dataset['files'] = files.pop(did)
        dataset['tags'] = tags.pop(did, [])
        collections.setdefault(dataset.pop('collection'), []).append(dataset)

    assert not comments, comments
    assert not files, files
    assert not tags, tags

    groups = {}
    group_t = tables.pop('group')
    user_group_t = tables.pop('user_group')
    stmt = select([user_group_t, group_t]).where(user_group_t.c.group_id == group_t.c.id)\
                                          .order_by(user_group_t.c.user_id, group_t.c.name)
    for uid, grp in groupby(rows2dcts(stmt), key=lambda x: x['user_id']):
        assert uid not in groups
        groups[uid] = lst = []
        for group in grp:
            del group['user_id']
            del group['group_id']
            del group['id']
            name = group.pop('name')
            assert not group
            lst.append(name)

    dump['users'] = users = []
    user_t = tables.pop('user')
    stmt = select([user_t]).order_by(user_t.c.name)
    for user in rows2dcts(select([user_t])):
        user_id = user.pop('id')  # Everything except this is included in the dump
        user['groups'] = groups.pop(user_id, [])
        users.append(user)

    assert not groups, groups
    assert not tables, tables.keys()

    return dump
