# -*- coding: utf-8 -*-

# In this module is the consolidated data for reports about
# Jaobi usage.

import os
import re
from collections import defaultdict, OrderedDict
import datetime
import time
from mongomotor.document import Document, MapReduceDocument
from mongomotor.fields import StringField, DateTimeField
from pyrocumulus.conf import settings
from pyrocumulus.utils import fqualname
from tornado import gen
from jaobi import models
from jaobi.utils import log
from jaobi.scheduler import scheduler

HERE = os.path.abspath(os.path.dirname(__file__))


class ReportMixin:

    """Base mixin for reports"""

    base_collection = models.ContentConsumption

    @classmethod
    def _format_kw(cls, **kwargs):
        kw = {}
        for k, v in kwargs.items():
            if not k.startswith('id__'):
                k = 'id__{}'.format(k)
            kw[k] = v
        return kw

    @classmethod
    def _format_result(cls, result):

        results = result['result']
        rdict = {}
        for r in results:
            rdict[r['_id']] = r['total']
        return rdict

    @classmethod
    def _format_history_result(cls, key, result):
        rdict = defaultdict(list)
        for r in result['result']:
            rdict[r['_id'][key]].append({'date': r['_id']['date'],
                                         'total': r['total']})

        for k, v in rdict.items():
            rdict[k] = sorted(v, key=lambda d: d['date'], reverse=True)

        return rdict

    @classmethod
    def get_sort_pipeline(cls, sort):
        """Returns a pipeline (list) for sorting aggregation.
        If not ``sort`` returns an empty list.

        :param sort: Sort criteria."""

        pipeline = []
        if sort:
            pipeline.append({'$sort': sort})

        return pipeline

    @classmethod
    def get_skip_pipeline(cls, skip):
        """Returns a pipeline (list) for skipping aggregation.
        If not ``skip`` returns an empty list.

        :param skip: skip quantity."""

        pipeline = []
        if skip:
            pipeline.append({'$skip': skip})

        return pipeline

    @classmethod
    def get_limit_pipeline(cls, limit):
        """Returns a pipeline (list) for limiting aggregation.
        If not ``limit`` returns an empty list.

        :param limit: limit quantity."""

        pipeline = []
        if limit:
            pipeline.append({'$limit': limit})

        return pipeline


class AggregatedReportMixin(ReportMixin):

    """Mixin for reports that use aggregation"""

    aggregation_fields = OrderedDict()

    @classmethod
    def get_dated_project(cls):
        """Returns a list representing the two first steps of a
        pipeline for grouping this by day.
        """

        pipeline = [
            {'$project': dict(date='$inclusion_date',
                              h={'$hour': '$inclusion_date'},
                              m={'$minute': '$inclusion_date'},
                              s={'$second': '$inclusion_date'},
                              ms={'$millisecond': '$inclusion_date'},
                              **cls.aggregation_fields)},
            {'$project': dict(date={'$subtract':
                                    ['$date',
                                     {'$add': ['$ms',
                                               {'$multiply': ['$s', 1000]},
                                               {'$multiply': ['$m', 60, 1000]},
                                               {'$multiply': ['$h',
                                                              3600, 1000]}]}]},
                              **cls.aggregation_fields)}]
        return pipeline

    @classmethod
    def get_out_step(cls):
        """Returns the step for sending the result of an aggregation
        to a collection. It must be the last step in a pipeline."""

        return {'$out': cls._get_collection_name()}

    @classmethod
    def get_pipeline(cls):
        """Returns the pipeline for aggregation. Must be implemented in
        subclasses."""
        raise NotImplementedError

    @classmethod
    @gen.coroutine
    def do_aggregate(cls, **kwargs):
        """Aggregates ``cls.base_collection`` to count consumers by
        field. It does an 'incremental' aggregation by sending the
        aggregation result to a temp collection and map reduce it to
        the final collection (read the code!)

        :param kwargs: Filter to queryset.
        """

        qs = cls.base_collection.objects.filter(**kwargs)
        pipeline = cls.get_pipeline()
        # The thing here is that the $out for aggregation in mongo
        # always overwrite the output collection, so I save it to a
        # temp collection and after that do a map reduce to the final
        # collection.
        out = pipeline[-1]['$out']
        tmp_out = '{}_{}'.format(out, str(time.time())[-7:].strip('.'))
        pipeline[-1]['$out'] = tmp_out
        r = yield qs.aggregate(*pipeline, allowDiskUse=True)
        # Here I have an instance of MotorCollection for the
        # temp collection so I can map reduce it to the final collection
        # and then drop it.
        coll = cls.base_collection._collection
        db = coll.database
        tmp_coll = coll.__class__(db, tmp_out)

        mapf = """function(){emit(this._id, this.value);}"""
        reducef = """function(key, values){return values;}"""
        opts = {'out': {'merge': out}}
        yield tmp_coll.map_reduce(mapf, reducef, **opts)
        yield tmp_coll.drop()

        return r


class ConsumptionReportMixin(AggregatedReportMixin):

    """Mixin for reports related to consumption."""

    @classmethod
    def get_pipeline(cls):
        """Returns the pipeline for consumption aggregation."""

        pipeline = super().get_dated_project()

        d = cls.aggregation_fields.copy()
        d.update({'date': '$date'})
        group = {'$group': {'_id': d, 'value': {'$sum': 1}}}
        pipeline.append(group)
        pipeline.append(cls.get_out_step())
        return pipeline


class ConsumerReportMixin(AggregatedReportMixin):

    """Mixin for reports related to consumers."""

    @classmethod
    def get_pipeline(cls):
        """Returns the pipeline for consumers aggretation."""

        d = cls.aggregation_fields.copy()
        cls.aggregation_fields.update({'consumer': '$consumer'})
        pipeline = super().get_dated_project()

        d.update({'date': '$date'})
        # Here we group unique consumers by key/day
        group = {'$group': {'_id': d, 'consumers': {'$addToSet': '$consumer'}}}
        pipeline.append(group)
        # Here we have the count of consumers
        project = {'$project': {'_id': '$_id',
                                'value': {'$size': '$consumers'}}}
        pipeline.append(project)
        # And here we direct the output to a collection
        pipeline.append(cls.get_out_step())
        return pipeline


class ThemeReportMixin(ReportMixin):

    """A Mixin for reports related to themes"""

    @classmethod
    def get_pipeline(cls):
        """Returns a pipeline with themes unwinded."""

        pipeline = super().get_pipeline()
        # unwind themes before aggregation so we count
        # consumers by theme.
        pipeline.insert(2, {'$unwind': '$themes'})

        return pipeline

    @classmethod
    @gen.coroutine
    def get_themes_report(cls, sort=None, skip=None, limit=None, **kwargs):
        """Returns the consolidated data for themes reports.

        :param sort: Order for aggretation results.
        :param skip: Offset for aggregation results.
        :param limit: Limit for aggregation results.
        :param kwargs: Arguments to filter the queryset before
          aggregating it."""

        kw = super()._format_kw(**kwargs)

        group = {"$group":
                 {'_id': '$_id.themes',
                  'total': {'$sum': '$value'}}}
        pipeline = [group]
        pipeline += cls.get_sort_pipeline(sort)
        pipeline += cls.get_skip_pipeline(skip)
        pipeline += cls.get_limit_pipeline(limit)
        r = yield cls.objects.filter(**kw).aggregate(*pipeline)

        return super()._format_result(r)

    @classmethod
    @gen.coroutine
    def get_themes_report_history(cls, sort=None, skip=None, limit=None,
                                  **kwargs):
        """Returns the history data for themes reports.

        :param sort: Order for aggretation results.
        :param skip: Offset for aggregation results.
        :param limit: Limit for aggregation results.
        :param kwargs: Arguments to filter the queryset before
          aggregating it."""

        kw = super()._format_kw(**kwargs)

        group = {'$group':
                 {'_id': {'theme': '$_id.themes',
                          'date': '$_id.date'},
                  'total': {'$sum': '$value'}}}

        pipeline = [group]
        pipeline += cls.get_sort_pipeline(sort)
        pipeline += cls.get_skip_pipeline(skip)
        pipeline += cls.get_limit_pipeline(limit)
        r = yield cls.objects.filter(**kw).aggregate(*pipeline)

        return super()._format_history_result('theme', r)


class SiteReportMixin(ReportMixin):

    """A Mixin for reports related to sites."""

    @classmethod
    @gen.coroutine
    def get_sites_report(cls, sort=None, skip=None, limit=None, **kwargs):
        """Returns the consolidated data for sites reports.

        :param sort: Order for aggretation results.
        :param skip: Offset for aggregation results.
        :param limit: Limit for aggregation results.
        :param kwargs: Arguments to filter the queryset before
          aggregating it."""

        kw = super()._format_kw(**kwargs)

        group = {"$group":
                 {'_id': '$_id.origin',
                  'total': {'$sum': '$value'}}}

        pipeline = [group]
        pipeline += cls.get_sort_pipeline(sort)
        pipeline += cls.get_skip_pipeline(skip)
        pipeline += cls.get_limit_pipeline(limit)
        r = yield cls.objects.filter(**kw).aggregate(*pipeline)

        return super()._format_result(r)

    @classmethod
    @gen.coroutine
    def get_sites_report_history(cls, sort=None, skip=None, limit=None,
                                 **kwargs):
        """Returns the history data for sites reports.

        :param sort: Order for aggretation results.
        :param skip: Offset for aggregation results.
        :param limit: Limit for aggregation results.
        :param kwargs: Arguments to filter the queryset before
          aggregating it."""

        kw = super()._format_kw(**kwargs)

        group = {'$group':
                 {'_id': {'origin': '$_id.origin',
                          'date': '$_id.date'},
                  'total': {'$sum': '$value'}}}

        pipeline = [group]
        pipeline += cls.get_sort_pipeline(sort)
        pipeline += cls.get_skip_pipeline(skip)
        pipeline += cls.get_limit_pipeline(limit)
        r = yield cls.objects.filter(**kw).aggregate(*pipeline)

        return super()._format_history_result('origin', r)


class ReferrerReportMixin(ReportMixin):

    """Mixin for reports about referer."""

    @classmethod
    @gen.coroutine
    def get_referrer_report(cls, sort=None, skip=None, limit=None, **kwargs):
        """Returns the consolidated data for referrer reports.

        :param sort: Order for aggretation results.
        :param skip: Offset for aggregation results.
        :param limit: Limit for aggregation results.
        :param kwargs: Arguments to filter the queryset before
          aggregating it."""

        kw = super()._format_kw(**kwargs)

        group = {"$group":
                 {'_id': '$_id.referrer',
                  'total': {'$sum': '$value'}}}
        pipeline = [group]
        pipeline += cls.get_sort_pipeline(sort)
        pipeline += cls.get_skip_pipeline(skip)
        pipeline += cls.get_limit_pipeline(limit)
        r = yield cls.objects.filter(**kw).aggregate(*pipeline)

        return super()._format_result(r)


class ContentReportMixin(ReportMixin):

    """ Mixin for reports related to content."""

    @classmethod
    @gen.coroutine
    def _format_content_result(cls, result):
        rdict = OrderedDict()
        ids = [r['_id'] for r in result['result']]
        contents = yield models.Content.objects.filter(id__in=ids).to_list()

        def get_result(rid):
            for c in result['result']:
                if c['_id'] == rid:
                    return c['total']

        def get_content(rid):
            for c in contents:
                if rid == c.id:
                    return c

        for i, rid in enumerate(ids):
            rdict[get_content(rid)] = get_result(rid)

        return rdict

    @classmethod
    @gen.coroutine
    def get_content_report(cls, sort=None, skip=None, limit=None, **kwargs):
        """Returns the consolidated data for content reports.

        :param sort: Order for aggretation results.
        :param skip: Offset for aggregation results.
        :param limit: Limit for aggregation results.
        :param kwargs: Arguments to filter the queryset before
          aggregating it."""

        kw = super()._format_kw(**kwargs)

        group = {'$group':
                 {'_id': '$_id.content',
                  'total': {'$sum': '$value'}}}
        pipeline = [group]
        pipeline += cls.get_sort_pipeline(sort)
        pipeline += cls.get_skip_pipeline(skip)
        pipeline += cls.get_limit_pipeline(limit)

        r = yield cls.objects.filter(**kw).aggregate(*pipeline)
        formated = yield cls._format_content_result(r)
        return formated


class ThemeConsumption(MapReduceDocument, ThemeReportMixin,
                       ConsumptionReportMixin):

    """ This Document is the result of a map-reduce done in the
    content_consumption collection. It reduces the consumption to
    a consolidated consumption by theme/site/day.

    """

    aggregation_fields = OrderedDict({'origin': '$origin',
                                      'themes': '$themes'})


class SiteConsumption(MapReduceDocument, SiteReportMixin,
                      ConsumptionReportMixin):

    """ This Document is the result of an aggregation done in the
    content_consumption collection. It reduces the consumption to
    a consolidated consumption by site/day.
    """

    aggregation_fields = OrderedDict({'origin': '$origin'})


class ThemeConsumers(MapReduceDocument, ThemeReportMixin, ConsumerReportMixin):

    """ This Document is the result of an aggregation done in the
    content_consumption collection. It reduces the consumption to
    consolidated consumers by theme/day.
    """

    aggregation_fields = OrderedDict({'themes': '$themes'})


class SiteConsumers(MapReduceDocument, SiteReportMixin, ConsumerReportMixin):

    """ This Document is the result of an aggregation done in the
    content_consumption collection. It reduces the consumption to
    consolidated consumers by site/day.
    """

    aggregation_fields = OrderedDict({'origin': '$origin'})


class ContentConsumptionReport(MapReduceDocument, ContentReportMixin,
                               ConsumptionReportMixin):

    """ This Document is the result of an aggregation done in the
    content_consumption collection. It reduces the consumption to
    consolidated consumption by content/day."""

    aggregation_fields = OrderedDict({'content': '$content'})


class ContentConsumersReport(MapReduceDocument, ContentReportMixin,
                             ConsumerReportMixin):

    """ This Document is the result of an aggretation done in the
    content_consumption collection. It reduces the consumption to
    consolidated consumers by content/day."""

    aggregation_fields = OrderedDict({'content': '$content'})


class ReferrerConsumption(MapReduceDocument, ReferrerReportMixin,
                          ConsumptionReportMixin):

    """ This Document is the result of an aggretation done in the
    content_consumption collection. It reduces the consumption to
    consolidated consumers by referrer/day."""

    aggregation_fields = OrderedDict({'referrer': '$referrer'})


class IncrementalMapReduce(Document):

    """This class takes care of incrementally execute the map
    reduce (or aggretation) for reports."""

    collection = StringField(unique=True, required=True)
    last_incr = DateTimeField()
    _reduced = []
    _incr_lock = None
    _today_incr_lock = None
    _yesterday_incr_lock = None

    @classmethod
    @gen.coroutine
    def _get_since_for(cls, collection_name):
        try:
            col_incr = yield cls.objects.get(collection=collection_name)
        except cls.DoesNotExist:
            col_incr = cls(collection=collection_name)
            yield col_incr.save()

        begin_of_all_times = settings.BEGIN_OF_ALL_TIMES if hasattr(
            settings, 'BEGIN_OF_ALL_TIMES') else None

        since = col_incr.last_incr or begin_of_all_times
        return since

    @classmethod
    @gen.coroutine
    def _set_since_for(cls, collection_name):
        col_incr = yield cls.objects.get(collection=collection_name)
        col_incr.last_incr = datetime.datetime.now()
        yield col_incr.save()

    @classmethod
    def add(cls, reduced_cls):
        """Adds a reduced report to the map-reduce queue."""

        cls._reduced.append(reduced_cls)

    @classmethod
    @gen.coroutine
    def do_incremental_map_reduce(cls):
        """Do an incremental map reduce since the last time
        it was done."""

        if cls._incr_lock is not None:
            return []

        cls._incr_lock = 'lóqui'
        futures = []
        for reduced_cls in cls._reduced:
            kw = {}
            coll_name = reduced_cls._get_collection_name()
            since = yield cls._get_since_for(coll_name)
            if since:
                kw = {'inclusion_date__gte': since}

            future = reduced_cls.do_aggregate(**kw)
            futures.append(future)

            yield cls._set_since_for(coll_name)

        cls._incr_lock = None
        return futures

    @classmethod
    @gen.coroutine
    def do_today_map_reduce(cls):
        """Do map reduce for the current day."""

        if cls._today_incr_lock is not None:
            return []

        cls._today_incr_lock = 'lóqui'
        futures = []
        now = datetime.datetime.now()
        today = datetime.datetime(now.year, now.month, now.day)
        kw = {'inclusion_date__gte': today}
        for reduced_cls in cls._reduced:
            log('today map reduce for {}'.format(fqualname(reduced_cls)),
                level='warning')
            future = reduced_cls.do_aggregate(**kw)
            futures.append(future)

        cls._today_incr_lock = None
        return futures

    @classmethod
    @gen.coroutine
    def do_yesterday_map_reduce(cls):
        """Do map reduce for the last day."""

        if cls._yesterday_incr_lock is not None:
            return []

        cls._yesterday_incr_lock = 'lóqui'
        futures = []
        now = datetime.datetime.now()
        today = datetime.datetime(now.year, now.month, now.day)
        yesterday = today - datetime.timedelta(days=1)

        kw = {'inclusion_date__gte': yesterday,
              'inclusion_date__lt': today}
        for reduced_cls in cls._reduced:
            log('yesterday map reduce for {}'.format(fqualname(reduced_cls)),
                level='warning')
            future = reduced_cls.do_aggregate(**kw)
            futures.append(future)

        cls._yesterday_incr_lock = None
        return futures


@gen.coroutine
def _hack_clean_bad_collections():
    """There is a bug with the aggregate/map-reduce I use. Sometimes, I don't
    know why the tmp collection is not dropped. This shit here is to drop
    these collections."""

    log('hack to clean!!!', level='warning')
    coll = models.ContentConsumption._collection
    db = coll.database
    collections = yield db.collection_names()
    tmp_pat = re.compile('.+?_\d+?')
    tmp_colls = [c for c in collections if re.match(tmp_pat, c)]
    for tmp in tmp_colls:
        tmp_coll = coll.__class__(db, tmp)
        yield tmp_coll.drop()


if hasattr(settings, 'USE_REDUCED_REPORTS') and settings.USE_REDUCED_REPORTS:
    reduced_reports = [ThemeConsumption, SiteConsumption,
                       SiteConsumers, ThemeConsumers, ContentConsumersReport,
                       ContentConsumptionReport, ReferrerConsumption]
    for r in reduced_reports:
        IncrementalMapReduce.add(r)

    # 2 hours
    secs = 3600 * 2
    print('adding incremental daily map reduce to scheduler')
    scheduler.add(IncrementalMapReduce.do_today_map_reduce, secs)
    scheduler.add(IncrementalMapReduce.do_yesterday_map_reduce,
                  hour_min=(0, 0))

    print('adding clean stuff')
    # 1 hour
    secs = 3600
    scheduler.add(_hack_clean_bad_collections, secs)
