# -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014, 2015 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import os
import mimetypes
from io import BytesIO
from datetime import timedelta, datetime
from unittest.mock import patch

from bson import ObjectId
from eve.utils import ParsedRequest
from nose.tools import assert_raises

from superdesk import get_resource_service, etree
from superdesk.utc import utcnow
from superdesk.errors import SuperdeskApiError, ProviderError
from superdesk.tests import TestCase
from superdesk.tests.setup_teardown import setup_providers, teardown_providers
from superdesk.io import get_feeding_service
from superdesk.io.commands.remove_expired_content import RemoveExpiredContent, get_expired_items
from superdesk.io.feeding_services.file_service import FileFeedingService
from superdesk.io.commands.update_ingest import (
    is_scheduled, update_provider, filter_expired_items, apply_rule_set, get_task_ttl, get_task_id, get_is_idle,
    ingest_item
)


reuters_guid = 'tag_reuters.com_2014_newsml_KBN0FL0NM:10'


def get_file(url, request_kwargs=None):
    """Return file data tuple as returned by media_operations.download_file_from_url"""
    filename = os.path.basename(url)
    dirname = os.path.dirname(os.path.realpath(__file__))
    filepath = os.path.normpath(os.path.join(dirname, 'fixtures', filename))
    mime = mimetypes.guess_type(filepath)[0]
    ext = str(mime).split('/')[1]
    name = str(ObjectId()) + ext
    with open(filepath, 'rb') as f:
        data = f.read()
    return BytesIO(data), name, mime


class UpdateIngestTest(TestCase):

    def setUp(self):
        setup_providers(self)

    def tearDown(self):
        teardown_providers(self)

    def _get_provider(self, provider_name):
        return get_resource_service('ingest_providers').find_one(name=provider_name, req=None)

    def _get_provider_service(self, provider):
        return get_feeding_service(provider['feeding_service'])

    def setup_reuters_provider(self):
        provider_name = 'reuters'
        provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
        provider_service = self._get_provider_service(provider)
        provider_service.provider = provider
        provider_service.URL = provider.get('config', {}).get('url')
        return provider, provider_service

    def test_ingest_items(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        items.extend(provider_service.fetch_ingest(reuters_guid))
        self.assertEqual(12, len(items))
        self.ingest_items(items, provider, provider_service)

    def test_ingest_item_expiry(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        self.assertIsNone(items[1].get('expiry'))
        items[1]['versioncreated'] = utcnow()
        self.ingest_items([items[1]], provider, provider_service)
        self.assertIsNotNone(items[1].get('expiry'))

    def test_ingest_item_sync_if_missing_from_elastic(self):
        provider, provider_service = self.setup_reuters_provider()
        item = provider_service.fetch_ingest(reuters_guid)[0]
        # insert in mongo
        ids = self.app.data._backend('ingest').insert('ingest', [item])
        # check that item is not in elastic
        elastic_item = self.app.data._search_backend('ingest').find_one('ingest', _id=ids[0], req=None)
        self.assertIsNone(elastic_item)
        # trigger sync by fetch
        old_item = get_resource_service('ingest').find_one(_id=ids[0], req=None)
        self.assertIsNotNone(old_item)
        # check that item is synced in elastic
        elastic_item = self.app.data._search_backend('ingest').find_one('ingest', _id=ids[0], req=None)
        self.assertIsNotNone(elastic_item)

    def test_ingest_provider_closed_raises_exception(self):
        provider = {
            'name': 'aap',
            'is_closed': True,
            'source': 'aap',
            'feeding_service': 'file', 'feed_parser': 'nitf',
            'config': {
                'path': '/'
            }
        }

        with assert_raises(SuperdeskApiError) as error_context:
            aap = self._get_provider_service(provider)
            aap.update(provider, {})
        ex = error_context.exception
        self.assertTrue(ex.status_code == 500)

    def test_ingest_provider_closed_when_critical_error_raised(self):
        provider_name = 'AAP'
        provider = self._get_provider(provider_name)
        self.assertFalse(provider.get('is_closed'))
        provider_service = self._get_provider_service(provider)
        provider_service.provider = provider
        provider_service.close_provider(provider, ProviderError.anpaError())
        provider = self._get_provider(provider_name)
        self.assertTrue(provider.get('is_closed'))

    def test_ingest_provider_calls_close_provider(self):
        def mock_update(provider, update):
            raise ProviderError.anpaError()

        provider_name = 'AAP'
        provider = self._get_provider(provider_name)
        self.assertFalse(provider.get('is_closed'))
        provider_service = self._get_provider_service(provider)
        provider_service.provider = provider
        provider_service._update = mock_update
        with assert_raises(ProviderError):
            provider_service.update(provider, {})
        provider = self._get_provider(provider_name)
        self.assertTrue(provider.get('is_closed'))

    def test_is_scheduled(self):
        self.assertTrue(is_scheduled({}), 'run after create')
        self.assertFalse(is_scheduled({'last_updated': utcnow()}), 'wait default time 5m')
        self.assertTrue(is_scheduled({'last_updated': utcnow() - timedelta(minutes=6)}), 'run after 5m')
        self.assertFalse(is_scheduled({
            'last_updated': utcnow() - timedelta(minutes=6),
            'update_schedule': {'minutes': 10}
        }), 'or wait if provider has specific schedule')
        self.assertTrue(is_scheduled({
            'last_updated': utcnow() - timedelta(minutes=11),
            'update_schedule': {'minutes': 10}
        }), 'and run eventually')

    def test_change_last_updated(self):
        ingest_provider = {'name': 'test', 'feeding_service': 'file', 'feed_parser': 'nitf', '_etag': 'test'}
        self.app.data.insert('ingest_providers', [ingest_provider])

        update_provider(ingest_provider)
        provider = self.app.data.find_one('ingest_providers', req=None, _id=ingest_provider['_id'])
        self.assertGreaterEqual(utcnow(), provider.get('last_updated'))
        self.assertEqual('test', provider.get('_etag'))

    def test_filter_expired_items(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        for item in items[:4]:
            item['expiry'] = utcnow() + timedelta(minutes=11)
        self.assertEqual(4, len(filter_expired_items(provider, items)))

    def test_filter_expired_items_with_no_expiry(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        self.assertEqual(0, len(filter_expired_items(provider, items)))

    def test_query_getting_expired_content(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        now = utcnow()
        for i, item in enumerate(items):
            item['ingest_provider'] = provider['_id']
            expiry_time = now - timedelta(hours=11)
            if i > 4:
                expiry_time = now + timedelta(minutes=11)

            item['expiry'] = item['versioncreated'] = expiry_time

        service = get_resource_service('ingest')
        service.post(items)
        expiredItems = get_expired_items(provider, 'ingest')
        self.assertEqual(5, expiredItems.count())

    def test_expiring_with_content(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        now = utcnow()
        for i, item in enumerate(items):
            item['ingest_provider'] = provider['_id']
            expiry_time = now - timedelta(hours=11)
            if i > 4:
                expiry_time = now + timedelta(minutes=11)

            item['expiry'] = item['versioncreated'] = expiry_time

        service = get_resource_service('ingest')
        service.post(items)

        # ingest the items and expire them
        before = service.get(req=None, lookup={})
        self.assertEqual(6, before.count())

        remove = RemoveExpiredContent()
        remove.run(provider.get('type'))

        # only one left in ingest
        after = service.get(req=None, lookup={})
        self.assertEqual(1, after.count())

        req = ParsedRequest()
        self.assertEqual(1, self.app.data.elastic.find('ingest', req, {}).count())
        self.assertEqual(1, self.app.data.mongo.find('ingest', req, {})[1])

    def test_removing_expired_items_from_elastic_only(self):
        now = utcnow()
        self.app.data.elastic.insert('ingest', [
            {'_id': 'foo', 'expiry': now - timedelta(minutes=30)},
            {'_id': 'bar', 'expiry': now + timedelta(minutes=30)},
        ])

        RemoveExpiredContent().run()
        self.assertEqual(1, self.app.data.elastic.find('ingest', ParsedRequest(), {}).count())

    def test_expiring_content_with_files(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        for item in items:
            item['ingest_provider'] = provider['_id']

        now = utcnow()
        items[0]['expiry'] = now - timedelta(hours=11)
        items[1]['expiry'] = now - timedelta(hours=11)
        items[2]['expiry'] = now + timedelta(hours=11)
        items[5]['versioncreated'] = now + timedelta(minutes=11)

        # ingest the items and expire them
        self.ingest_items(items, provider, provider_service)

        # four files in grid fs
        current_files = self.app.media.fs('upload').find()
        self.assertEqual(4, current_files.count())

        remove = RemoveExpiredContent()
        remove.run(provider.get('type'))

        # all gone
        current_files = self.app.media.fs('upload').find()
        self.assertEqual(0, current_files.count())

    def test_apply_rule_set(self):
        item = {'body_html': '@@body@@'}

        provider_name = 'reuters'
        provider = self._get_provider(provider_name)
        self.assertEqual('body', apply_rule_set(item, provider)['body_html'])

        item = {'body_html': '@@body@@'}
        provider_name = 'AAP'
        provider = self._get_provider(provider_name)
        self.assertEqual('@@body@@', apply_rule_set(item, provider)['body_html'])

    def test_all_ingested_items_have_sequence(self):
        provider, provider_service = self.setup_reuters_provider()
        guid = 'tag_reuters.com_2014_newsml_KBN0FL0NM:10'
        item = provider_service.fetch_ingest(guid)[0]
        get_resource_service("ingest").set_ingest_provider_sequence(item, provider)
        self.assertIsNotNone(item['ingest_provider_sequence'])

    def test_get_task_ttl(self):
        self.assertEqual(300, get_task_ttl({}))
        provider = {'update_schedule': {'minutes': 10}}
        self.assertEqual(600, get_task_ttl(provider))
        provider['update_schedule']['hours'] = 1
        provider['update_schedule']['minutes'] = 1
        self.assertEqual(3660, get_task_ttl(provider))

    def test_get_task_id(self):
        provider = {'name': 'foo', '_id': 'abc'}
        self.assertEqual('update-ingest-foo-abc', get_task_id(provider))

    def test_is_idle(self):
        provider = dict(idle_time=dict(hours=1, minutes=0))
        provider['last_item_update'] = utcnow()
        self.assertEqual(get_is_idle(provider), False)
        provider['idle_time']['hours'] = -1
        self.assertEqual(get_is_idle(provider), True)
        provider['idle_time'] = dict(hours=0, minutes=0)
        self.assertEqual(get_is_idle(provider), False)

    def test_files_dont_duplicate_ingest(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)

        for item in items:
            item['ingest_provider'] = provider['_id']
            item['expiry'] = utcnow() + timedelta(hours=11)

        # ingest the items
        self.ingest_items(items, provider, provider_service)

        items = provider_service.fetch_ingest(reuters_guid)
        for item in items:
            item['ingest_provider'] = provider['_id']
            item['expiry'] = utcnow() + timedelta(hours=11)

        # ingest them again
        self.ingest_items(items, provider, provider_service)

        # 12 files in grid fs
        current_files = self.app.media.fs('upload').find()
        self.assertEqual(12, current_files.count())

    def test_anpa_category_to_subject_derived_ingest(self):
        vocab = [{'_id': 'categories', 'items': [{'is_active': True, 'name': 'Domestic Sport', 'qcode': 's',
                                                  "subject": "15000000"}]}]
        self.app.data.insert('vocabularies', vocab)

        provider_name = 'DPA'
        provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
        file_path = os.path.join(provider.get('config', {}).get('path', ''), 'IPTC7901_odd_charset.txt')
        provider_service = self._get_provider_service(provider)
        feeding_parser = provider_service.get_feed_parser(provider)
        items = [feeding_parser.parse(file_path, provider)]

        # ingest the items and check the subject code has been derived
        self.ingest_items(items, provider, provider_service)
        self.assertEqual(items[0]['subject'][0]['qcode'], '15000000')

    def test_anpa_category_to_subject_derived_ingest_ignores_inactive_categories(self):
        vocab = [{'_id': 'categories', 'items': [{'is_active': False, 'name': 'Domestic Sport', 'qcode': 's',
                                                  "subject": "15000000"}]}]
        self.app.data.insert('vocabularies', vocab)

        provider_name = 'DPA'
        provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
        file_path = os.path.join(provider.get('config', {}).get('path', ''), 'IPTC7901_odd_charset.txt')
        provider_service = self._get_provider_service(provider)
        feeding_parser = provider_service.get_feed_parser(provider)
        items = [feeding_parser.parse(file_path, provider)]

        # ingest the items and check the subject code has been derived
        self.ingest_items(items, provider, provider_service)
        self.assertNotIn('subject', items[0])

    def test_subject_to_anpa_category_derived_ingest(self):
        vocab = [{'_id': 'iptc_category_map',
                  'items': [{'name': 'Finance', 'category': 'f', 'qcode': '04000000', 'is_active': True}]},
                 {'_id': 'categories',
                  'items': [{'is_active': True, 'name': 'Australian Weather', 'qcode': 'b', 'subject': '17000000'},
                            {'is_active': True, 'name': 'Finance', 'qcode': 'f'}]},
                 {'_id': 'genre', 'items': [{'qcode': 'feature', 'name': 'feature'}]}]

        self.app.data.insert('vocabularies', vocab)

        provider_name = 'AAP'
        provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
        file_path = os.path.join(provider.get('config', {}).get('path', ''), 'nitf-fishing.xml')
        provider_service = self._get_provider_service(provider)
        feeding_parser = provider_service.get_feed_parser(provider)
        with open(file_path, 'r') as f:
            xml_string = etree.etree.fromstring(f.read())
            items = [feeding_parser.parse(xml_string, provider)]
            for item in items:
                item['ingest_provider'] = provider['_id']
                item['expiry'] = utcnow() + timedelta(hours=11)

            # ingest the items and check the subject code has been derived
            self.ingest_items(items, provider, provider_service)
            self.assertEqual(items[0]['anpa_category'][0]['qcode'], 'f')

    def test_subject_to_anpa_category_derived_ingest_ignores_inactive_map_entries(self):
        vocab = [{'_id': 'iptc_category_map',
                  'items': [{'name': 'Finance', 'category': 'f', 'qcode': '04000000', 'is_active': False}]},
                 {'_id': 'categories',
                  'items': [{'is_active': True, 'name': 'Australian Weather', 'qcode': 'b', 'subject': '17000000'}]},
                 {'_id': 'genre', 'items': [{'qcode': 'feature', 'name': 'feature'}]}]

        self.app.data.insert('vocabularies', vocab)

        provider_name = 'AAP'
        provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
        file_path = os.path.join(provider.get('config', {}).get('path', ''), 'nitf-fishing.xml')
        provider_service = self._get_provider_service(provider)
        feeding_parser = provider_service.get_feed_parser(provider)
        with open(file_path, 'r') as f:
            xml_string = etree.etree.fromstring(f.read())
            items = [feeding_parser.parse(xml_string, provider)]
            for item in items:
                item['ingest_provider'] = provider['_id']
                item['expiry'] = utcnow() + timedelta(hours=11)

            # ingest the items and check the subject code has been derived
            self.ingest_items(items, provider, provider_service)
            self.assertNotIn('anpa_category', items[0])

    def test_ingest_cancellation(self):
        provider, provider_service = self.setup_reuters_provider()
        guid = 'tag_reuters.com_2016_newsml_L1N14N0FF:978556838'
        items = provider_service.fetch_ingest(guid)
        for item in items:
            item['ingest_provider'] = provider['_id']
            item['expiry'] = utcnow() + timedelta(hours=11)
        self.ingest_items(items, provider, provider_service)
        guid = 'tag_reuters.com_2016_newsml_L1N14N0FF:1542761538'
        items = provider_service.fetch_ingest(guid)
        for item in items:
            item['ingest_provider'] = provider['_id']
            item['expiry'] = utcnow() + timedelta(hours=11)
        self.ingest_items(items, provider, provider_service)
        ingest_service = get_resource_service('ingest')
        lookup = {'uri': items[0].get('uri')}
        family_members = ingest_service.get_from_mongo(req=None, lookup=lookup)
        self.assertEqual(family_members.count(), 2)
        for relative in family_members:
            self.assertEqual(relative['pubstatus'], 'canceled')
            self.assertEqual(relative['state'], 'killed')

    def test_ingest_update(self):
        provider, provider_service = self.setup_reuters_provider()
        items = provider_service.fetch_ingest(reuters_guid)
        items[0]['ingest_provider'] = provider['_id']
        items[0]['expiry'] = utcnow() + timedelta(hours=11)

        self.ingest_items(items, provider, provider_service)

        self.assertEqual(items[0]['unique_id'], 1)
        original_id = items[0]['_id']

        items = provider_service.fetch_ingest(reuters_guid)
        items[0]['ingest_provider'] = provider['_id']
        items[0]['expiry'] = utcnow() + timedelta(hours=11)
        # change the headline
        items[0]['headline'] = 'Updated headline'

        # ingest the item again
        self.ingest_items(items, provider, provider_service)

        # see the update to the headline and unique_id survives
        elastic_item = self.app.data._search_backend('ingest').find_one('ingest', _id=original_id, req=None)
        self.assertEqual(elastic_item['headline'], 'Updated headline')
        self.assertEqual(elastic_item['unique_id'], 1)
        self.assertEqual(elastic_item['unique_name'], '#1')

    def test_get_article_ids(self):
        provider_name = 'reuters'
        provider, provider_service = self.setup_reuters_provider()
        ids = provider_service._get_article_ids('channel1', utcnow(), utcnow() + timedelta(minutes=-10))
        self.assertEqual(len(ids), 3)
        provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
        self.assertEqual(provider['tokens']['poll_tokens']['channel1'], 'ExwaY31kfnR2Z2J1cWZ2YnxoYH9kfw==')

    def test_unknown_category_ingested_is_removed(self):
        vocab = [
            {'_id': 'categories',
             'items': [{'is_active': True, 'name': 'Australian Weather', 'qcode': 'b', 'subject': '17000000'},
                       {'is_active': True, 'name': 'Finance', 'qcode': 'f'}]}
        ]

        self.app.data.insert('vocabularies', vocab)

        provider_name = 'AP'
        provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
        file_path = os.path.join(provider.get('config', {}).get('path', ''), 'ap_anpa-3.tst')
        provider_service = self._get_provider_service(provider)
        feeding_parser = provider_service.get_feed_parser(provider)
        items = [feeding_parser.parse(file_path, provider)]

        # ingest the items and check the subject code has been derived
        items[0]['versioncreated'] = utcnow()
        self.ingest_items(items, provider, provider_service)
        self.assertTrue(len(items[0]['anpa_category']) == 0)

    def test_ingest_with_routing_keeps_elastic_in_sync(self):
        provider, provider_service = self.setup_reuters_provider()

        desk = {'name': 'foo'}
        self.app.data.insert('desks', [desk])
        self.assertIsNotNone(desk['_id'])
        self.assertIsNotNone(desk['incoming_stage'])

        now = datetime.now()

        items = [
            {
                'guid': 'main_text',
                'versioncreated': now,
                'headline': 'Headline of the text item',
            },
            {'guid': 'image_1', 'type': 'picture', 'versioncreated': now},
            {'guid': 'image_2', 'type': 'picture', 'versioncreated': now},
            {
                'type': 'composite',
                'guid': 'package:guid:abcd123',
                'versioncreated': now,
                'headline': 'Headline of the text item',
                'groups': [
                    {
                        'id': 'root',
                        'role': 'grpRole:NEP',
                        'refs': [{'idRef': 'main'}],
                    }, {
                        'id': 'main',
                        'role': 'main',
                        'refs': [
                            {'residRef': 'main_text'},
                            {'residRef': 'image_1'},
                            {'residRef': 'image_2'},
                        ],
                    }
                ]
            }
        ]

        all_week_schedule = {
            "day_of_week": [
                "MON",
                "TUE",
                "WED",
                "THU",
                "FRI",
                "SAT",
                "SUN"
            ],
            "hour_of_day_from": "00:00:00",
            "hour_of_day_to": "23:55:00",
            "time_zone": "Europe/Prague"
        }

        routing_scheme = {
            "name": "autofetch",
            "rules": [
                {
                    "filter": None,
                    "actions": {
                        "exit": False,
                        "publish": [],
                        "fetch": [
                            {
                                "stage": desk['incoming_stage'],
                                "desk": desk['_id']
                            }
                        ]
                    },
                    "schedule": all_week_schedule,
                    "name": "fetch"
                },
                {
                    "filter": None,
                    "actions": {
                        "exit": False,
                        "publish": [],
                        "fetch": []
                    },
                    "schedule": all_week_schedule,
                    "name": "empty"
                },
            ]
        }

        ingest_service = get_resource_service('ingest')
        self.ingest_items(items, provider, provider_service, routing_scheme=routing_scheme)

        self.assertEqual(4, ingest_service.get_from_mongo(None, {}).count())
        self.assertEqual(4, ingest_service.get(None, {}).count())

        for item in items:
            lookup = {'guid': item['guid']}
            mongo_item = ingest_service.get_from_mongo(None, lookup)[0]
            elastic_item = ingest_service.get(None, lookup)[0]
            self.assertEqual(mongo_item['_etag'], elastic_item['_etag'], mongo_item['guid'])

    @patch('superdesk.media.renditions.download_file_from_url', get_file)
    def test_ingest_associated_item_renditions(self):
        provider = {'feeding_service': 'ninjs', '_id': self.providers['ninjs']}
        provider_service = FileFeedingService()
        item = {
            'guid': 'foo',
            'type': 'text',
            'versioncreated': utcnow(),
            'associations': {
                'featuremedia': {
                    'guid': 'bar',
                    'type': 'picture',
                    'versioncreated': utcnow(),
                    'renditions': {
                        'original': {
                            'href': 'https://farm8.staticflickr.com/7300/picture_bug.jpg',
                            'mimetype': 'image/jpeg',
                            'width': 640,
                            'height': 426,
                        }
                    }
                },
                'foo': {
                    'guid': 'foo',
                    'type': 'picture',
                    'versioncreated': utcnow(),
                    'renditions': {
                        'original': {
                            'href': 'https://farm8.staticflickr.com/7300/picture_bug.jpg',
                            'mimetype': 'image/jpeg',
                            'width': 640,
                            'height': 426,
                        },
                    },
                },
            }
        }

        # avoid transfer_renditions call which would store the picture locally
        # and it would fetch it using superdesk url which doesn't work in test
        with patch('superdesk.io.commands.update_ingest.transfer_renditions'):
            status, ids = ingest_item(item, provider, provider_service)

        self.assertTrue(status)
        self.assertEqual(3, len(ids))
        self.assertIn('thumbnail', item['associations']['featuremedia']['renditions'])
        self.assertIn('thumbnail', item['associations']['foo']['renditions'])
