from __future__ import absolute_import
import os
import json
import random
import datetime
import time
import six
from collections import OrderedDict  # from python 2.7
import pytest

from nose.tools import make_decorator
import mock
import responses
from sqlalchemy import MetaData, Table
from sqlalchemy.sql import select

import ckan.plugins as p

try:
    config = p.toolkit.config
except AttributeError:
    from pylons import config

from ckanext.xloader import jobs
from ckanext.xloader import db as jobs_db
from ckanext.xloader.loader import get_write_engine

from ckan.tests import helpers, factories

SOURCE_URL = "http://www.example.com/static/file"


def mock_actions(func):
    """
    Decorator that mocks actions used by these tests
    Based on ckan.test.helpers.mock_action
    """

    def wrapper(*args, **kwargs):
        # Mock CKAN's resource_show API
        from ckan.logic import get_action as original_get_action

        def side_effect(called_action_name):
            if called_action_name == "resource_show":

                def mock_resource_show(context, data_dict):
                    return {
                        "id": data_dict["id"],
                        "name": "short name",
                        "url": SOURCE_URL,
                        "format": "",
                        "package_id": "test-pkg",
                    }

                return mock_resource_show
            elif called_action_name == "package_show":

                def mock_package_show(context, data_dict):
                    return {
                        "id": data_dict["id"],
                        "name": "pkg-name",
                    }

                return mock_package_show
            else:
                return original_get_action(called_action_name)

        try:
            with mock.patch(
                "ckanext.xloader.jobs.get_action"
            ) as mock_get_action:
                mock_get_action.side_effect = side_effect

                return_value = func(*args, **kwargs)
        finally:
            pass
            # Make sure to stop the mock, even with an exception
            # mock_action.stop()
        return return_value

    return make_decorator(func)(wrapper)


@pytest.mark.skip
@pytest.mark.usefixtures("full_reset", "with_plugins")
@pytest.mark.ckan_config("ckan.plugins", "datastore xloader")
class TestxloaderDataIntoDatastore(object):

    @pytest.fixture(autouse=True)
    def setup_class(self):
        self.host = "www.ckan.org"
        self.api_key = "my-fake-key"
        self.resource_id = "foo-bar-42"
        res = factories.Resource(id=self.resource_id)
        jobs_db.init(config, echo=False)
        # drop test table
        engine, conn = self.get_datastore_engine_and_connection()
        conn.execute('DROP TABLE IF EXISTS "{}"'.format(self.resource_id))
        yield
        if "_datastore" in dir(self):
            connection = self._datastore[1]
            connection.close()

    def register_urls(
        self, filename="simple.csv", content_type="application/csv"
    ):
        """Mock some test URLs with responses.

        Mocks some URLs related to a data file and a CKAN resource that
        contains the data file, including the URL of the data file itself and
        the resource_show, resource_update and datastore_delete URLs.

        :returns: a 2-tuple containing the URL of the data file itself and the
            resource_show URL for the resource that contains the data file

        """
        responses.add_passthru(config["solr_url"])

        # A URL that just returns a static file
        responses.add(
            responses.GET,
            SOURCE_URL,
            body=get_sample_file(filename),
            content_type=content_type,
        )

        # A URL that mocks the response that CKAN's resource_update API would
        # give after successfully updating a resource.
        resource_update_url = (
            "http://www.ckan.org/api/3/action/resource_update"
        )
        responses.add(
            responses.POST,
            resource_update_url,
            body=json.dumps({"success": True}),
            content_type="application/json",
        )

        # A URL that mock's the response that CKAN's datastore plugin's
        # datastore_delete API would give after successfully deleting a
        # resource from the datastore.
        datastore_del_url = "http://www.ckan.org/api/3/action/datastore_delete"
        responses.add(
            responses.POST,
            datastore_del_url,
            body=json.dumps({"success": True}),
            content_type="application/json",
        )

        self.callback_url = "http://www.ckan.org/api/3/action/xloader_hook"
        responses.add(
            responses.POST,
            self.callback_url,
            body=json.dumps({"success": True}),
            content_type="application/json",
        )

    @classmethod
    def get_datastore_engine_and_connection(cls):
        if "_datastore" not in dir(cls):
            engine = get_write_engine()
            conn = engine.connect()
            cls._datastore = (engine, conn)
        return cls._datastore

    def get_datastore_table(self):
        engine, conn = self.get_datastore_engine_and_connection()
        meta = MetaData(bind=engine, reflect=True)
        table = Table(
            self.resource_id, meta, autoload=True, autoload_with=engine
        )
        s = select([table])
        with conn.begin():
            result = conn.execute(s)
        return dict(
            num_rows=result.rowcount,
            headers=list(result.keys()),
            header_dict=OrderedDict(
                [(c.key, six.text_type(c.type)) for c in table.columns]
            ),
            rows=result.fetchall(),
        )

    def get_load_logs(self, task_id):
        conn = jobs_db.ENGINE.connect()
        logs = jobs_db.LOGS_TABLE
        result = conn.execute(
            select([logs.c.level, logs.c.message]).where(
                logs.c.job_id == task_id
            )
        )
        return Logs(result.fetchall())

    def get_time_of_last_analyze(self):
        # When ANALYZE runs it appears to take a moment for the
        # pg_stat_user_tables to update, which we use to check analyze runs,
        # so sadly we need a sleep :(
        # DR: 0.25 is pretty reliable on my machine, but give a wide margin
        time.sleep(1)
        engine, conn = self.get_datastore_engine_and_connection()
        result = conn.execute(
            """
            SELECT last_analyze, last_autoanalyze
            FROM pg_stat_user_tables
            WHERE relname='{}';
            """.format(
                self.resource_id
            )
        )
        last_analyze_datetimes = result.fetchall()[0]
        return max([x for x in last_analyze_datetimes if x] or [None])

    @mock_actions
    @responses.activate
    def test_simple_csv(self):
        # Test not only the load and xloader_hook is called at the end
        self.register_urls(filename="simple.csv")
        data = {
            "api_key": self.api_key,
            "job_type": "xloader_to_datastore",
            "result_url": self.callback_url,
            "metadata": {
                "ckan_url": "http://%s/" % self.host,
                "resource_id": self.resource_id,
            },
        }
        job_id = "test{}".format(random.randint(0, 1e5))

        with mock.patch(
            "ckanext.xloader.jobs.set_resource_metadata"
        ) as mocked_set_resource_metadata:
            # in tests we call jobs directly, rather than use rq, so mock
            # get_current_job()
            with mock.patch(
                "ckanext.xloader.jobs.get_current_job",
                return_value=mock.Mock(id=job_id),
            ):
                result = jobs.xloader_data_into_datastore(data)
        assert result is None, jobs_db.get_job(job_id)["error"]["message"]

        # Check it said it was successful
        assert (
            responses.calls[-1].request.url
            == "http://www.ckan.org/api/3/action/xloader_hook"
        )
        job_dict = json.loads(responses.calls[-1].request.body)
        assert job_dict["status"] == u"complete", job_dict
        assert job_dict == {
            u"metadata": {
                u"datastore_contains_all_records_of_source_file": True,
                u"datastore_active": True,
                u"ckan_url": u"http://www.ckan.org/",
                u"resource_id": u"foo-bar-42",
            },
            u"status": u"complete",
        }

        # Check the load
        data = self.get_datastore_table()
        assert data["headers"] == [
            "_id",
            "_full_text",
            "date",
            "temperature",
            "place",
        ]
        assert data["header_dict"]["date"] == "TEXT"
        # 'TIMESTAMP WITHOUT TIME ZONE')
        assert data["header_dict"]["temperature"] == "TEXT"  # 'NUMERIC')
        assert data["header_dict"]["place"] == "TEXT"  # 'TEXT')
        assert data["num_rows"] == 6
        assert data["rows"][0][2:] == (u"2011-01-01", u"1", u"Galway")
        # (datetime.datetime(2011, 1, 1), 1, 'Galway'))

        # Check it wanted to set the datastore_active=True
        mocked_set_resource_metadata.assert_called_once()
        assert mocked_set_resource_metadata.call_args[1]["update_dict"] == {
            "datastore_contains_all_records_of_source_file": True,
            "datastore_active": True,
            "ckan_url": "http://www.ckan.org/",
            "resource_id": "foo-bar-42",
        }

        logs = self.get_load_logs(job_id)
        logs.assert_no_errors()

        job = jobs_db.get_job(job_id)
        assert job["status"] == u"complete"
        assert job["error"] == None

        # Check ANALYZE was run
        last_analyze = self.get_time_of_last_analyze()
        assert last_analyze

    @mock_actions
    @responses.activate
    @mock.patch("ckanext.xloader.jobs.MAX_CONTENT_LENGTH", 10000)
    @mock.patch("ckanext.xloader.jobs.MAX_EXCERPT_LINES", 100)
    def test_too_large_csv(self):

        # Test not only the load and xloader_hook is called at the end
        self.register_urls(filename="simple-large.csv")
        data = {
            "api_key": self.api_key,
            "job_type": "xloader_to_datastore",
            "result_url": self.callback_url,
            "metadata": {
                "ckan_url": "http://%s/" % self.host,
                "resource_id": self.resource_id,
            },
        }
        job_id = "test{}".format(random.randint(0, 1e5))

        with mock.patch(
            "ckanext.xloader.jobs.set_resource_metadata"
        ) as mocked_set_resource_metadata:
            # in tests we call jobs directly, rather than use rq, so mock
            # get_current_job()
            with mock.patch(
                "ckanext.xloader.jobs.get_current_job",
                return_value=mock.Mock(id=job_id),
            ):
                result = jobs.xloader_data_into_datastore(data)
        assert result is None, jobs_db.get_job(job_id)["error"]["message"]

        # Check it said it was successful
        assert (
            responses.calls[-1].request.url
            == "http://www.ckan.org/api/3/action/xloader_hook"
        )
        job_dict = json.loads(responses.calls[-1].request.body)
        assert job_dict["status"] == u"complete", job_dict
        assert job_dict == {
            u"metadata": {
                u"datastore_contains_all_records_of_source_file": False,
                u"datastore_active": True,
                u"ckan_url": u"http://www.ckan.org/",
                u"resource_id": u"foo-bar-42",
            },
            u"status": u"complete",
        }

        # Check the load
        data = self.get_datastore_table()
        assert data["headers"] == ["_id", "_full_text", "id", "text"]
        assert data["header_dict"]["id"] == "TEXT"
        # 'TIMESTAMP WITHOUT TIME ZONE')
        assert data["header_dict"]["text"] == "TEXT"
        assert data["num_rows"] <= 100
        assert data["num_rows"] > 0
        assert data["rows"][0][2:] == (u"1", u"a")

        # Check it wanted to set the datastore_active=True
        mocked_set_resource_metadata.assert_called_once()
        assert mocked_set_resource_metadata.call_args[1]["update_dict"] == {
            "datastore_contains_all_records_of_source_file": False,
            "datastore_active": True,
            "ckan_url": "http://www.ckan.org/",
            "resource_id": "foo-bar-42",
        }

        logs = self.get_load_logs(job_id)
        logs.assert_no_errors()

        job = jobs_db.get_job(job_id)
        assert job["status"] == u"complete"
        assert job["error"] == None

        # Check ANALYZE was run
        last_analyze = self.get_time_of_last_analyze()
        assert last_analyze

    @mock_actions
    @responses.activate
    @mock.patch("ckanext.xloader.jobs.MAX_CONTENT_LENGTH", 10000)
    @mock.patch("ckanext.xloader.jobs.MAX_EXCERPT_LINES", 100)
    def test_too_large_xls(self):

        # Test not only the load and xloader_hook is called at the end
        self.register_urls(filename="simple-large.xls")
        data = {
            "api_key": self.api_key,
            "job_type": "xloader_to_datastore",
            "result_url": self.callback_url,
            "metadata": {
                "ckan_url": "http://%s/" % self.host,
                "resource_id": self.resource_id,
            },
        }
        job_id = "test{}".format(random.randint(0, 1e5))

        with mock.patch("ckanext.xloader.jobs.set_resource_metadata"):
            # in tests we call jobs directly, rather than use rq, so mock
            # get_current_job()
            with mock.patch(
                "ckanext.xloader.jobs.get_current_job",
                return_value=mock.Mock(id=job_id),
            ):
                result = jobs.xloader_data_into_datastore(data)
        assert result is not None, jobs_db.get_job(job_id)["error"]["message"]

        # Check it said it was successful
        assert (
            responses.calls[-1].request.url
            == "http://www.ckan.org/api/3/action/xloader_hook"
        )
        job_dict = json.loads(responses.calls[-1].request.body)
        assert job_dict["status"] == u"error", job_dict
        assert job_dict == {
            u"status": u"error",
            u"metadata": {
                u"ckan_url": u"http://www.ckan.org/",
                u"datastore_contains_all_records_of_source_file": False,
                u"resource_id": u"foo-bar-42",
            },
            u"error": u"Loading file raised an error: array index out of range",
        }

        job = jobs_db.get_job(job_id)
        assert job["status"] == u"error"
        assert job["error"] == {
            u"message": u"Loading file raised an error: array index out of range"
        }

    @mock_actions
    @responses.activate
    def test_messytables(self):
        # xloader's COPY can't handle xls, so it will be dealt with by
        # messytables
        self.register_urls(
            filename="simple.xls", content_type="application/vnd.ms-excel"
        )
        data = {
            "api_key": self.api_key,
            "job_type": "xloader_to_datastore",
            "result_url": self.callback_url,
            "metadata": {
                "ckan_url": "http://%s/" % self.host,
                "resource_id": self.resource_id,
            },
        }
        job_id = "test{}".format(random.randint(0, 1e5))

        with mock.patch(
            "ckanext.xloader.jobs.set_resource_metadata"
        ) as mocked_set_resource_metadata:
            # in tests we call jobs directly, rather than use rq, so mock
            # get_current_job()
            with mock.patch(
                "ckanext.xloader.jobs.get_current_job",
                return_value=mock.Mock(id=job_id),
            ):
                result = jobs.xloader_data_into_datastore(data)
        assert result == None

        # Check it said it was successful
        assert (
            responses.calls[-1].request.url
            == "http://www.ckan.org/api/3/action/xloader_hook"
        )
        job_dict = json.loads(responses.calls[-1].request.body)
        assert job_dict["status"] == u"complete", job_dict
        assert job_dict == {
            u"metadata": {
                u"datastore_contains_all_records_of_source_file": True,
                u"datastore_active": True,
                u"ckan_url": u"http://www.ckan.org/",
                u"resource_id": u"foo-bar-42",
            },
            u"status": u"complete",
        }

        # Check the load
        data = self.get_datastore_table()
        assert data["headers"] == [
            "_id",
            "_full_text",
            "date",
            "temperature",
            "place",
        ]
        assert data["header_dict"]["date"] == "TIMESTAMP WITHOUT TIME ZONE"
        assert data["header_dict"]["temperature"] == "NUMERIC"
        assert data["header_dict"]["place"] == "TEXT"
        assert data["num_rows"] == 6
        assert data["rows"][0][2:] == (
            datetime.datetime(2011, 1, 1),
            1,
            u"Galway",
        )

        # Check it wanted to set the datastore_active=True
        mocked_set_resource_metadata.assert_called_once()
        assert mocked_set_resource_metadata.call_args[1]["update_dict"] == {
            "ckan_url": "http://www.ckan.org/",
            "datastore_contains_all_records_of_source_file": True,
            "datastore_active": True,
            "resource_id": "foo-bar-42",
        }

        # check logs have the error doing the COPY
        logs = self.get_load_logs(job_id)
        copy_error_index = None
        for i, log in enumerate(logs):
            if log[0] == "WARNING" and log[1].startswith(
                "Load using COPY failed: Error during the load into PostgreSQL"
            ):
                copy_error_index = i
                break
        assert copy_error_index, "Missing COPY error"

        # check messytable portion of the logs
        logs = Logs(logs[copy_error_index + 1 :])
        assert logs[0] == (u"INFO", u"Trying again with messytables")
        logs.assert_no_errors()

        # Check ANALYZE was run
        last_analyze = self.get_time_of_last_analyze()
        assert last_analyze

    @mock_actions
    @responses.activate
    def test_umlaut_and_extra_comma(self):
        self.register_urls(filename="umlaut_and_extra_comma.csv")
        # This csv has an extra comma which causes the COPY to throw a
        # psycopg2.DataError and the umlaut can cause problems for logging the
        # error. We need to check that it correctly reverts to using
        # messytables to load it
        data = {
            "api_key": self.api_key,
            "job_type": "xloader_to_datastore",
            "result_url": self.callback_url,
            "metadata": {
                "ckan_url": "http://%s/" % self.host,
                "resource_id": self.resource_id,
            },
        }
        job_id = "test{}".format(random.randint(0, 1e5))

        with mock.patch("ckanext.xloader.jobs.set_resource_metadata"):
            # in tests we call jobs directly, rather than use rq, so mock
            # get_current_job()
            with mock.patch(
                "ckanext.xloader.jobs.get_current_job",
                return_value=mock.Mock(id=job_id),
            ):
                result = jobs.xloader_data_into_datastore(data)
        assert result is None, jobs_db.get_job(job_id)["error"]["message"]

        # Check it said it was successful
        assert (
            responses.calls[-1].request.url
            == "http://www.ckan.org/api/3/action/xloader_hook"
        )
        job_dict = json.loads(responses.calls[-1].request.body)
        assert job_dict["status"] == u"complete", job_dict
        assert job_dict == {
            u"metadata": {
                u"datastore_contains_all_records_of_source_file": True,
                u"datastore_active": True,
                u"ckan_url": u"http://www.ckan.org/",
                u"resource_id": u"foo-bar-42",
            },
            u"status": u"complete",
        }

        logs = self.get_load_logs(job_id)
        logs.assert_no_errors()

        job = jobs_db.get_job(job_id)
        assert job["status"] == u"complete"
        assert job["error"] == None

    @mock_actions
    @responses.activate
    def test_first_request_is_202_pending_response(self):
        # when you first get the CSV it returns this 202 response, which is
        # what this server does: https://data-cdfw.opendata.arcgis.com/datasets
        responses.add(
            responses.GET,
            SOURCE_URL,
            status=202,
            body='{"processingTime":"8.716 seconds","status":"Processing","generating":{}}',
            content_type="application/json",
        )
        # subsequent GETs of the CSV work fine
        self.register_urls()
        data = {
            "api_key": self.api_key,
            "job_type": "xloader_to_datastore",
            "result_url": self.callback_url,
            "metadata": {
                "ckan_url": "http://%s/" % self.host,
                "resource_id": self.resource_id,
            },
        }
        job_id = "test{}".format(random.randint(0, 1e5))

        with mock.patch(
            "ckanext.xloader.jobs.set_resource_metadata"
        ) as mocked_set_resource_metadata:
            # in tests we call jobs directly, rather than use rq, so mock
            # get_current_job()
            with mock.patch(
                "ckanext.xloader.jobs.get_current_job",
                return_value=mock.Mock(id=job_id),
            ):
                result = jobs.xloader_data_into_datastore(data)
        assert result is None, jobs_db.get_job(job_id)["error"]["message"]

        # Check it said it was successful
        assert (
            responses.calls[-1].request.url
            == "http://www.ckan.org/api/3/action/xloader_hook"
        )
        job_dict = json.loads(responses.calls[-1].request.body)
        assert job_dict["status"] == u"complete", job_dict
        assert job_dict == {
            u"metadata": {
                u"ckan_url": u"http://www.ckan.org/",
                u"datastore_contains_all_records_of_source_file": True,
                u"datastore_active": True,
                u"resource_id": u"foo-bar-42",
            },
            u"status": u"complete",
        }

        # Check the load
        data = self.get_datastore_table()
        assert data["headers"] == [
            "_id",
            "_full_text",
            "date",
            "temperature",
            "place",
        ]
        assert data["header_dict"]["date"] == "TEXT"
        # 'TIMESTAMP WITHOUT TIME ZONE')
        assert data["header_dict"]["temperature"] == "TEXT"  # 'NUMERIC')
        assert data["header_dict"]["place"] == "TEXT"  # 'TEXT')
        assert data["num_rows"] == 6
        assert data["rows"][0][2:] == (u"2011-01-01", u"1", u"Galway")
        # (datetime.datetime(2011, 1, 1), 1, 'Galway'))

        # Check it wanted to set the datastore_active=True
        mocked_set_resource_metadata.assert_called_once()
        assert mocked_set_resource_metadata.call_args[1]["update_dict"] == {
            "datastore_contains_all_records_of_source_file": True,
            "datastore_active": True,
            "ckan_url": "http://www.ckan.org/",
            "resource_id": "foo-bar-42",
        }

        logs = self.get_load_logs(job_id)
        logs.assert_no_errors()

        job = jobs_db.get_job(job_id)
        assert job["status"] == u"complete"
        assert job["error"] is None


class Logs(list):
    def get_errors(self):
        return [message for level, message in self if level == "ERROR"]

    def grep(self, text):
        return [message for level, message in self if text in message]

    def assert_no_errors(self):
        errors = self.get_errors()
        assert not errors, errors


def get_sample_file(filename):
    filepath = os.path.join(os.path.dirname(__file__), "samples", filename)
    return open(filepath).read()


class TestSetResourceMetadata(object):
    @classmethod
    def setup_class(cls):
        helpers.reset_db()

    def test_simple(self):
        resource = factories.Resource()

        jobs.set_resource_metadata(
            {
                "datastore_contains_all_records_of_source_file": True,
                "datastore_active": True,
                "ckan_url": "http://www.ckan.org/",
                "resource_id": resource["id"],
            }
        )

        resource = helpers.call_action("resource_show", id=resource["id"])
        from pprint import pprint

        pprint(resource)
        assert resource["datastore_contains_all_records_of_source_file"] in (
            True,
            u"True",
        )
        # I'm not quite sure why this is a string on travis - I get the bool
        # locally

        assert resource["datastore_active"]
        assert resource["ckan_url"] == "http://www.ckan.org/"
