#
# (C) Copyright IBM Corp. 2020
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
import json
import pickle
import argparse
import unittest
import logging
import inspect
import pywren_ibm_cloud as pywren
import urllib.request
from pywren_ibm_cloud.storage import InternalStorage
from pywren_ibm_cloud.config import default_config, extract_storage_config
from concurrent.futures import ThreadPoolExecutor

CONFIG = None
STORAGE_CONFIG = None
STORAGE = None

PREFIX = '__pywren.test'
TEST_FILES_URLS = ["http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.enron.txt",
                   "http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.kos.txt",
                   "http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.nips.txt",
                   "http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.nytimes.txt",
                   "http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.pubmed.txt"]


class TestUtils:

    @staticmethod
    def initTests():
        def up(param):
            i, url = param
            content = urllib.request.urlopen(url).read()
            STORAGE.put_object(bucket_name=STORAGE_CONFIG['bucket'],
                               key='{}/test{}'.format(PREFIX, str(i)),
                               data=content)
            return len(content.split())

        with ThreadPoolExecutor() as pool:
            results = list(pool.map(up, enumerate(TEST_FILES_URLS)))

        result_to_compare = sum(results)
        return result_to_compare

    @staticmethod
    def list_test_keys():
        return STORAGE.list_keys(bucket_name=STORAGE_CONFIG['bucket'], prefix=PREFIX + '/')

    @staticmethod
    def cleanTests():
        for key in TestUtils.list_test_keys():
            STORAGE.delete_object(bucket_name=STORAGE_CONFIG['bucket'],
                                  key=key)


class TestMethods:

    @staticmethod
    def hello_world(param):
        return "Hello World!"

    @staticmethod
    def concat(lst):
        return " ".join(lst)

    @staticmethod
    def simple_map_function(x, y):
        return x + y

    @staticmethod
    def simple_reduce_function(results):
        total = 0
        for map_result in results:
            total = total + map_result
        return total

    @staticmethod
    def pywren_inside_pywren_map_function(x):
        def _func(x):
            return x

        pw = pywren.function_executor()
        pw.map(_func, range(x))
        return pw.get_result()

    @staticmethod
    def pywren_return_futures_map_function1(x):
        def _func(x):
            return x + 1

        pw = pywren.function_executor()
        return pw.map(_func, range(x))

    @staticmethod
    def pywren_return_futures_map_function2(x):
        def _func(x):
            return x + 1

        pw = pywren.function_executor()
        return pw.call_async(_func, x + 5)

    @staticmethod
    def pywren_return_futures_map_function3(x):
        def _func(x):
            return x + 1

        pw = pywren.function_executor()
        fut1 = pw.map(_func, range(x))
        fut2 = pw.map(_func, range(x))
        return fut1 + fut2

    @staticmethod
    def my_map_function_obj(obj, id):
        print('Bucket: {}'.format(obj.bucket))
        print('Key: {}'.format(obj.key))
        print('Partition num: {}'.format(obj.part))
        print('Action id: {}'.format(id))
        counter = {}
        data = obj.data_stream.read()
        for line in data.splitlines():
            for word in line.decode('utf-8').split():
                if word not in counter:
                    counter[word] = 1
                else:
                    counter[word] += 1
        return counter

    @staticmethod
    def my_map_function_url(url):
        print('I am processing the object from {}'.format(url.path))
        counter = {}
        data = url.data_stream.read()
        for line in data.splitlines():
            for word in line.decode('utf-8').split():
                if word not in counter:
                    counter[word] = 1
                else:
                    counter[word] += 1
        return counter

    @staticmethod
    def my_map_function_storage(key_i, bucket_name, storage):
        print('I am processing the object /{}/{}'.format(bucket_name, key_i))
        counter = {}
        data = storage.get_object(Bucket=bucket_name, Key=key_i)['Body'].read()
        for line in data.splitlines():
            for word in line.decode('utf-8').split():
                if word not in counter:
                    counter[word] = 1
                else:
                    counter[word] += 1
        return counter

    @staticmethod
    def my_reduce_function(results):
        final_result = 0
        for count in results:
            for word in count:
                final_result += count[word]
        return final_result

    @staticmethod
    def my_cloudobject_put(obj, storage):
        counter = TestMethods.my_map_function_obj(obj, 0)
        cloudobject = storage.put_cobject(pickle.dumps(counter))
        return cloudobject

    @staticmethod
    def my_cloudobject_get(results, storage):
        data = [pickle.loads(storage.get_cobject(cloudobject)) for cloudobject in results]
        return TestMethods.my_reduce_function(data)


class TestPywren(unittest.TestCase):
    cos_result_to_compare = None

    @classmethod
    def setUpClass(cls):
        print('Uploading test files...')
        cls.cos_result_to_compare = TestUtils.initTests()

    @classmethod
    def tearDownClass(cls):
        print('Deleting test files...')
        TestUtils.cleanTests()

    def test_call_async(self):
        print('Testing call_async()...')
        pw = pywren.function_executor(config=CONFIG)
        pw.call_async(TestMethods.hello_world, "")
        result = pw.get_result()
        self.assertEqual(result, "Hello World!")

        pw = pywren.function_executor(config=CONFIG)
        pw.call_async(TestMethods.concat, ["a", "b"])
        result = pw.get_result()
        self.assertEqual(result, "a b")

        pw = pywren.function_executor(config=CONFIG)
        pw.call_async(TestMethods.simple_map_function, [4, 6])
        result = pw.get_result()
        self.assertEqual(result, 10)

        pw = pywren.function_executor(config=CONFIG)
        pw.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8})
        result = pw.get_result()
        self.assertEqual(result, 10)

    def test_map(self):
        print('Testing map()...')
        iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
        pw = pywren.function_executor(config=CONFIG)
        pw.map(TestMethods.simple_map_function, iterdata)
        result = pw.get_result()
        self.assertEqual(result, [2, 4, 6, 8])

        pw = pywren.function_executor(config=CONFIG, workers=1)
        pw.map(TestMethods.simple_map_function, iterdata)
        result = pw.get_result()
        self.assertEqual(result, [2, 4, 6, 8])

        pw = pywren.function_executor(config=CONFIG)
        set_iterdata = set(range(2))
        pw.map(TestMethods.hello_world, set_iterdata)
        result = pw.get_result()
        self.assertEqual(result, ['Hello World!'] * 2)

        pw = pywren.function_executor(config=CONFIG)
        generator_iterdata = range(2)
        pw.map(TestMethods.hello_world, generator_iterdata)
        result = pw.get_result()
        self.assertEqual(result, ['Hello World!'] * 2)

        pw = pywren.function_executor(config=CONFIG)
        listDicts_iterdata = [{'x': 2, 'y': 8}, {'x': 2, 'y': 8}]
        pw.map(TestMethods.simple_map_function, listDicts_iterdata)
        result = pw.get_result()
        self.assertEqual(result, [10, 10])

        pw = pywren.function_executor(config=CONFIG)
        set_iterdata = [["a", "b"], ["c", "d"]]
        pw.map(TestMethods.concat, set_iterdata)
        result = pw.get_result()
        self.assertEqual(result, ["a b", "c d"])

    def test_map_reduce(self):
        print('Testing map_reduce()...')
        iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
        pw = pywren.function_executor(config=CONFIG)
        pw.map_reduce(TestMethods.simple_map_function, iterdata, TestMethods.simple_reduce_function)
        result = pw.get_result()
        self.assertEqual(result, 20)

    def test_multiple_executions(self):
        print('Testing multiple executions...')
        pw = pywren.function_executor(config=CONFIG)
        iterdata = [[1, 1], [2, 2]]
        pw.map(TestMethods.simple_map_function, iterdata)
        iterdata = [[3, 3], [4, 4]]
        pw.map(TestMethods.simple_map_function, iterdata)
        result = pw.get_result()
        self.assertEqual(result, [2, 4, 6, 8])

        iterdata = [[1, 1], [2, 2]]
        pw.map(TestMethods.simple_map_function, iterdata)
        result = pw.get_result()
        self.assertEqual(result, [2, 4])

        iterdata = [[1, 1], [2, 2]]
        futures1 = pw.map(TestMethods.simple_map_function, iterdata)
        result1 = pw.get_result(fs=futures1)
        iterdata = [[3, 3], [4, 4]]
        futures2 = pw.map(TestMethods.simple_map_function, iterdata)
        result2 = pw.get_result(fs=futures2)
        self.assertEqual(result1, [2, 4])
        self.assertEqual(result2, [6, 8])

    def test_internal_executions(self):
        print('Testing internal executions...')
        pw = pywren.function_executor(config=CONFIG)
        pw.map(TestMethods.pywren_inside_pywren_map_function, range(1, 11))
        result = pw.get_result()
        self.assertEqual(result, [list(range(i)) for i in range(1, 11)])

        pw = pywren.function_executor(config=CONFIG)
        pw.call_async(TestMethods.pywren_return_futures_map_function1, 3)
        pw.get_result()

        pw = pywren.function_executor(config=CONFIG)
        pw.call_async(TestMethods.pywren_return_futures_map_function2, 3)
        pw.get_result()

        pw = pywren.function_executor(config=CONFIG)
        pw.call_async(TestMethods.pywren_return_futures_map_function3, 3)
        pw.wait()
        pw.get_result()

    def test_map_reduce_obj_bucket(self):
        print('Testing map_reduce() over a bucket...')
        sb = STORAGE_CONFIG['backend']
        data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
        pw = pywren.function_executor(config=CONFIG)
        pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function)
        result = pw.get_result()
        self.assertEqual(result, self.__class__.cos_result_to_compare)

    def test_map_reduce_obj_bucket_one_reducer_per_object(self):
        print('Testing map_reduce() over a bucket with one reducer per object...')
        sb = STORAGE_CONFIG['backend']
        data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
        pw = pywren.function_executor(config=CONFIG)
        pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function,
                      reducer_one_per_object=True)
        result = pw.get_result()
        self.assertEqual(sum(result), self.__class__.cos_result_to_compare)

    def test_map_reduce_obj_key(self):
        print('Testing map_reduce() over object keys...')
        sb = STORAGE_CONFIG['backend']
        bucket_name = STORAGE_CONFIG['bucket']
        iterdata = [sb + '://' + bucket_name + '/' + key for key in TestUtils.list_test_keys()]
        pw = pywren.function_executor(config=CONFIG)
        pw.map_reduce(TestMethods.my_map_function_obj, iterdata, TestMethods.my_reduce_function)
        result = pw.get_result()
        self.assertEqual(result, self.__class__.cos_result_to_compare)

    def test_map_reduce_obj_key_one_reducer_per_object(self):
        print('Testing map_reduce() over object keys with one reducer per object...')
        sb = STORAGE_CONFIG['backend']
        bucket_name = STORAGE_CONFIG['bucket']
        iterdata = [sb + '://' + bucket_name + '/' + key for key in TestUtils.list_test_keys()]
        pw = pywren.function_executor(config=CONFIG)
        pw.map_reduce(TestMethods.my_map_function_obj, iterdata, TestMethods.my_reduce_function,
                      reducer_one_per_object=True)
        result = pw.get_result()
        self.assertEqual(sum(result), self.__class__.cos_result_to_compare)

    def test_map_reduce_url(self):
        print('Testing map_reduce() over URLs...')
        pw = pywren.function_executor(config=CONFIG)
        pw.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS, TestMethods.my_reduce_function)
        result = pw.get_result()
        self.assertEqual(result, self.__class__.cos_result_to_compare)

    def test_storage_handler(self):
        print('Testing "storage" function arg...')
        iterdata = [[key, STORAGE_CONFIG['bucket']] for key in TestUtils.list_test_keys()]
        pw = pywren.function_executor(config=CONFIG)
        pw.map_reduce(TestMethods.my_map_function_storage, iterdata, TestMethods.my_reduce_function)
        result = pw.get_result()
        self.assertEqual(result, self.__class__.cos_result_to_compare)

    def test_chunks_bucket(self):
        print('Testing chunks on a bucket...')
        sb = STORAGE_CONFIG['backend']
        data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'

        pw = pywren.function_executor(config=CONFIG)
        futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function,
                                chunk_size=1 * 1024 ** 2)
        result = pw.get_result(futures)
        self.assertEqual(result, self.__class__.cos_result_to_compare)
        self.assertEqual(len(futures), 8)

        pw = pywren.function_executor(config=CONFIG)
        futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_n=2)
        result = pw.get_result(futures)
        self.assertEqual(result, self.__class__.cos_result_to_compare)
        self.assertEqual(len(futures), 11)

    def test_chunks_bucket_one_reducer_per_object(self):
        print('Testing chunks on a bucket with one reducer per object...')
        sb = STORAGE_CONFIG['backend']
        data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'

        pw = pywren.function_executor(config=CONFIG)
        futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function,
                                chunk_size=1 * 1024 ** 2, reducer_one_per_object=True)
        result = pw.get_result(futures)
        self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
        self.assertEqual(len(futures), 12)

        pw = pywren.function_executor(config=CONFIG)
        futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_n=2,
                                reducer_one_per_object=True)
        result = pw.get_result(futures)
        self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
        self.assertEqual(len(futures), 15)

    def test_cloudobject(self):
        print('Testing cloudobjects...')
        sb = STORAGE_CONFIG['backend']
        data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
        with pywren.function_executor(config=CONFIG) as pw:
            pw.map_reduce(TestMethods.my_cloudobject_put, data_prefix, TestMethods.my_cloudobject_get)
            result = pw.get_result()
            self.assertEqual(result, self.__class__.cos_result_to_compare)


def print_help():
    print("Available test functions:")
    func_names = filter(lambda s: s[:4] == 'test',
                        map(lambda t: t[0], inspect.getmembers(TestPywren(), inspect.ismethod)))
    for func_name in func_names:
        print(f'-> {func_name}')


def run_tests(test_to_run, config=None):
    global CONFIG, STORAGE_CONFIG, STORAGE

    CONFIG = json.load(args.config) if config else default_config()
    STORAGE_CONFIG = extract_storage_config(CONFIG)
    STORAGE = InternalStorage(STORAGE_CONFIG).storage_handler

    suite = unittest.TestSuite()
    if test_to_run == 'all':
        suite.addTest(unittest.makeSuite(TestPywren))
    else:
        try:
            suite.addTest(TestPywren(test_to_run))
        except ValueError:
            print("unknown test, use: --help")
            sys.exit()

    runner = unittest.TextTestRunner()
    runner.run(suite)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="test all PyWren's functionality",
                                     usage='python -m pywren_ibm_cloud.tests [-c CONFIG] [-t TESTNAME]')
    parser.add_argument('-c', '--config', type=argparse.FileType('r'), metavar='', default=None,
                        help="use json config file")
    parser.add_argument('-t', '--test', metavar='', default='all',
                        help='run a specific test, type "-t help" for tests list')
    parser.add_argument('-d', '--debug', action='store_true', default=False,
                        help='activate debug logging')
    args = parser.parse_args()

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)

    if args.test == 'help':
        print_help()
    else:
        run_tests(args.test, args.config)
