# Check that the LNT REST JSON API is working.
# create temporary instance
# RUN: rm -rf %t.instance
# RUN: %{utils}/with_postgres.sh %t.pg.log \
# RUN:     %{utils}/with_temporary_instance.py %t.instance \
# RUN:         %{shared_inputs}/base-reports \
# RUN:         %{shared_inputs}/extra-reports \
# RUN:         -- python %s %t.instance

import os
import sys

TESTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
UI_DIR = os.path.join(TESTS_DIR, 'ui')
sys.path.insert(0, UI_DIR)

from V4Pages import check_json
import lnt.server.db.migrate
import lnt.server.ui.app
import logging
import sys
import unittest
import yaml

logging.basicConfig(level=logging.DEBUG)

machines_expected_properties = [{u'hardware': u'x86_64',
                                 u'os': u'Darwin 11.3.0',
                                 u'name': u'localhost__clang_DEV__x86_64',
                                 u'hostname': u'localhost',
                                 u'uname': u'Darwin localhost 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12'
                                           u' 18:47:41 PST 2012; root:xnu-1699.24.23~1/RELEASE_X86_64 x86_64'},
                                {u'hardware': u'AArch64',
                                 u'os': u'linux',
                                 u'name': u'machine2'},
                                {u'hardware': u'AArch64',
                                 u'os': u'linux',
                                 u'name': u'machine3'}]

possible_run_keys = {
    u'ARCH',
    u'CC_UNDER_TEST_IS_CLANG',
    u'CC_UNDER_TEST_TARGET_IS_X86_64',
    u'DISABLE_CBE',
    u'DISABLE_JIT',
    u'ENABLE_HASHED_PROGRAM_OUTPUT',
    u'ENABLE_OPTIMIZED',
    u'LLC_OPTFLAGS',
    u'LLI_OPTFLAGS',
    u'OPTFLAGS',
    u'TARGET_CC',
    u'TARGET_CXX',
    u'TARGET_FLAGS',
    u'TARGET_LLVMGCC',
    u'TARGET_LLVMGXX',
    u'TEST',
    u'USE_REFERENCE_OUTPUT',
    u'__report_version__',
    u'cc1_exec_hash',
    u'cc_alt_src_branch',
    u'cc_alt_src_revision',
    u'cc_as_version',
    u'cc_build',
    u'cc_exec_hash',
    u'cc_ld_version',
    u'cc_name',
    u'cc_src_branch',
    u'cc_src_revision',
    u'cc_target',
    u'cc_version',
    u'cc_version_number',
    u'end_time',
    u'id',
    u'inferred_run_order',
    u'llvm_project_revision',
    u'no_errors',
    u'order_by',
    u'order_id',
    u'start_time',
    u'sw_vers',
    u'test_suite_revision',
}

possible_machine_keys = {u'name',
                         u'hostname',
                         u'hardware',
                         u'os',
                         u'id',
                         u'uname'}


def _strip_id(d):
    """Return a copy of dict d with the 'id' key removed."""
    return {k: v for k, v in d.items() if k != 'id'}


class JSONAPITester(unittest.TestCase):
    """Test the REST api."""

    def setUp(self):
        """Bind to the LNT test instance."""
        _, instance_path = sys.argv
        self.instance_path = instance_path
        app = lnt.server.ui.app.App.create_standalone(instance_path)
        app.testing = True
        self.client = app.test_client()

        # Build ID maps for dynamic lookups.
        client = self.client
        j = check_json(client, 'api/db_default/v4/nts/machines/')
        self.machines_by_name = {m['name']: m for m in j['machines']}
        self.machine1 = self.machines_by_name['localhost__clang_DEV__x86_64']
        self.machine2 = self.machines_by_name['machine2']

        # Get runs for machine1.
        m1_data = check_json(
            client, f'api/db_default/v4/nts/machines/{self.machine1["id"]}')
        self.m1_runs = m1_data['runs']
        self.m1_run1_id = self.m1_runs[0]['id']

        # Get run detail for first run (to extract order_id, etc).
        self.m1_run1_detail = check_json(
            client, f'api/db_default/v4/nts/runs/{self.m1_run1_id}')

        # Get tests by name.
        tests_j = check_json(client, 'api/db_default/v4/nts/tests')
        self.tests_by_name = {t['name']: t for t in tests_j['tests']}

    def _check_response_is_well_formed(self, response):
        """API Should always return the generated by field in the top level dict."""
        # All API calls should return a top level dict.
        self.assertEqual(type(response), dict)

        # There should be no unexpected top level keys.
        all_top_level_keys = {'generated_by', 'machine', 'machines', 'runs', 'run', 'orders', 'tests', 'samples'}
        keys = set(response.keys())
        self.assertTrue(keys.issubset(all_top_level_keys),
                        "{} not subset of {}".format(keys, all_top_level_keys))
        # All API calls should return as generated by.
        self.assertIn("LNT Server v", response['generated_by'])

    def test_machine_api(self):
        """Check /machines/ and /machines/n return expected results from testdb.
        """
        client = self.client

        # All machines returns the list of machines with parameters, but no runs.
        j = check_json(client, 'api/db_default/v4/nts/machines/')
        self._check_response_is_well_formed(j)
        actual = [_strip_id(m) for m in j['machines']]
        self.assertEqual(actual, machines_expected_properties)
        self.assertIsNone(j.get('runs'))

        j = check_json(client, 'api/db_default/v4/nts/machines')
        self._check_response_is_well_formed(j)
        actual = [_strip_id(m) for m in j['machines']]
        self.assertEqual(actual, machines_expected_properties)
        self.assertIsNone(j.get('runs'))

        # Machine + properties + run information.
        j = check_json(client, f'api/db_default/v4/nts/machines/{self.machine1["id"]}')
        self._check_response_is_well_formed(j)
        self.assertEqual(_strip_id(j['machine']),
                         machines_expected_properties[0])

        self.assertEqual(len(j['runs']), 2)
        for run in j['runs']:
            self.assertTrue(set(run.keys()).issubset(possible_run_keys),
                            f"Unexpected keys: {set(run.keys()) - possible_run_keys}")

        # Specify machine by name
        j = check_json(client, 'api/db_default/v4/nts/machines/localhost__clang_DEV__x86_64')
        self._check_response_is_well_formed(j)
        self.assertEqual(_strip_id(j['machine']),
                         machines_expected_properties[0])

        # Invalid machine ids are 404.
        check_json(client, 'api/db_default/v4/nts/machines/99', expected_code=404)
        check_json(client, 'api/db_default/v4/nts/machines/foo', expected_code=404)

    def test_run_api(self):
        """Check /runs/n returns expected run information."""
        client = self.client
        j = check_json(
            client, f'api/db_default/v4/nts/runs/{self.m1_run1_id}')
        self._check_response_is_well_formed(j)
        expected = {"end_time": "2012-04-11T16:28:58",
                    "start_time": "2012-04-11T16:28:23",
                    "llvm_project_revision": u'154331'}
        self.assertEqual(j['run'], j['run'] | expected)
        self.assertEqual(len(j['tests']), 2)
        # This should not be a run.
        check_json(client, 'api/db_default/v4/nts/runs/100', expected_code=404)

    def test_order_api(self):
        """Check /orders/n returns the expected order information."""
        client = self.client
        order_id = self.m1_run1_detail['run']['order_id']
        j = check_json(
            client, f'api/db_default/v4/nts/orders/{order_id}')
        self.assertEqual(j['orders'][0]['llvm_project_revision'], u'154331')
        self._check_response_is_well_formed(j)
        check_json(client, 'api/db_default/v4/nts/orders/100', expected_code=404)

    def test_single_sample_api(self):
        """Check /samples/n returns the expected sample information."""
        client = self.client
        # Get first sample from first run.
        sample_id = self.m1_run1_detail['tests'][0]['id']
        j = check_json(
            client, f'api/db_default/v4/nts/samples/{sample_id}')
        self._check_response_is_well_formed(j)
        sample = j['samples'][0]
        self.assertAlmostEqual(sample['execution_time'], 0.0003)
        self.assertAlmostEqual(sample['compile_time'], 0.007)
        check_json(client, 'api/db_default/v4/nts/samples/1000', expected_code=404)

    def test_graph_api(self):
        """Check that /graph/x/y/z returns what we expect."""
        client = self.client

        # Graph for machine2, test1, field 2 (score).
        m2_id = self.machine2['id']
        test1_id = self.tests_by_name['test1']['id']

        j = check_json(
            client, f'api/db_default/v4/nts/graph/{m2_id}/{test1_id}/2')
        # Verify graph data: two points for revisions 152292 and 152293.
        self.assertEqual(len(j), 2)
        self.assertEqual(j[0][0], u'152292')
        self.assertEqual(j[0][1], 1.0)
        self.assertEqual(j[1][0], u'152293')
        self.assertEqual(j[1][1], 10.0)

        # Now check that limit works.
        j2 = check_json(
            client, f'api/db_default/v4/nts/graph/{m2_id}/{test1_id}/2?limit=1')
        self.assertEqual(len(j2), 1)
        self.assertEqual(j2[0][0], u'152293')
        self.assertEqual(j2[0][1], 10.0)

    def test_samples_api(self):
        """Samples API."""
        client = self.client
        run_id = self.m1_run1_id
        # Run IDs must be passed, so 400 if they are not.
        check_json(client, 'api/db_default/v4/nts/samples',
                   expected_code=400)

        # Simple single run.
        j = check_json(
            client, f'api/db_default/v4/nts/samples?runid={run_id}')
        self._check_response_is_well_formed(j)
        samples = j['samples']
        self.assertEqual(len(samples), 2)
        # Check sample properties.
        names = sorted(s['name'] for s in samples)
        self.assertEqual(names, [
            u'SingleSource/UnitTests/2006-12-01-float_varg',
            u'SingleSource/UnitTests/2006-12-04-DynAllocAndRestore'])
        for s in samples:
            self.assertEqual(s['run_id'], run_id)

        # Check that other args are ignored.
        extra_param = check_json(
            client, f'api/db_default/v4/nts/samples?runid={run_id}&foo=bar')
        self._check_response_is_well_formed(extra_param)
        self.assertEqual(j, extra_param)

        # The second run on this machine has no tests.
        run2_id = self.m1_runs[1]['id']
        two_runs = check_json(
            client,
            f'api/db_default/v4/nts/samples?runid={run_id}&runid={run2_id}')
        self._check_response_is_well_formed(two_runs)
        self.assertEqual(j, two_runs)

    def test_fields_api(self):
        """Fields API."""
        client = self.client
        j = check_json(client, 'api/db_default/v4/nts/fields')

        fields = j['fields']

        # check number of fields
        self.assertEqual(9, len(fields))

        # check first field
        f0 = fields[0]
        self.assertEqual(0, f0['column_id'])
        self.assertEqual('compile_time', f0['column_name'])
        self.assertEqual('FLOAT', f0['column_type'])

    def test_tests_api(self):
        """Tests API."""
        client = self.client
        j = check_json(client, 'api/db_default/v4/nts/tests')

        tests = j['tests']

        # check number of tests
        self.assertEqual(9, len(tests))

        # check a known test exists
        float_varg = self.tests_by_name['SingleSource/UnitTests/2006-12-01-float_varg']
        self.assertIsInstance(float_varg['id'], int)
        self.assertEqual('SingleSource/UnitTests/2006-12-01-float_varg',
                         float_varg['name'])

    def test_schema(self):
        client = self.client
        rest_schema = check_json(client, 'api/db_default/v4/nts/schema')

        # The reported schema should be well-formed and match the yaml file
        # that lnt create generated (which may differ slightly from the repo's
        # schemas/nts.yaml due to symlink timing).
        self.assertEqual(rest_schema['format_version'], '2')
        self.assertIn('metrics', rest_schema)
        self.assertIn('run_fields', rest_schema)
        self.assertIn('machine_fields', rest_schema)

        # Verify the schema matches the instance's yaml on disk.
        with open(f'{self.instance_path}/schemas/nts.yaml') as syaml:
            yaml_schema = yaml.safe_load(syaml)
            for m in yaml_schema['metrics']:
                if 'unit' not in m:
                    m['unit'] = None
                if 'unit_abbrev' not in m:
                    m['unit_abbrev'] = None
                if 'display_name' not in m:
                    m['display_name'] = m['name']
                if 'bigger_is_better' not in m:
                    m['bigger_is_better'] = False
                if 'ignore_same_hash' not in m:
                    m['ignore_same_hash'] = False
            yaml_schema['metrics'].sort(key=lambda x: x['name'])
            yaml_schema['run_fields'].sort(key=lambda x: x['name'])
            yaml_schema['machine_fields'].sort(key=lambda x: x['name'])
            yaml_schema.pop('name', None)

        # Compare individual sections rather than the whole dict, since
        # lnt create may produce a schema that differs from the symlinked YAML.
        self.assertEqual(rest_schema['format_version'],
                         yaml_schema['format_version'])
        self.assertEqual(len(rest_schema['metrics']),
                         len(yaml_schema['metrics']))
        self.assertEqual(len(rest_schema['run_fields']),
                         len(yaml_schema['run_fields']))
        self.assertEqual(len(rest_schema['machine_fields']),
                         len(yaml_schema['machine_fields']))


if __name__ == '__main__':
    unittest.main(argv=[sys.argv[0], ])
