"""
Tests that rely on a server running
"""
import base64
import json
import datetime
import os
from unittest import mock

import pytest
from omnisci import connect, ProgrammingError, DatabaseError
from omnisci.cursor import Cursor
from omnisci._parsers import Description, ColumnDetails
from omnisci.thrift.ttypes import TOmniSciException
from omnisci.common.ttypes import TDatumType

import geopandas as gpd
import pandas as pd
import numpy as np
import pyarrow as pa
from pandas.api.types import is_object_dtype, is_categorical_dtype
import pandas.testing as tm
import shapely
from shapely.geometry import Point, LineString, Polygon, MultiPolygon
import textwrap
from .conftest import no_gpu
from .data import dashboard_metadata

omniscihost = os.environ.get('OMNISCI_HOST', 'localhost')

# XXX: Make it hashable to silence warnings; see if this can be done upstream
# This isn't a huge deal, but our testing context mangers for asserting
# exceptions need hashability
TOmniSciException.__hash__ = lambda x: id(x)


def _cursor2df(cursor):
    col_types = {c.name: c.type_code for c in cursor.description}
    has_geodata = {
        k: v
        in [
            TDatumType.POINT,
            TDatumType.LINESTRING,
            TDatumType.POLYGON,
            TDatumType.MULTIPOLYGON,
        ]
        for k, v in col_types.items()
    }
    col_names = list(col_types.keys())

    df_class = gpd.GeoDataFrame if any(has_geodata.values()) else pd.DataFrame
    df = df_class(cursor.fetchall(), columns=col_names)

    for c, _has_geodata in has_geodata.items():
        if _has_geodata:
            df.loc[:, c] = df.loc[:, c].apply(shapely.wkt.loads)
    return df


@pytest.mark.usefixtures("mapd_server")
class TestIntegration:
    def test_connect_binary(self):
        con = connect(
            user="admin",
            password='HyperInteractive',
            host=omniscihost,
            port=6274,
            protocol='binary',
            dbname='omnisci',
        )
        assert con is not None

    def test_connect_http(self):
        con = connect(
            user="admin",
            password='HyperInteractive',
            host=omniscihost,
            port=6278,
            protocol='http',
            dbname='omnisci',
        )
        assert con is not None

    def test_connect_uri(self):
        uri = (
            'omnisci://admin:HyperInteractive@{0}:6274/omnisci?'
            'protocol=binary'.format(omniscihost)
        )
        con = connect(uri=uri)
        assert con._user == 'admin'
        assert con._password == 'HyperInteractive'
        assert con._host == omniscihost
        assert con._port == 6274
        assert con._dbname == 'omnisci'
        assert con._protocol == 'binary'

    def test_connect_uri_and_others_raises(self):
        uri = (
            'omnisci://admin:HyperInteractive@{0}:6274/omnisci?'
            'protocol=binary'.format(omniscihost)
        )
        with pytest.raises(TypeError):
            connect(username='omnisci', uri=uri)

    def test_invalid_sql(self, con):
        with pytest.raises(ProgrammingError) as r:
            con.cursor().execute("this is invalid;")
        return r.match("SQL Error:")

    def test_nonexistant_table(self, con):
        with pytest.raises(DatabaseError) as r:
            con.cursor().execute("select it from fake_table;")
        r.match("Table 'FAKE_TABLE' does not exist|Object 'fake_table' not")

    def test_connection_execute(self, con):
        result = con.execute("drop table if exists FOO;")
        result = con.execute("create table FOO (a int);")
        assert isinstance(result, Cursor)
        con.execute("drop table if exists FOO;")

    def test_select_sets_description(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        c.execute("select * from stocks")
        expected = [
            Description('date_', 6, None, None, None, None, True),
            Description('trans', 6, None, None, None, None, True),
            Description('symbol', 6, None, None, None, None, True),
            Description('qty', 1, None, None, None, None, True),
            Description('price', 3, None, None, None, None, True),
            Description('vol', 3, None, None, None, None, True),
        ]
        assert c.description == expected
        c.execute('drop table if exists stocks;')

    def test_select_parametrized(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        c.execute(
            'select symbol, qty from stocks where symbol = :symbol',
            {'symbol': 'GOOG'},
        )
        result = list(c)
        expected = [
            ('GOOG', 100),
        ]  # noqa
        assert result == expected
        c.execute('drop table if exists stocks;')

    def test_executemany_parametrized(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        parameters = [{'symbol': 'GOOG'}, {'symbol': "RHAT"}]
        expected = [[('GOOG', 100)], [('RHAT', 100)]]
        query = 'select symbol, qty from stocks where symbol = :symbol'
        c = con.cursor()
        result = c.executemany(query, parameters)
        assert result == expected
        c.execute('drop table if exists stocks;')

    def test_executemany_parametrized_insert(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        c = con.cursor()
        c.execute("drop table if exists stocks2;")
        # Create table
        c.execute('CREATE TABLE stocks2 (symbol text, qty int);')
        params = [{"symbol": "GOOG", "qty": 10}, {"symbol": "AAPL", "qty": 20}]
        query = "INSERT INTO stocks2 VALUES (:symbol, :qty);"
        result = c.executemany(query, params)
        assert result == [[], []]  # TODO: not sure if this is standard
        c.execute("drop table stocks2;")
        c.execute('drop table if exists stocks;')

    @pytest.mark.parametrize(
        'query, parameters',
        [
            ('select qty, price from stocks', None),
            ('select qty, price from stocks where qty=:qty', {'qty': 100}),
        ],
    )
    def test_select_ipc_parametrized(self, con, query, parameters):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        result = con.select_ipc(query, parameters=parameters)
        expected = pd.DataFrame(
            {
                "qty": np.array([100, 100], dtype=np.int32),
                "price": np.array(
                    [35.13999938964844, 12.140000343322754], dtype=np.float32
                ),
            }
        )[['qty', 'price']]
        tm.assert_frame_equal(result, expected)
        c.execute('drop table if exists stocks;')

    def test_select_ipc_first_n(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        result = con.select_ipc("select * from stocks", first_n=1)
        assert len(result) == 1
        c.execute('drop table if exists stocks;')

    @pytest.mark.parametrize(
        'query, parameters',
        [
            ('select qty, price from stocks', None),
            ('select qty, price from stocks where qty=:qty', {'qty': 100}),
        ],
    )
    @pytest.mark.skipif(no_gpu(), reason="No GPU available")
    def test_select_ipc_gpu(self, con, query, parameters):

        from cudf.core.dataframe import DataFrame

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        result = con.select_ipc_gpu("select qty, price from stocks")
        assert isinstance(result, DataFrame)

        dtypes = dict(qty=np.int32, price=np.float32)
        expected = pd.DataFrame(
            [[100, 35.14], [100, 12.14]], columns=['qty', 'price']
        ).astype(dtypes)

        result = result.to_pandas()[['qty', 'price']]  # column order
        pd.testing.assert_frame_equal(result, expected)
        c.execute('drop table if exists stocks;')

    @pytest.mark.skipif(no_gpu(), reason="No GPU available")
    def test_select_text_ipc_gpu(self, con):

        from cudf.core.dataframe import DataFrame

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)

        symbols = set(['GOOG', 'RHAT', 'IBM', 'NVDA'])
        for i, sym in enumerate(symbols):
            stmt = "INSERT INTO stocks VALUES ('2006-01-05_{}','BUY','{}',{},35.{},{}.1);".format(  # noqa
                i, sym, i, i, i
            )  # noqa
            # insert twice so we can test
            # that duplicated text values
            # are deserialized properly
            c.execute(stmt)
            c.execute(stmt)

        result = con.select_ipc_gpu(
            "select trans, symbol, qty, price from stocks"
        )  # noqa
        assert isinstance(result, DataFrame)

        assert len(result) == 8
        assert set(result['trans'].to_pandas()) == set(["BUY"])
        assert set(result['symbol'].to_pandas()) == symbols
        c.execute('drop table if exists stocks;')

    @pytest.mark.skipif(no_gpu(), reason="No GPU available")
    def test_select_gpu_first_n(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        result = con.select_ipc_gpu("select * from stocks", first_n=1)
        assert len(result) == 1
        c.execute('drop table if exists stocks;')

    def test_fetchone(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        c.execute("select symbol, qty from stocks")
        result = c.fetchone()
        expected = ('RHAT', 100)
        assert result == expected
        c.execute('drop table if exists stocks;')

    def test_fetchmany(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        c.execute("select symbol, qty from stocks")
        result = c.fetchmany()
        expected = [('RHAT', 100)]
        assert result == expected

        c.execute("select symbol, qty from stocks")
        result = c.fetchmany(size=10)
        expected = [('RHAT', 100), ('GOOG', 100)]
        assert result == expected
        c.execute('drop table if exists stocks;')

    def test_select_dates(self, con):

        c = con.cursor()
        c.execute('drop table if exists dates;')
        c.execute(
            'create table dates (date_ DATE, datetime_ TIMESTAMP, '
            'time_ TIME);'
        )
        i1 = (
            "INSERT INTO dates VALUES ('2006-01-05','2006-01-01T12:00:00',"
            "'12:00:00');"
        )
        i2 = (
            "INSERT INTO dates VALUES ('1901-12-14','1901-12-13T20:45:53',"
            "'23:59:00');"
        )
        c.execute(i1)
        c.execute(i2)

        result = list(c.execute("select * from dates"))
        expected = [
            (
                datetime.date(2006, 1, 5),
                datetime.datetime(2006, 1, 1, 12),
                datetime.time(12),
            ),
            (
                datetime.date(1901, 12, 14),
                datetime.datetime(1901, 12, 13, 20, 45, 53),
                datetime.time(23, 59),
            ),
        ]
        assert result == expected
        c.execute('drop table if exists dates;')


class TestOptionalImports:
    def test_select_gpu(self, con):
        with mock.patch.dict(
            "sys.modules", {"cudf": None, "cudf.core.dataframe": None}
        ):
            with pytest.raises(ImportError) as m:
                con.select_ipc_gpu("select * from foo;")
        assert m.match("The 'cudf' package is required")


class TestExtras:
    def test_get_tables(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float);'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1);"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2);"  # noqa

        c.execute(i1)
        c.execute(i2)

        result = con.get_tables()
        assert isinstance(result, list)
        assert 'stocks' in result
        c.execute('drop table if exists stocks;')

    def test_get_table_details(self, con):

        c = con.cursor()
        c.execute('drop table if exists stocks;')
        create = (
            'create table stocks (date_ text, trans text, symbol text, '
            'qty int, price float, vol float, '
            'exchanges TEXT [] ENCODING DICT(32));'
        )
        c.execute(create)
        i1 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14,1.1,{'NYSE', 'NASDAQ', 'AMEX'});"  # noqa
        i2 = "INSERT INTO stocks VALUES ('2006-01-05','BUY','GOOG',100,12.14,1.2,{'NYSE', 'NASDAQ'});"  # noqa

        c.execute(i1)
        c.execute(i2)

        result = con.get_table_details('stocks')
        expected = [
            ColumnDetails(
                name='date_',
                type='STR',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=32,
                encoding='DICT',
                is_array=False,
            ),
            ColumnDetails(
                name='trans',
                type='STR',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=32,
                encoding='DICT',
                is_array=False,
            ),
            ColumnDetails(
                name='symbol',
                type='STR',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=32,
                encoding='DICT',
                is_array=False,
            ),
            ColumnDetails(
                name='qty',
                type='INT',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=0,
                encoding='NONE',
                is_array=False,
            ),
            ColumnDetails(
                name='price',
                type='FLOAT',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=0,
                encoding='NONE',
                is_array=False,
            ),
            ColumnDetails(
                name='vol',
                type='FLOAT',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=0,
                encoding='NONE',
                is_array=False,
            ),
            ColumnDetails(
                name='exchanges',
                type='STR',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=32,
                encoding='DICT',
                is_array=True,
            ),
        ]
        assert result == expected
        c.execute('drop table if exists stocks;')


class TestLoaders:
    @staticmethod
    def check_empty_insert(result, expected):
        assert len(result) == 3
        assert expected[0][0] == result[0][0]
        assert expected[0][2] == result[0][2]
        assert abs(expected[0][1] - result[0][1]) < 1e-7  # floating point

    def test_load_empty_table(self, con):

        con.execute("drop table if exists baz;")
        con.execute("create table baz (a int, b float, c text);")

        data = [(1, 1.1, 'a'), (2, 2.2, '2'), (3, 3.3, '3')]
        con.load_table("baz", data)
        result = sorted(con.execute("select * from baz"))
        self.check_empty_insert(result, data)
        con.execute("drop table if exists baz;")

    def test_load_empty_table_pandas(self, con):

        con.execute("drop table if exists baz;")
        con.execute("create table baz (a int, b float, c text);")

        data = [(1, 1.1, 'a'), (2, 2.2, '2'), (3, 3.3, '3')]
        df = pd.DataFrame(data, columns=list('abc'))
        con.load_table("baz", df, method='columnar')
        result = sorted(con.execute("select * from baz"))
        self.check_empty_insert(result, data)
        con.execute("drop table if exists baz;")

    def test_load_empty_table_arrow(self, con):

        con.execute("drop table if exists baz;")
        con.execute("create table baz (a int, b float, c text);")

        data = [(1, 1.1, 'a'), (2, 2.2, '2'), (3, 3.3, '3')]

        df = pd.DataFrame(data, columns=list('abc')).astype(
            {'a': 'int32', 'b': 'float32'}
        )

        table = pa.Table.from_pandas(df, preserve_index=False)
        con.load_table("baz", table, method='arrow')
        result = sorted(con.execute("select * from baz"))
        self.check_empty_insert(result, data)
        con.execute("drop table if exists baz;")

    @pytest.mark.parametrize(
        'df, table_fields',
        [
            pytest.param(
                pd.DataFrame(
                    {
                        "a": [1, 2, 3],
                        "b": [1.1, 2.2, 3.3],
                        "c": ['a', '2', '3'],
                    },
                ),
                'a int, b float, c text',
                id='scalar_values',
            ),
            pytest.param(
                pd.DataFrame(
                    {
                        "a": [
                            np.datetime64('2010-01-01 01:01:01.001001001'),
                            np.datetime64('2011-01-01 01:01:01.001001001'),
                            np.datetime64('2012-01-01 01:01:01.001001001'),
                        ],
                    },
                ),
                'a TIMESTAMP(9)',
                id='scalar_datetime_nanoseconds',
            ),
            pytest.param(
                pd.DataFrame(
                    {
                        "a": [
                            datetime.datetime.fromtimestamp(
                                float(1600443582510) / 1e3
                            ),
                            datetime.datetime.fromtimestamp(
                                float(1600443582510) / 1e3
                            ),
                            datetime.datetime.fromtimestamp(
                                float(1600443582510) / 1e3
                            ),
                        ],
                    },
                ),
                'a TIMESTAMP(3)',
                id='scalar_datetime_ms',
            ),
            pytest.param(
                pd.DataFrame(
                    {
                        "a": [
                            datetime.datetime.fromtimestamp(
                                float(1600443582510) / 1e6
                            ),
                            datetime.datetime.fromtimestamp(
                                float(1600443582510) / 1e6
                            ),
                            datetime.datetime.fromtimestamp(
                                float(1600443582510) / 1e6
                            ),
                        ],
                    },
                ),
                'a TIMESTAMP(6)',
                id='scalar_datetime_us',
            ),
            pytest.param(
                pd.DataFrame(
                    [
                        {'ary': [2, 3, 4]},
                        {'ary': [4444]},
                        {'ary': []},
                        {'ary': None},
                        {'ary': [2, 3, 4]},
                    ]
                ),
                'ary INT[]',
                id='array_values',
            ),
            pytest.param(
                pd.DataFrame(
                    [
                        {'ary': [2, 3, 4], 'strtest': 'teststr'},
                        {'ary': None, 'strtest': 'teststr'},
                        {'ary': [4444], 'strtest': 'teststr'},
                        {'ary': [], 'strtest': 'teststr'},
                        {'ary': [2, 3, 4], 'strtest': 'teststr'},
                    ]
                ),
                'ary INT[], strtest TEXT',
                id='mix_scalar_array_values_with_none_and_empty_list',
            ),
            pytest.param(
                gpd.GeoDataFrame(
                    {
                        'a': [Point(0, 0), Point(1, 1)],
                        'b': [
                            LineString([(2, 0), (2, 4), (3, 4)]),
                            LineString([(0, 0), (1, 1)]),
                        ],
                        'c': [
                            Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]),
                            Polygon([(0, 0), (4, 0), (4, 4), (0, 4), (0, 0)]),
                        ],
                        'd': [
                            MultiPolygon(
                                [
                                    Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]),
                                    Polygon(
                                        [
                                            (0, 0),
                                            (4, 0),
                                            (4, 4),
                                            (0, 4),
                                            (0, 0),
                                        ]
                                    ),
                                ]
                            ),
                            MultiPolygon(
                                [
                                    Polygon(
                                        [
                                            (0, 0),
                                            (4, 0),
                                            (4, 4),
                                            (0, 4),
                                            (0, 0),
                                        ]
                                    ),
                                    Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]),
                                ]
                            ),
                        ],
                    }
                ),
                'a POINT, b LINESTRING, c POLYGON, d MULTIPOLYGON',
                id='geo_values',
            ),
        ],
    )
    def test_load_table_columnar(self, con, tmp_table, df, table_fields):
        con.execute("create table {} ({});".format(tmp_table, table_fields))
        con.load_table_columnar(tmp_table, df)
        result = _cursor2df(con.execute('select * from {}'.format(tmp_table)))
        pd.testing.assert_frame_equal(df, result)

    def test_load_infer(self, con):

        con.execute("drop table if exists baz;")
        con.execute("create table baz (a int, b float, c text);")

        data = pd.DataFrame(
            {
                'a': np.array([0, 1], dtype=np.int32),
                'b': np.array([1.1, 2.2], dtype=np.float32),
                'c': ['a', 'b'],
            }
        )
        con.load_table("baz", data)
        con.execute("drop table if exists baz;")

    def test_load_infer_bad(self, con):

        con.execute("drop table if exists baz;")
        con.execute("create table baz (a int, b float, c text);")

        with pytest.raises(TypeError):
            con.load_table("baz", [], method='thing')

        con.execute("drop table if exists baz;")

    def test_infer_non_pandas(self, con):

        con.execute("drop table if exists baz;")
        con.execute("create table baz (a int, b float, c text);")

        with pytest.raises(TypeError):
            con.load_table("baz", [], method='columnar')

        con.execute("drop table if exists baz;")

    def test_load_columnar_pandas_all(self, con):

        c = con.cursor()
        c.execute('drop table if exists all_types;')
        create = textwrap.dedent(
            '''\
        create table all_types (
            boolean_ BOOLEAN,
            smallint_ SMALLINT,
            int_ INT,
            bigint_ BIGINT,
            float_ FLOAT,
            double_ DOUBLE,
            varchar_ VARCHAR(40),
            text_ TEXT,
            time_ TIME,
            timestamp_ TIMESTAMP,
            date_ DATE
        );'''
        )
        # skipping decimal for now
        c.execute(create)

        data = pd.DataFrame(
            {
                "boolean_": [True, False, True, False],
                "smallint_": np.array([0, 1, 0, 1], dtype=np.int16),
                "int_": np.array([0, 1, 0, 1], dtype=np.int32),
                "bigint_": np.array([0, 1, 0, 1], dtype=np.int64),
                "float_": np.array([0, 1, 0, 1], dtype=np.float32),
                "double_": np.array([0, 1, 0, 1], dtype=np.float64),
                "varchar_": ["a", "b", "a", "b"],
                "text_": ['a', 'b', 'a', 'b'],
                "time_": [
                    datetime.time(0, 11, 59),
                    datetime.time(13),
                    datetime.time(22, 58, 59),
                    datetime.time(7, 13, 43),
                ],
                "timestamp_": [
                    pd.Timestamp("2016"),
                    pd.Timestamp("2017"),
                    pd.Timestamp(
                        '2017-11-28 23:55:59.342380', tz='US/Eastern'
                    ),
                    pd.Timestamp(
                        '2018-11-28 23:55:59.342380', tz='Asia/Calcutta'
                    ),
                ],
                "date_": [
                    datetime.date(2016, 1, 1),
                    datetime.date(2017, 1, 1),
                    datetime.date(2017, 11, 28),
                    datetime.date(2018, 11, 28),
                ],
            },
            columns=[
                'boolean_',
                'smallint_',
                'int_',
                'bigint_',
                'float_',
                'double_',
                'varchar_',
                'text_',
                'time_',
                'timestamp_',
                'date_',
            ],
        )
        con.load_table_columnar("all_types", data, preserve_index=False)

        result = list(c.execute("select * from all_types"))
        expected = [
            (
                1,
                0,
                0,
                0,
                0.0,
                0.0,
                'a',
                'a',
                datetime.time(0, 11, 59),
                datetime.datetime(2016, 1, 1, 0, 0),
                datetime.date(2016, 1, 1),
            ),
            (
                0,
                1,
                1,
                1,
                1.0,
                1.0,
                'b',
                'b',
                datetime.time(13, 0),
                datetime.datetime(2017, 1, 1, 0, 0),
                datetime.date(2017, 1, 1),
            ),
            (
                1,
                0,
                0,
                0,
                0.0,
                0.0,
                'a',
                'a',
                datetime.time(22, 58, 59),
                datetime.datetime(2017, 11, 29, 4, 55, 59),
                datetime.date(2017, 11, 28),
            ),
            (
                0,
                1,
                1,
                1,
                1.0,
                1.0,
                'b',
                'b',
                datetime.time(7, 13, 43),
                datetime.datetime(2018, 11, 28, 18, 25, 59),
                datetime.date(2018, 11, 28),
            ),
        ]

        assert result == expected
        c.execute('drop table if exists all_types;')

    def test_load_table_columnar_arrow_all(self, con):

        c = con.cursor()
        c.execute('drop table if exists all_types;')
        create = textwrap.dedent(
            '''\
        create table all_types (
            boolean_ BOOLEAN,
            smallint_ SMALLINT,
            int_ INT,
            bigint_ BIGINT,
            float_ FLOAT,
            double_ DOUBLE,
            varchar_ VARCHAR(40),
            text_ TEXT,
            time_ TIME,
            timestamp_ TIMESTAMP,
            date_ DATE
        );'''
        )
        # skipping decimal for now
        c.execute(create)

        names = [
            'boolean_',
            'smallint_',
            'int_',
            'bigint_',
            'float_',
            'double_',
            'varchar_',
            'text_',
            'time_',
            'timestamp_',
            'date_',
        ]

        columns = [
            pa.array([True, False, None], type=pa.bool_()),
            pa.array([1, 0, None]).cast(pa.int16()),
            pa.array([1, 0, None]).cast(pa.int32()),
            pa.array([1, 0, None]),
            pa.array([1.0, 1.1, None]).cast(pa.float32()),
            pa.array([1.0, 1.1, None]),
            # no fixed-width string
            pa.array(['a', 'b', None]),
            pa.array(['a', 'b', None]),
            (pa.array([1, 2, None]).cast(pa.int32()).cast(pa.time32('s'))),
            pa.array(
                [
                    datetime.datetime(2016, 1, 1, 12, 12, 12),
                    datetime.datetime(2017, 1, 1),
                    None,
                ]
            ),
            pa.array(
                [datetime.date(2016, 1, 1), datetime.date(2017, 1, 1), None]
            ),
        ]
        table = pa.Table.from_arrays(columns, names=names)
        con.load_table_arrow("all_types", table)
        c.execute('drop table if exists all_types;')

    def test_select_null(self, con):
        con.execute("drop table if exists pymapd_test_table;")
        con.execute("create table pymapd_test_table (a int);")
        con.execute("insert into pymapd_test_table VALUES (1);")
        con.execute("insert into pymapd_test_table VALUES (null);")
        # the test

        c = con.cursor()
        result = c.execute("select * from pymapd_test_table")
        expected = [(1,), (None,)]
        assert result.fetchall() == expected

        # cleanup
        con.execute("drop table if exists pymapd_test_table;")

    @pytest.mark.parametrize(
        'df, expected',
        [
            (
                pd.DataFrame(
                    {
                        "a": [1, 2],
                        "b": [1.0, 2.0],
                        "c": [
                            datetime.date(2016, 1, 1),
                            datetime.date(2017, 1, 1),
                        ],
                        "d": [
                            np.datetime64("2010-01-01T01:01:01.001001001"),
                            np.datetime64("2011-01-01T01:01:01.001001001"),
                        ],
                    }
                ),
                {
                    'a': {'type': 'BIGINT', 'is_array': False},
                    'b': {'type': 'DOUBLE', 'is_array': False},
                    'c': {'type': 'DATE', 'is_array': False},
                    'd': {
                        'type': 'TIMESTAMP',
                        'is_array': False,
                        'precision': 9,
                    },
                },
            ),
            (
                pd.DataFrame(
                    {
                        'a': [[1, 2], [1, 2], None, []],
                        'b': ['A', 'B', 'C', 'D'],
                        'c': [[1.0, 2.2], [1.0, 2.2], [], None],
                        'd': [
                            [
                                9007199254740991,
                                9007199254740992,
                                9007199254740993,
                            ],
                            [],
                            None,
                            [
                                9007199254740994,
                                9007199254740995,
                                9007199254740996,
                            ],
                        ],
                    }
                ),
                {
                    'a': {'type': 'BIGINT', 'is_array': True},
                    'b': {'type': 'STR', 'is_array': False},
                    'c': {'type': 'DOUBLE', 'is_array': True},
                    'd': {'type': 'BIGINT', 'is_array': True},
                },
            ),
            (
                gpd.GeoDataFrame(
                    {
                        'a': [Point(0, 0), Point(1, 1)],
                        'b': [
                            LineString([(2, 0), (2, 4), (3, 4)]),
                            LineString([(0, 0), (1, 1)]),
                        ],
                        'c': [
                            Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]),
                            Polygon([(0, 0), (4, 0), (4, 4), (0, 4), (0, 0)]),
                        ],
                        'd': [
                            MultiPolygon(
                                [
                                    Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]),
                                    Polygon(
                                        [
                                            (0, 0),
                                            (4, 0),
                                            (4, 4),
                                            (0, 4),
                                            (0, 0),
                                        ]
                                    ),
                                ]
                            ),
                            MultiPolygon(
                                [
                                    Polygon(
                                        [
                                            (0, 0),
                                            (4, 0),
                                            (4, 4),
                                            (0, 4),
                                            (0, 0),
                                        ]
                                    ),
                                    Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]),
                                ]
                            ),
                        ],
                    }
                ),
                {
                    'a': {'type': 'POINT', 'is_array': True},
                    'b': {'type': 'LINESTRING', 'is_array': True},
                    'c': {'type': 'POLYGON', 'is_array': True},
                    'd': {'type': 'MULTIPOLYGON', 'is_array': True},
                },
            ),
        ],
    )
    def test_create_table(self, con, tmp_table, df, expected):
        con.create_table(tmp_table, df)
        for col in con.get_table_details(tmp_table):
            assert expected[col.name]['type'] == col.type
            if 'precision' in expected[col.name]:
                assert expected[col.name]['precision'] == col.precision

    def test_load_table_creates(self, con):

        data = pd.DataFrame(
            {
                "boolean_": [True, False],
                "smallint_cast": np.array([0, 1], dtype=np.int8),
                "smallint_": np.array([0, 1], dtype=np.int16),
                "int_": np.array([0, 1], dtype=np.int32),
                "bigint_": np.array([0, 1], dtype=np.int64),
                "float_": np.array([0, 1], dtype=np.float32),
                "double_": np.array([0, 1], dtype=np.float64),
                "varchar_": ["a", "b"],
                "text_": ['a', 'b'],
                "time_": [datetime.time(0, 11, 59), datetime.time(13)],
                "timestamp1_": [pd.Timestamp("2016"), pd.Timestamp("2017")],
                "timestamp2_": [
                    np.datetime64("2010-01-01T01:01:01.001001001"),
                    np.datetime64("2011-01-01T01:01:01.001001001"),
                ],
                "date_": [
                    datetime.date(2016, 1, 1),
                    datetime.date(2017, 1, 1),
                ],
            },
            columns=[
                'boolean_',
                'smallint_',
                'int_',
                'bigint_',
                'float_',
                'double_',
                'varchar_',
                'text_',
                'time_',
                'timestamp1_',
                'timestamp2_',
                'date_',
            ],
        )

        con.execute("drop table if exists test_load_table_creates;")
        con.load_table("test_load_table_creates", data, create=True)
        con.execute("drop table if exists test_load_table_creates;")

    def test_array_in_result_set(self, con):

        # text
        con.execute("DROP TABLE IF EXISTS test_lists;")
        con.execute(
            "CREATE TABLE IF NOT EXISTS test_lists \
                    (col1 TEXT, col2 TEXT[]);"
        )

        row = [
            ("row1", "{hello,goodbye,aloha}"),
            ("row2", "{hello2,goodbye2,aloha2}"),
        ]

        con.load_table_rowwise("test_lists", row)
        ans = con.execute("select * from test_lists").fetchall()

        expected = [
            ('row1', ['hello', 'goodbye', 'aloha']),
            ('row2', ['hello2', 'goodbye2', 'aloha2']),
        ]

        assert ans == expected

        # int
        con.execute("DROP TABLE IF EXISTS test_lists;")
        con.execute(
            "CREATE TABLE IF NOT EXISTS test_lists \
                    (col1 TEXT, col2 INT[]);"
        )

        row = [("row1", "{10,20,30}"), ("row2", "{40,50,60}")]

        con.load_table_rowwise("test_lists", row)
        ans = con.execute("select * from test_lists").fetchall()

        expected = [('row1', [10, 20, 30]), ('row2', [40, 50, 60])]

        assert ans == expected

        # timestamp
        con.execute("DROP TABLE IF EXISTS test_lists;")
        con.execute(
            "CREATE TABLE IF NOT EXISTS test_lists \
                    (col1 TEXT, col2 TIMESTAMP[], col3 TIMESTAMP(9));"
        )

        row = [
            (
                "row1",
                "{2019-03-02 00:00:00,2019-03-02 00:00:00,2019-03-02 00:00:00}",  # noqa
                "2010-01-01T01:01:01.001001001",
            ),
            (
                "row2",
                "{2019-03-02 00:00:00,2019-03-02 00:00:00,2019-03-02 00:00:00}",  # noqa
                "2011-01-01T01:01:01.001001001",
            ),
        ]

        con.load_table_rowwise("test_lists", row)
        ans = con.execute("select * from test_lists").fetchall()

        expected = [
            (
                'row1',
                [
                    datetime.datetime(2019, 3, 2, 0, 0),
                    datetime.datetime(2019, 3, 2, 0, 0),
                    datetime.datetime(2019, 3, 2, 0, 0),
                ],
                np.datetime64("2010-01-01T01:01:01.001001001"),
            ),
            (
                'row2',
                [
                    datetime.datetime(2019, 3, 2, 0, 0),
                    datetime.datetime(2019, 3, 2, 0, 0),
                    datetime.datetime(2019, 3, 2, 0, 0),
                ],
                np.datetime64("2011-01-01T01:01:01.001001001"),
            ),
        ]
        assert ans == expected

        # date
        con.execute("DROP TABLE IF EXISTS test_lists;")
        con.execute(
            "CREATE TABLE IF NOT EXISTS test_lists \
                    (col1 TEXT, col2 DATE[]);"
        )

        row = [
            ("row1", "{2019-03-02,2019-03-02,2019-03-02}"),
            ("row2", "{2019-03-02,2019-03-02,2019-03-02}"),
        ]

        con.load_table_rowwise("test_lists", row)
        ans = con.execute("select * from test_lists").fetchall()

        expected = [
            (
                'row1',
                [
                    datetime.date(2019, 3, 2),
                    datetime.date(2019, 3, 2),
                    datetime.date(2019, 3, 2),
                ],
            ),
            (
                'row2',
                [
                    datetime.date(2019, 3, 2),
                    datetime.date(2019, 3, 2),
                    datetime.date(2019, 3, 2),
                ],
            ),
        ]

        assert ans == expected

        # time
        con.execute("DROP TABLE IF EXISTS test_lists;")
        con.execute(
            "CREATE TABLE IF NOT EXISTS test_lists \
                    (col1 TEXT, col2 TIME[]);"
        )

        row = [
            ("row1", "{23:59:00,23:59:00,23:59:00}"),
            ("row2", "{23:59:00,23:59:00,23:59:00}"),
        ]

        con.load_table_rowwise("test_lists", row)
        ans = con.execute("select * from test_lists").fetchall()

        expected = [
            (
                'row1',
                [
                    datetime.time(23, 59),
                    datetime.time(23, 59),
                    datetime.time(23, 59),
                ],
            ),
            (
                'row2',
                [
                    datetime.time(23, 59),
                    datetime.time(23, 59),
                    datetime.time(23, 59),
                ],
            ),
        ]

        assert ans == expected
        con.execute("DROP TABLE IF EXISTS test_lists;")

    def test_upload_pandas_categorical_ipc(self, con):

        con.execute("DROP TABLE IF EXISTS test_categorical;")

        df = pd.DataFrame({"A": ["a", "b", "c", "a"]})
        df["B"] = df["A"].astype('category')

        # test that table created correctly when it doesn't exist on server
        con.load_table("test_categorical", df)
        ans = con.execute("select * from test_categorical").fetchall()

        assert ans == [('a', 'a'), ('b', 'b'), ('c', 'c'), ('a', 'a')]

        assert con.get_table_details("test_categorical") == [
            ColumnDetails(
                name='A',
                type='STR',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=32,
                encoding='DICT',
                is_array=False,
            ),
            ColumnDetails(
                name='B',
                type='STR',
                nullable=True,
                precision=0,
                scale=0,
                comp_param=32,
                encoding='DICT',
                is_array=False,
            ),
        ]

        # load row-wise
        con.load_table("test_categorical", df, method="rows")

        # load columnar
        con.load_table("test_categorical", df, method="columnar")

        # load arrow
        con.load_table("test_categorical", df, method="arrow")

        # test end result
        df_ipc = con.select_ipc("select * from test_categorical")
        assert df_ipc.shape == (16, 2)

        res = df.append([df, df, df]).reset_index(drop=True)
        res["A"] = res["A"].astype('category')
        res["B"] = res["B"].astype('category')
        assert pd.DataFrame.equals(df_ipc, res)

        # test that input df wasn't mutated
        # original input is object, categorical
        # to load via Arrow, converted internally to object, object
        assert is_object_dtype(df["A"])
        assert is_categorical_dtype(df["B"])
        con.execute("DROP TABLE IF EXISTS test_categorical;")

    def test_insert_unicode(self, con):

        """INSERT Unicode using bind_params"""

        c = con.cursor()
        c.execute('drop table if exists text_holder;')
        create = 'create table text_holder (the_text text);'
        c.execute(create)
        first = {"value": "我和我的姐姐吃米饭鸡肉"}
        second = {"value": "El camina a case en bicicleta es relajante"}

        i1 = "INSERT INTO text_holder VALUES ( :value );"

        c.execute(i1, parameters=first)
        c.execute(i1, parameters=second)

        c.execute('drop table if exists text_holder;')

    def test_execute_leading_space_and_params(self, con):

        # https://github.com/omnisci/pymapd/issues/263

        """Ensure that leading/trailing spaces in execute statements
        don't cause issues
        """

        c = con.cursor()
        c.execute('drop table if exists test_leading_spaces;')
        create = 'create table test_leading_spaces (the_text text);'
        c.execute(create)
        first = {"value": "我和我的姐姐吃米饭鸡肉"}
        second = {"value": "El camina a case en bicicleta es relajante"}

        i1 = """

                    INSERT INTO test_leading_spaces


                    VALUES ( :value );

                            """

        c.execute(i1, parameters=first)
        c.execute(i1, parameters=second)

        c.execute('drop table if exists test_leading_spaces;')

    def test_dashboard_duplication_remap(self, con):
        # This test relies on the test_data_no_nulls_ipc table
        for dashver in [2, 3]:
            # Setup our testing variables
            old_dashboard_state = getattr(
                dashboard_metadata, 'old_dashboard_state_v{0}'.format(dashver)
            )
            old_dashboard_name = dashboard_metadata.old_dashboard_name
            new_dashboard_name = "new_test_v{0}".format(dashver)
            meta_data = {
                "table": "test_data_no_nulls_ipc",
                "version": "v{0}".format(dashver),
            }
            remap = {
                "test_data_no_nulls_ipc": {
                    "name": new_dashboard_name,
                    "title": new_dashboard_name,
                }
            }
            dashboards = []

            # Create testing dashboard
            try:
                dashboard_id = con._client.create_dashboard(
                    session=con._session,
                    dashboard_name=old_dashboard_name,
                    dashboard_state=(
                        base64.b64encode(
                            json.dumps(old_dashboard_state).encode("utf-8")
                        )
                    ),
                    image_hash="",
                    dashboard_metadata=json.dumps(meta_data),
                )
            except TOmniSciException:
                dashboards = con._client.get_dashboards(con._session)
                for dash in dashboards:
                    if dash.dashboard_name == old_dashboard_name:
                        con._client.delete_dashboard(
                            con._session, dash.dashboard_id
                        )
                        break
                dashboard_id = con._client.create_dashboard(
                    session=con._session,
                    dashboard_name=old_dashboard_name,
                    dashboard_state=(
                        base64.b64encode(
                            json.dumps(old_dashboard_state).encode("utf-8")
                        )
                    ),
                    image_hash="",
                    dashboard_metadata=json.dumps(meta_data),
                )

            # Duplicate and remap our dashboard
            try:
                dashboard_id = con.duplicate_dashboard(
                    dashboard_id, new_dashboard_name, remap
                )
            except TOmniSciException:
                dashboards = con._client.get_dashboards(con._session)
                for dash in dashboards:
                    if dash.dashboard_name == new_dashboard_name:
                        con._client.delete_dashboard(
                            con._session, dash.dashboard_id
                        )
                        break
                dashboard_id = con.duplicate_dashboard(
                    dashboard_id, new_dashboard_name, remap
                )

            # Get our new dashboard from the database
            d = con.get_dashboard(dashboard_id=dashboard_id)
            remapped_dashboard = json.loads(
                base64.b64decode(d.dashboard_state).decode()
            )

            # Assert that the table and title changed
            if dashver == 2:
                assert (
                    remapped_dashboard['dashboard']['title']
                    == new_dashboard_name
                )
                # Ensure the datasources change
                for key, val in remapped_dashboard['dashboard'][
                    'dataSources'
                ].items():
                    for col in val['columnMetadata']:
                        assert col['table'] == new_dashboard_name
            else:
                assert (
                    remapped_dashboard['tabs']['-MGoY-5OxdW3ANABL9PK'][
                        'dashboard'
                    ]['title']
                    == new_dashboard_name
                )
                # Ensure the datasources change
                for key, val in remapped_dashboard['tabs'][
                    '-MGoY-5OxdW3ANABL9PK'
                ]['dashboard']['dataSources'].items():
                    for col in val['columnMetadata']:
                        assert col['table'] == new_dashboard_name
