import testing.postgresql
import numpy as np
import asyncpg
import datetime
import itertools
import tempfile
import shutil
import os
import sys
import unittest
import asyncio

from joule.models import DataStream, Element, pipes
from joule.models.data_store import psql_helpers
from joule.models.data_store.timescale import TimescaleStore
from tests import helpers
from tests.models.pipes.reader import QueueReader

SQL_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'src', 'joule', 'sql')


class TestTimescale(unittest.IsolatedAsyncioTestCase):
    use_default_loop = False
    forbid_get_event_loop = True

    async def asyncSetUp(self):
        # set up the pscql database
        self.psql_dir = tempfile.TemporaryDirectory()
        # run this asynchronously to avoid blocking the event loop
        def start_postgresql_instance():
            return testing.postgresql.Postgresql(base_dir=self.psql_dir.name)
        self.postgresql = await asyncio.to_thread(start_postgresql_instance)
        await asyncio.to_thread(self.postgresql.stop)

        # now that the directory structure is created, customize the *.conf file
        src = os.path.join(os.path.dirname(__file__), "postgresql.conf")
        dest = os.path.join(self.psql_dir.name, "data", "postgresql.conf")
        shutil.copyfile(src, dest)
        # restart the database
        self.postgresql = await asyncio.to_thread(start_postgresql_instance)

        self.db_url = self.postgresql.url()
        conn: asyncpg.Connection = await asyncpg.connect(self.db_url)
        await conn.execute("CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE ")
        await conn.execute("CREATE USER joule_module")
        # load custom functions
        for file in os.listdir(SQL_DIR):
            file_path = os.path.join(SQL_DIR, file)
            with open(file_path, 'r') as f:
                await conn.execute(f.read())
        await conn.close()

    async def asyncTearDown(self):
        await asyncio.to_thread(self.postgresql.stop)
        self.psql_dir.cleanup()

    async def test_runner(self):
        tests = [
            self._test_merge_gaps_on_write,
            self._test_basic_insert_extract,
            self._test_extract_data_with_intervals,
            self._test_extract_decimated_data,
            self._test_db_info,
            self._test_info,
            self._test_intervals,
            self._test_remove,
            self._test_destroy,
            self._test_row_count,
            self._test_actions_on_empty_streams,
            self._test_consolidate,
            self._test_consolidate_with_time_bounds,
            self._test_remove_decimations,
            self._test_redecimates_data]
        for test in tests:
            conn: asyncpg.Connection = await asyncpg.connect(self.db_url)
            await conn.execute("DROP SCHEMA IF EXISTS data CASCADE")
            await conn.execute("CREATE SCHEMA data")
            await conn.execute("GRANT ALL ON SCHEMA data TO public")

            self.store = TimescaleStore(self.db_url, 0, 60,)
            await self.store.initialize([])
            # make a sample stream with data
            self.test_stream = DataStream(id=100, name="stream1", datatype=DataStream.DATATYPE.FLOAT32,
                                          keep_us=DataStream.KEEP_ALL, decimate=True,
                                          elements=[Element(name="e%d" % x) for x in range(3)])
            pipe = pipes.LocalPipe(self.test_stream.layout)
            self.test_data = helpers.create_data(layout=self.test_stream.layout, length=1005)
            task = self.store.spawn_inserter(self.test_stream, pipe)
            await pipe.write(self.test_data)
            await pipe.close()
            runner = await task
            await runner
            await conn.close()
            # await self.store.initialize([])
            await test()
            # simulate the nose2 test output
            sys.stdout.write('o')
            await self.store.close()
            sys.stdout.flush()

    async def _test_basic_insert_extract(self):
        stream_id = 990
        self.store.extract_block_size = 500
        psql_types = ['double precision', 'real', 'bigint', 'integer', 'smallint']
        datatypes = [DataStream.DATATYPE.FLOAT64, DataStream.DATATYPE.FLOAT32, DataStream.DATATYPE.INT64,
                     DataStream.DATATYPE.INT32, DataStream.DATATYPE.INT16]
        conn: asyncpg.Connection = await asyncpg.connect(self.db_url)
        for i in range(len(datatypes)):
            datatype = datatypes[i]
            psql_type = psql_types[i]
            for n_elements in range(1, 5):
                test_stream = DataStream(id=stream_id, name="stream1", datatype=datatype, keep_us=DataStream.KEEP_ALL,
                                         elements=[Element(name="e%d" % x) for x in range(n_elements)])
                test_stream.decimate = True
                source = QueueReader()
                pipe = pipes.InputPipe(stream=test_stream, reader=source)
                nrows = 955
                data = helpers.create_data(layout=test_stream.layout, length=nrows)
                task = await self.store.spawn_inserter(test_stream, pipe)
                for chunk in helpers.to_chunks(data, 300):
                    await source.put(chunk.tobytes())
                await task

                # make sure the correct tables have been created
                records = await conn.fetch('''SELECT table_name FROM information_schema.tables 
                                                    WHERE table_schema='data';''')
                tables = list(itertools.chain(*records))
                for table in ['stream%d' % stream_id, 'stream%d_intervals' % stream_id]:
                    self.assertIn(table, tables)

                # check the column data types
                records = await conn.fetch('''SELECT column_name, data_type FROM information_schema.columns 
                                                            WHERE table_name='stream%d' AND table_schema='data';''' % stream_id)
                (names, types) = zip(*records)
                expected_elements = ['time'] + ['elem%d' % x for x in range(n_elements)]
                self.assertCountEqual(names, expected_elements)
                expected_psql_types = tuple(['timestamp without time zone'] + [psql_type for x in range(n_elements)])
                self.assertEqual(types, expected_psql_types)
                self.assertEqual(len(records), n_elements + 1)

                # extract raw data
                extracted_data = []

                async def callback(rx_data, layout, factor):
                    self.assertEqual(layout, test_stream.layout)
                    self.assertEqual(factor, 1)
                    extracted_data.append(rx_data)

                await self.store.extract(test_stream, start=None, end=None, callback=callback)
                extracted_data = np.hstack(extracted_data)
                np.testing.assert_array_equal(extracted_data, data)

                level = 64
                data_mean = np.mean(extracted_data['data'][:level], axis=0)
                data_max = np.max(extracted_data['data'][:level], axis=0)
                data_min = np.min(extracted_data['data'][:level], axis=0)

                # extract decimated data
                async def d_callback(rx_data, layout, factor):
                    self.assertEqual(layout, test_stream.decimated_layout)
                    self.assertEqual(factor, level)
                    extracted_data.append(rx_data)

                extracted_data = []
                await self.store.extract(test_stream, decimation_level=level,
                                         start=None, end=None, callback=d_callback)
                extracted_data = np.hstack(extracted_data)
                expected = np.hstack((data_mean, data_min, data_max))
                np.testing.assert_array_almost_equal(expected, extracted_data['data'][0])
                stream_id += 1
        await conn.close()

    async def _test_extract_data_with_intervals(self):
        test_stream = DataStream(id=1, name="stream1", datatype=DataStream.DATATYPE.FLOAT32, keep_us=DataStream.KEEP_ALL,
                                 decimate=True, elements=[Element(name="e%d" % x) for x in range(3)])
        pipe = pipes.LocalPipe(test_stream.layout)
        nrows = 955
        data = helpers.create_data(layout=test_stream.layout, length=nrows)
        task = await self.store.spawn_inserter(test_stream, pipe)
        for chunk in helpers.to_chunks(data, 300):
            await pipe.write(chunk)
            await pipe.close_interval()
        await pipe.close()
        await task

        # extract data
        extracted_data = []

        async def callback(rx_data, layout, factor):
            self.assertEqual(layout, test_stream.layout)
            self.assertEqual(factor, 1)
            extracted_data.append(rx_data)

        await self.store.extract(test_stream, start=None, end=None, callback=callback)
        extracted_data = np.hstack(extracted_data)
        # check for interval boundaries
        np.testing.assert_array_equal(extracted_data[300], pipes.interval_token(test_stream.layout))
        np.testing.assert_array_equal(extracted_data[601], pipes.interval_token(test_stream.layout))
        np.testing.assert_array_equal(extracted_data[902], pipes.interval_token(test_stream.layout))

    async def _test_merge_gaps_on_write(self):
        # make sure successive writes less than a merge_gap apart are merged
        test_stream = DataStream(id=1, name="stream1", datatype=DataStream.DATATYPE.FLOAT32, keep_us=DataStream.KEEP_ALL,
                                 decimate=True, elements=[Element(name="e%d" % x) for x in range(3)])
        nrows = 300
        data = helpers.create_data(layout=test_stream.layout, length=nrows, step=1000, start=15e6)
        data_1 = data[:100]
        data_2 = data[100:200]
        data_3 = data[200:]
        # write the first chunk
        pipe = pipes.LocalPipe(test_stream.layout)
        task = await self.store.spawn_inserter(test_stream, pipe)
        await pipe.write(data_1)
        await pipe.close()
        await task

        # write the second chunk inside the merge_gap setting
        pipe = pipes.LocalPipe(test_stream.layout)
        task = await self.store.spawn_inserter(test_stream, pipe, merge_gap=1100)
        await pipe.write(data_2)
        await pipe.close()
        await task

        # write the third chunk outside the merge_gap setting
        pipe = pipes.LocalPipe(test_stream.layout)
        task = await self.store.spawn_inserter(test_stream, pipe, merge_gap=900)
        await pipe.write(data_3)
        await pipe.close()
        await task

        # extract data
        extracted_data = []

        intervals = await self.store.intervals(test_stream,start=None,end=None)
        #print(intervals)
        async def callback(rx_data, layout, factor):
            self.assertEqual(layout, test_stream.layout)
            self.assertEqual(factor, 1)
            extracted_data.append(rx_data)

        await self.store.extract(test_stream, start=None, end=None, callback=callback)
        extracted_data = np.hstack(extracted_data)
        self.assertEqual(len(extracted_data),301)
        # check for interval boundaries
        for i in range(200):
            np.testing.assert_array_equal(extracted_data[i], data[i])
        np.testing.assert_array_equal(extracted_data[200], pipes.interval_token(test_stream.layout))
        for i in range(201, 301):
            np.testing.assert_array_equal(extracted_data[i], data[i-1])
       

    async def _test_extract_decimated_data(self):

        extracted_data = []

        async def callback(rx_data, layout, factor):
            self.assertEqual(layout, self.test_stream.decimated_layout)
            self.assertEqual(factor, 4)
            extracted_data.append(rx_data)

        await self.store.extract(self.test_stream, max_rows=16,
                                 start=self.test_data['timestamp'][66],
                                 end=self.test_data['timestamp'][64 + 4 * 16],
                                 callback=callback)
        extracted_data = np.hstack(extracted_data)
        self.assertLessEqual(len(extracted_data), 16)
        self.assertGreater(len(extracted_data), 4)

    async def _test_remove_decimations(self):

        # make sure the stream is decimated:
        conn: asyncpg.Connection = await asyncpg.connect(self.db_url)
        table_names = await psql_helpers.get_decimation_table_names(conn, self.test_stream, with_schema=False)
        for table in [f'stream100_{4**i}' for i in range(1,6)]:
           self.assertIn(table, table_names)
        await self.store.drop_decimations(self.test_stream)
        # drop the decimations
        table_names = await psql_helpers.get_decimation_table_names(conn, self.test_stream, with_schema=False)
        # make sure the tables are dropped
        self.assertEqual(len(table_names), 0)
        await conn.close()

    async def _test_redecimates_data(self):
        # create decimated data
        test_stream = DataStream(id=1, name="stream1", datatype=DataStream.DATATYPE.FLOAT32,
                                 keep_us=DataStream.KEEP_ALL,
                                 elements=[Element(name="e%d" % x) for x in range(3)])
        test_stream.decimate = True
        source = QueueReader()
        pipe = pipes.InputPipe(stream=test_stream, reader=source)
        nrows = 1024
        data = helpers.create_data(layout=test_stream.layout, length=nrows)
        task = await self.store.spawn_inserter(test_stream, pipe)
        for chunk in helpers.to_chunks(data, 16):
            await source.put(chunk.tobytes())
            await source.put(pipes.interval_token(test_stream.layout).tobytes())
        await task # put the data in the database, 16 rows per interval
        conn: asyncpg.Connection = await asyncpg.connect(self.db_url)
        table_names = await psql_helpers.get_decimation_table_names(conn, test_stream, with_schema=False)
        # interval breaks cause decimation to stop after level 16 (level 64 is created but empty)
        self.assertEqual(len(table_names),3)
        orig_data = await conn.fetch("SELECT * FROM data.stream1_16")
        # redecimate doesn't change anything
        await self.store.decimate(test_stream)
        redecimated_data = await conn.fetch("SELECT * FROM data.stream1_16")
        self.assertEqual(orig_data, redecimated_data)

        intervals = await self.store.intervals(test_stream, start=None, end=None)
        self.assertEqual(len(intervals), 64)
        # now consolidate the intervals
        await self.store.consolidate(test_stream, start=None, end=None,
                                     max_gap = data['timestamp'][1]-data['timestamp'][0])
        intervals = await self.store.intervals(test_stream, start=None, end=None)
        self.assertEqual(len(intervals), 1)
        # redecimate
        await self.store.decimate(test_stream)
        table_names = await psql_helpers.get_decimation_table_names(conn, test_stream, with_schema=False)
        for table in [f'stream1_{4**i}' for i in range(1,6)]:
           self.assertIn(table, table_names)
        await conn.close()

    async def _test_nondecimating_inserter(self):
        # TODO
        pass

    async def _test_row_count(self):
        test_data = helpers.create_data(layout=self.test_stream.layout, length=10000)
        test_stream = DataStream(id=95, name="stream1", datatype=DataStream.DATATYPE.FLOAT32, keep_us=DataStream.KEEP_ALL,
                                 decimate=True, elements=[Element(name="e%d" % x) for x in range(3)])
        pipe = pipes.LocalPipe(test_stream.layout)
        task = await self.store.spawn_inserter(test_stream, pipe)
        await pipe.write(test_data)
        await pipe.close()
        await task
        conn: asyncpg.Connection = await asyncpg.connect(self.db_url)
        # test to make sure nrows is within 10% of actual value
        # Test: [start, end]
        nrows = await psql_helpers.get_row_count(conn, test_stream,
                                                 None,
                                                 None)
        self.assertGreater(nrows, len(test_data) * 0.9)
        # Test: [ts,end]
        nrows = await psql_helpers.get_row_count(conn, test_stream, test_data[len(test_data) // 2][0], None)
        self.assertLess(abs(nrows - len(test_data) // 2), 0.1 * len(test_data))
        # Test: [start, ts]
        nrows = await psql_helpers.get_row_count(conn, test_stream,
                                                 None,
                                                 test_data[len(test_data) // 3][0])
        self.assertLess(abs(nrows - len(test_data) // 3), 0.1 * len(test_data))

        # Test: [ts, ts]
        nrows = await psql_helpers.get_row_count(conn, test_stream,
                                                 test_data[2 * len(test_data) // 6][0],
                                                 test_data[3 * len(test_data) // 6][0])
        self.assertLess(abs(nrows - len(test_data) // 6), 0.1 * len(test_data))

        # Test: [ts, ts] (no data)
        nrows = await psql_helpers.get_row_count(conn, test_stream,
                                                 test_data['timestamp'][0] - 100,
                                                 test_data['timestamp'][0] - 50)
        self.assertEqual(0, nrows)

        # Test row count for stream with no data tables
        empty_stream = DataStream(id=96, name="empty", datatype=DataStream.DATATYPE.FLOAT64,
                                  keep_us=100, decimate=True, elements=[Element(name="e%d" % x) for x in range(8)])
        nrows = await psql_helpers.get_row_count(conn, empty_stream,
                                                 None,
                                                 None)
        self.assertEqual(0, nrows)
        nrows = await psql_helpers.get_row_count(conn, empty_stream,
                                                 test_data[len(test_data) // 2][0],
                                                 None)
        self.assertEqual(0, nrows)
        nrows = await psql_helpers.get_row_count(conn, test_stream,
                                                 test_data['timestamp'][0] - 100,
                                                 test_data['timestamp'][0] - 50)
        self.assertEqual(0, nrows)
        await conn.close()

    async def _test_remove(self):

        #  XXXXXXX------XXXXXXX
        #        ^|     ^
        # remove a chunk of data from the middle
        await self.store.remove(self.test_stream,
                                self.test_data['timestamp'][300],
                                self.test_data['timestamp'][400])
        # XXXXXXX------===XXXX
        #       ^|        ^
        # remove another chunk
        await self.store.remove(self.test_stream,
                                self.test_data['timestamp'][350],
                                self.test_data['timestamp'][500])

        # XXX___----===XXX
        #   ^|         ^
        await self.store.remove(self.test_stream,
                                self.test_data['timestamp'][250],
                                self.test_data['timestamp'][300])

        # extract the data, should have an interval gap between 249 and 500
        # and a *single* interval boundary at 250

        extracted_data = []

        async def callback(rx_data, layout, decimated):
            extracted_data.append(rx_data)

        await self.store.extract(self.test_stream, start=None, end=None,
                                 callback=callback)
        extracted_data = np.hstack(extracted_data)

        # beginning is unchanged
        np.testing.assert_array_equal(extracted_data[:249], self.test_data[:249])
        # interval boundary marking missing data
        np.testing.assert_array_equal(extracted_data[250],
                                      pipes.interval_token(self.test_stream.layout))
        # end is unchanged (closing interval boundary ignored)
        np.testing.assert_array_equal(extracted_data[251:], self.test_data[500:])

        # two intervals of data
        intervals = await self.store.intervals(self.test_stream, start=None, end=None)
        ts = self.test_data['timestamp']
        expected = [[ts[0], ts[249] + 1],
                    [ts[500], ts[-1] + 1]]
        self.assertEqual(intervals, expected)

    async def _test_destroy(self):
        await self.store.destroy(self.test_stream)
        records = await self.store.info([self.test_stream])
        info = records[self.test_stream.id]
        self.assertIsNone(info.start)
        self.assertIsNone(info.end)
        self.assertEqual(info.total_time, 0)
        self.assertEqual(info.rows, 0)
        self.assertEqual(info.bytes, 0)

    async def _test_intervals(self):

        ts = self.test_data['timestamp']

        async def _insert_boundary(_conn, _ts: int):
            dt = datetime.datetime.fromtimestamp(_ts / 1e6, tz=datetime.timezone.utc)
            query = "INSERT INTO data.stream100_intervals (time) VALUES ('%s')" % dt
            await _conn.execute(query)

        conn: asyncpg.Connection = await asyncpg.connect(self.db_url)

        # XXXX|XXXXX|XXXXX
        await _insert_boundary(conn, ts[99] + 1)
        await _insert_boundary(conn, ts[199] + 1)
        expected = [[ts[0], ts[99] + 1],
                    [ts[100], ts[199] + 1],
                    [ts[200], ts[-1] + 1]]
        intervals = await self.store.intervals(self.test_stream, None, None)
        self.assertEqual(expected, intervals)

        # insert an extra boundary that does not create a new interval
        # XXXX||XXXXX|XXXXX
        await _insert_boundary(conn, ts[99] + 10)

        intervals = await self.store.intervals(self.test_stream, None, None)
        self.assertEqual(expected, intervals)

        # insert boundaries at the beginning and end of the data,
        # this should also not create any new intervals
        # |XXXX||XXXXX|XXXXX|

        await _insert_boundary(conn, ts[0] - 1)
        await _insert_boundary(conn, ts[-1] + 1)
        intervals = await self.store.intervals(self.test_stream, None, None)
        self.assertEqual(expected, intervals)

        # insert an actual boundary, this should create a new interval
        # |XXXX||XXXXX|XXX|X|
        await _insert_boundary(conn, ts[780] + 1)
        intervals = await self.store.intervals(self.test_stream, None, None)
        expected = [[ts[0], ts[99] + 1],
                    [ts[100], ts[199] + 1],
                    [ts[200], ts[780] + 1],
                    [ts[781], ts[-1] + 1]]
        self.assertEqual(expected, intervals)
        await conn.close()

    async def _test_consolidate(self):
        # intervals less than max_gap us apart are consolidated
        # data: 100 samples spaced at 1000us
        test_stream = DataStream(id=1, name="stream1", datatype=DataStream.DATATYPE.FLOAT32, keep_us=DataStream.KEEP_ALL,
                                 decimate=True, elements=[Element(name="e%d" % x) for x in range(3)])
        nrows = 955
        orig_data = helpers.create_data(layout=test_stream.layout, length=nrows)

        rx_chunks = []
        async def callback(rx_data, layout, factor):
            #if rx_data[0] != pipes.interval_token(layout):
            if rx_data[-1][0]==0:
                rx_chunks.append(rx_data[:-1])
            else:
                rx_chunks.append(rx_data)

        chunks = [orig_data[:300], orig_data[305:400], orig_data[402:700], orig_data[800:]]
        # data: |++++++|  |+++++++++|    |++++++|    |++++|
        #               ^--5000 us    ^--2000 us   ^---0.1 sec (retained)
        pipe = pipes.LocalPipe(test_stream.layout)
        task = await self.store.spawn_inserter(test_stream, pipe)
        for chunk in chunks:
            await pipe.write(chunk)
            await pipe.close_interval()
        await pipe.close()
        await task

        await self.store.consolidate(test_stream, start=None, end=None, max_gap=6e3)
        await self.store.extract(test_stream, start=None, end=None, callback=callback)
        # should only be two intervals left (the first three are consolidated)
        np.testing.assert_array_equal(rx_chunks[0], np.hstack(chunks[:3]))
        np.testing.assert_array_equal(rx_chunks[1], np.hstack(chunks[3:]))
        self.assertEqual(len(rx_chunks), 2)
        await self.store.remove(test_stream, start=None, end=None) # drop the data to get ready for the next test

        chunks = [orig_data[:300], orig_data[305:400], orig_data[402:700], orig_data[800:850], orig_data[852:]]
        # data: |++++++|  |+++++++++|    |++++++|    |++++|  |++++|
        #               ^--5000 us    ^--2000 us   |        ^--- 2000 us
        #                                          `---0.1 sec (retained)
        pipe = pipes.LocalPipe(test_stream.layout)
        task = await self.store.spawn_inserter(test_stream, pipe)
        for chunk in chunks:
            await pipe.write(chunk)
            await pipe.close_interval()
        await pipe.close()
        await task

        rx_chunks = [] # flush the rx buffer for the next test
        await self.store.consolidate(test_stream, start=None, end=None, max_gap=6e3)
        await self.store.extract(test_stream, start=None, end=None, callback=callback)

        # should only be two intervals left (the first two are consolidated)
        np.testing.assert_array_equal(rx_chunks[0], np.hstack(chunks[:3]))
        np.testing.assert_array_equal(rx_chunks[1], np.hstack(chunks[3:]))
        self.assertEqual(len(rx_chunks), 2)

    async def _test_consolidate_with_time_bounds(self):
        # intervals less than max_gap us apart between start and end are consolidated
        # data: 100 samples spaced at 1000us
        test_stream = DataStream(id=1, name="stream1", datatype=DataStream.DATATYPE.FLOAT32, keep_us=DataStream.KEEP_ALL,
                                 decimate=True, elements=[Element(name="e%d" % x) for x in range(3)])
        pipe = pipes.LocalPipe(test_stream.layout)
        nrows = 955
        orig_data = helpers.create_data(layout=test_stream.layout, length=nrows)
        chunks = [orig_data[:300], orig_data[305:400], orig_data[402:700], orig_data[800:850],orig_data[852:]]
        # data: |++++++|  |+++++++++|    |++++++|    |++++|  |++++|
        #               ^--(retained) ^--2000 us   |        ^--- 2000 us (retained)
        #                                          `---0.1 sec (retained)
        task = await self.store.spawn_inserter(test_stream, pipe)
        for chunk in chunks:
            await pipe.write(chunk)
            await pipe.close_interval()
        await pipe.close()
        await task

        # extract data
        rx_chunks = []

        async def callback(rx_data, layout, factor):
            if rx_data[-1][0]==0:
                rx_chunks.append(rx_data[:-1])
            else:
                rx_chunks.append(rx_data)

        await self.store.consolidate(test_stream, start=chunks[1][3]['timestamp'],
                                     end=chunks[3][3]['timestamp'], max_gap=6e3)
        await self.store.extract(test_stream, start=None, end=None, callback=callback)

        # should only be four intervals left (the inner short intervals are consolidated)
        np.testing.assert_array_equal(rx_chunks[0], chunks[0])
        np.testing.assert_array_equal(rx_chunks[1], np.hstack(chunks[1:3]))
        np.testing.assert_array_equal(rx_chunks[2], chunks[3])
        np.testing.assert_array_equal(rx_chunks[3], chunks[4])

        self.assertEqual(len(rx_chunks), 4)

    async def _test_db_info(self):
        db_info = await self.store.dbinfo()
        self.assertTrue(os.path.isdir(db_info.path))
        self.assertGreater(db_info.other, 0)
        self.assertGreater(db_info.reserved, 0)
        self.assertGreater(db_info.free, 0)
        self.assertGreater(db_info.size, 0)

    async def _test_info(self):
        # create another stream
        empty_stream = DataStream(id=103, name="empty stream", datatype=DataStream.DATATYPE.INT32,
                                  keep_us=DataStream.KEEP_ALL, decimate=True,
                                  elements=[Element(name="e%d" % x) for x in range(8)])
        stream2 = DataStream(id=104, name="stream2", datatype=DataStream.DATATYPE.INT32,
                             keep_us=DataStream.KEEP_ALL, decimate=True,
                             elements=[Element(name="e%d" % x) for x in range(8)])
        pipe = pipes.LocalPipe(stream2.layout)
        test_data = helpers.create_data(layout=stream2.layout, length=800)
        task = await self.store.spawn_inserter(stream2, pipe)
        await pipe.write(test_data)
        await pipe.close()
        await task
        records = await self.store.info([self.test_stream, stream2, empty_stream])
        # check stream1
        info = records[self.test_stream.id]
        self.assertEqual(info.start, self.test_data['timestamp'][0])
        self.assertEqual(info.end, self.test_data['timestamp'][-1])
        self.assertEqual(info.total_time, info.end - info.start)
        # rows are approximate
        self.assertLess(abs(len(self.test_data) - info.rows), len(self.test_data) * 0.1)
        self.assertGreater(info.bytes, 0)

        # check stream2
        info = records[stream2.id]
        self.assertEqual(info.start, test_data['timestamp'][0])
        self.assertEqual(info.end, test_data['timestamp'][-1])
        self.assertEqual(info.total_time, info.end - info.start)
        self.assertLess(abs(len(test_data) - info.rows), len(test_data) * 0.1)
        self.assertGreater(info.bytes, 0)

        # check the empty stream
        info = records[empty_stream.id]
        self.assertIsNone(info.start)
        self.assertIsNone(info.end)
        self.assertEqual(info.total_time, 0)
        self.assertEqual(info.rows, 0)
        self.assertEqual(info.bytes, 0)

    async def _test_actions_on_empty_streams(self):
        # make sure all actions can be performed on streams with no data
        empty_stream = DataStream(id=103, name="empty stream", datatype=DataStream.DATATYPE.INT32,
                                  keep_us=DataStream.KEEP_ALL, decimate=True,
                                  elements=[Element(name="e%d" % x) for x in range(8)])

        # ==== extract =====
        cb_executed = False
        cb_layout = None
        cb_factor = None

        async def callback(rx_data, layout, factor):
            nonlocal cb_executed, cb_layout, cb_factor
            cb_executed = True
            self.assertEqual(len(rx_data), 0)
            cb_layout = layout
            cb_factor = factor

        # stream with no data tables
        await self.store.extract(empty_stream, start=None, end=None, callback=callback)
        self.assertTrue(cb_executed)
        self.assertEqual(cb_layout, empty_stream.layout)
        self.assertEqual(cb_factor, 1)
        cb_executed = False
        await self.store.extract(empty_stream, start=100, end=None, callback=callback)
        self.assertTrue(cb_executed)
        self.assertEqual(cb_layout, empty_stream.layout)
        self.assertEqual(cb_factor, 1)
        cb_executed = False
        await self.store.extract(empty_stream, start=100, end=300, callback=callback)
        self.assertTrue(cb_executed)
        self.assertEqual(cb_layout, empty_stream.layout)
        self.assertEqual(cb_factor, 1)
        cb_executed = False
        await self.store.extract(empty_stream, start=None, end=100, callback=callback)
        self.assertTrue(cb_executed)
        self.assertEqual(cb_layout, empty_stream.layout)
        self.assertEqual(cb_factor, 1)
        cb_executed = False
        await self.store.extract(empty_stream, start=100, end=300, max_rows=40, callback=callback)
        self.assertTrue(cb_executed)
        self.assertEqual(cb_layout, empty_stream.layout)
        self.assertEqual(cb_factor, 1)
        cb_executed = False
        await self.store.extract(empty_stream, start=100, end=300, decimation_level=40, callback=callback)
        self.assertTrue(cb_executed)
        self.assertEqual(cb_layout, empty_stream.decimated_layout)
        self.assertEqual(cb_factor, 40)

        # non-existent decimation level
        await self.store.extract(self.test_stream, start=None, end=None, decimation_level=97,
                                 callback=callback)

        #  ==== intervals ====
        intervals = await self.store.intervals(empty_stream, None, None)
        self.assertEqual(0, len(intervals))
        intervals = await self.store.intervals(empty_stream, 100, None)
        self.assertEqual(0, len(intervals))
        intervals = await self.store.intervals(empty_stream, None, 100)
        self.assertEqual(0, len(intervals))
        intervals = await self.store.intervals(empty_stream, 100, 300)
        self.assertEqual(0, len(intervals))

        #  ==== remove ====
        await self.store.remove(empty_stream, None, None)
        await self.store.remove(empty_stream, 100, None)
        await self.store.remove(empty_stream, None, 100)
        await self.store.remove(empty_stream, 100, 300)

        # ==== destroy ====
        await self.store.destroy(empty_stream)
