"""
Created on Mon Jan 31 08:53:59 2022
@author: eduardoaraujo

Last change on 2022/09/22
This module is responsible retrieve generated chunks of data containing COVID
information collect via Socrata API from Colombia Governmental's data
collection. Connect to SQL Database and load chunks in order to update
positive_cases_covid_d table.

Methods
-------

gen_chunks_into_db():
    Generate chunks of data to insert into SQL Database using pangres.
"""
from datetime import datetime, timedelta

from loguru import logger
from pangres import upsert

from epigraphhub.connection import get_engine
from epigraphhub.data.data_collection.colombia.data_chunk import chunked_fetch
from epigraphhub.data.data_collection.config import COLOMBIA_CLIENT, COLOMBIA_LOG_PATH
from epigraphhub.settings import env

logger.add(COLOMBIA_LOG_PATH, retention="7 days")
client = COLOMBIA_CLIENT


def gen_chunks_into_db():
    """
    This method will receive chunks generated by chunked_fetch and load them
    into the SQL Database. Pangres receives the records found in the Colombia data
    through Socrata API, uses the generator to load chunks with size of 10000 into
    SQL DB using upsert method.
    @note Colombia sometimes has a post update in the data, so rows update
          in this case is required to retrieve the rows updated.
    @see epigraphhub.data.data_collection.colombia.data_chunk
    """
    slice_date = datetime.date(datetime.today()) - timedelta(200)
    slice_date = slice_date.strftime("%Y-%m-%d")

    # count the number of records that will be fetched
    records = client.get_all(
        "gt2j-8ykr",
        select="COUNT(*)",
        where=f'fecha_reporte_web > "{slice_date}"',
    )

    for i in records:
        record_count = i
        break

    del records

    maxrecords = int(record_count["COUNT"])

    engine = get_engine(env.db.default_credential)

    for df_new in chunked_fetch(maxrecords):

        # save the data
        with engine.connect() as conn:
            upsert(
                con=conn,
                df=df_new,
                table_name="positive_cases_covid_d",
                schema="colombia",
                if_row_exists="update",
                chunksize=1000,
                add_new_columns=True,
                create_table=False,
            )

    logger.info("Table positive_cases_covid_d updated.")
