import asyncio
import copy
import json
import logging
import os
import time
import uuid
from random import randrange

import aiohttp
import backoff
import pandas as pd
import requests
from shapely.geometry import shape

try:
    from .. import persistence
except ImportError:
    import persistence


ADDRESS_RUN_TABLE = "ADDRESS_RUN"
ADDRESS_CACHE_TABLE = "ADDRESS"
ADDRESS_QUEUE_TABLE = "ADDRESS_QUEUE"
CACHE_DATABASE = "CIM_DATA_ENABLEMENT_VALIDATION"
CACHE_SCHEMA = "XHXA053_CACHE"
FATAL_RESP_CODE = 403
EARTH_DEFINE_BUILDINGS_ENDPOINT = "https://api.buildings.earthdefine.com/v1/grainger-buildings"
EARTH_DEFINE_API_TOKEN = os.getenv("EARTH_DEFINE_APIKEY", "default")
MAX_PROCESS_ATTEMPTS = 1

logging.getLogger("EARTH_DEFINE_CACHE").addHandler(logging.StreamHandler())


def get_cache_schema():
    """
    Returns the name of the cache schema, depending on whether a test run_schema_name has been set
    Returns: the name of the cache schema

    """
    if "test" in os.getenv("RUN_SCHEMA_NAME", ""):
        return CACHE_SCHEMA + os.getenv("RUN_SCHEMA_NAME", "")
    return CACHE_SCHEMA


def get_cache_database():
    """
    Returns the name of the cache database, depending on whether a test run_schema_name has been set
    Returns: the name of the cache database

    """
    if "test" in os.getenv("RUN_SCHEMA_NAME", ""):
        return "CIMI_VAL"
    return CACHE_DATABASE


def set_env_config(**config):
    """
    Sets the environment variables based on the incoming config
    Args:
        **config: dict of config data to set

    Returns: None

    """
    for key, val in config.items():
        logging.info(
            f'Setting environment variable {"CACHE_" + key.upper() + " = " + str(val)}'
        )
        os.environ[f"CACHE_{key.upper()}"] = str(val)


def set_address_run_initial(pod_id, **config):
    """
    Sets initial address run record with a UUID ID
    Args:
        pod_id: id of Airflow pod
        **config: dict of config data to insert

    Returns: the address run ID

    """
    address_run_id = str(uuid.uuid4())
    sql = f"""INSERT INTO {get_cache_database()}.{get_cache_schema()}.{ADDRESS_RUN_TABLE} 
    (JOB_ID, ADDRESS_RUN_ID, POD_ID, CONFIG, SOURCE, START_TIME)
    SELECT
        '{os.getenv("JOB_ID")}',
        '{address_run_id}',
        '{pod_id}',
        to_array(parse_json('{json.dumps(config)}')),
        'EARTH_DEFINE',
        CURRENT_TIMESTAMP()
    """
    persistence.execute_query(sql)
    logging.info(f"Setting Address_Run_ID to {address_run_id}")
    return address_run_id


def set_address_run_end(
        address_run_id, pod_id, status, exception=None, **update_column_values
):
    """
    Updates the address run record with the associated metadata
    Args:
        address_run_id: address run ID to be updated
        pod_id: the name of the Airflow pod that is performing the address run
        status: the status of the address run, either FAILED, COMPLETE, or IN PROGRESS
        exception: the exception generated by the address run, if there is one
        **update_column_values: metadata to update record

    Returns: None

    """
    column_list = []
    for key, value in update_column_values.items():
        column_list.append(f" {key} = '{value}'")
    column_list_string = ",".join(column_list)
    if len(column_list_string) > 0:
        column_list_string = column_list_string + ","
    find_run_id_sql = f"""SELECT ADDRESS_RUN_ID 
                            FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_RUN_TABLE} 
                            WHERE ADDRESS_RUN_ID = '{address_run_id}'"""
    response_df = persistence.get_df(find_run_id_sql)
    if response_df.size < 1:
        raise ValueError(f"Address Run ID {address_run_id} not found")

    logging.info(f"Updating metrics for Address_Run_ID {address_run_id}")
    sql = f"""UPDATE {get_cache_database()}.{get_cache_schema()}.{ADDRESS_RUN_TABLE} SET
        {column_list_string}
        STATUS = '{status}',
        STACK_TRACE = IFF('{json.dumps(str(exception).replace("'",'"'))}' = '"None"', NULL, '{str(exception).replace("'",'"')}'),
        END_TIME = CURRENT_TIMESTAMP(),
        UPDATED_AT = CURRENT_TIMESTAMP()
        WHERE ADDRESS_RUN_ID = '{address_run_id}'
          AND POD_ID = '{pod_id}'"""
    persistence.execute_query(sql)


def get_address_run_config(address_run_id):
    """
    Gets the run configuration data for a given address run ID
    Args:
        address_run_id: current address run id

    Returns: DataFrame of configuration data

    """
    sql = f"SELECT * FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_RUN_TABLE} where id = '{address_run_id}'"
    return persistence.get_df(sql)


def get_address_queue_count():
    """
    Gets the count of records in the address queue
    Returns: the count of records in the address queue

    """
    sql = f"""SELECT COUNT(*) AS CNT 
                FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE} 
                WHERE PROCESS_ATTEMPTS < {MAX_PROCESS_ATTEMPTS}"""
    return int(persistence.get_df(sql)["CNT"][0])


def get_address_queue_records(batch_size, pod_id):
    """
    Gets records from the queue, up to the batch size
    Args:
        batch_size: max number of records to get
        pod_id: id of Airflow pod

    Returns: DataFrame of records from the queue

    """
    reserve_sql = f"""UPDATE {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}
                        SET POD_ID = '{pod_id}',
                            PROCESS_ATTEMPTS = PROCESS_ATTEMPTS + 1,
                            UPDATED_AT = CURRENT_TIMESTAMP()
                      WHERE ID IN (
                        SELECT ID FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}
                          WHERE PROCESS_ATTEMPTS < {MAX_PROCESS_ATTEMPTS}
                            AND NOT IS_FAILED
                          ORDER BY CREATED_AT
                          LIMIT {batch_size}
                      )"""
    persistence.execute_query(reserve_sql)

    sql = f"""SELECT ID, REQUEST_STREET_ADDRESS, REQUEST_CITY, REQUEST_STATE, REQUEST_ZIPCODE FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}
                WHERE POD_ID = '{pod_id}' AND NOT IS_FAILED"""
    address_df = persistence.get_df(sql)

    return address_df


def fatal_code(e):
    return e == FATAL_RESP_CODE


def get_api_address(request_street_address, request_city, request_state, request_zipcode):
    """
    Sends single address to EarthDefine, returns DataFrame of the parsed response
    Args:
        request_street_address: street address to geocode
        request_city: city of address to geocode
        request_state: state of address to geocode
        request_zipcode: zipcode of address to geocode

    Returns: DataFrame of parsed EarthDefine response

    """
    request_df = pd.DataFrame([[request_street_address,
                                request_city, request_state,
                                request_zipcode]],
                              columns=["REQUEST_STREET_ADDRESS",
                                       "REQUEST_CITY",
                                       "REQUEST_STATE",
                                       "REQUEST_ZIPCODE"])
    return asyncio.run(get_api_address_bulk(request_df))


def set_address_queue(request_street_address, request_city, request_state, request_zipcode, **update_column_values):
    """
    Updates an existing record in the address queue
    Args:
        request_street_address: the street address from the queue to be updated
        request_city: the city of the address to be updated
        request_state: the state of the address to be updated
        request_zipcode: the zipcode of the address to be updated
        **update_column_values: dict of column names and values to be updated

    Returns: None

    """
    column_list = []
    for key, value in update_column_values.items():
        column_list.append(f" {key} = '{value}'")
    column_list_string = ",".join(column_list)
    if len(column_list_string) > 0:
        column_list_string = column_list_string + ","
    find_address_sql = f"""SELECT ID 
                            FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE} 
                            WHERE ID = '{get_address_id(request_street_address, request_city, request_state, request_zipcode)}'"""
    response_df = persistence.get_df(find_address_sql)
    if response_df.size < 1:
        raise ValueError(f"Request address {get_address_string(request_street_address, request_city, request_state, request_zipcode)} not found in queue")

    logging.info(f"Updating queued address {get_address_string(request_street_address, request_city, request_state, request_zipcode)}")
    sql = f"""UPDATE {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE} SET
        {column_list_string}
        UPDATED_AT = CURRENT_TIMESTAMP()
        WHERE ID = '{get_address_id(request_street_address, request_city, request_state, request_zipcode)}'"""
    persistence.execute_query(sql)


def reset_queue():
    """
    Sets the number of process attempts to 0 for any non-failed records in the address queue
    Returns: None

    """
    sql = f"""UPDATE {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE} SET
                PROCESS_ATTEMPTS = 0,
                POD_ID = NULL,
                UPDATED_AT = CURRENT_TIMESTAMP()
              WHERE NOT IS_FAILED"""
    persistence.execute_query(sql)


def reset_cache_pod_ids():
    """
    Unsets the pod id for all records in the cache
    Returns: None

    """
    sql = f"""UPDATE {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE} SET
                POD_ID = NULL,
                UPDATED_AT = CURRENT_TIMESTAMP()
              WHERE POD_ID IS NOT NULL"""
    persistence.execute_query(sql)


@backoff.on_exception(
    backoff.expo,
    requests.exceptions.RequestException,
    max_tries=os.getenv("CACHE_MAX_RETRIES", 5),
    giveup=fatal_code,
)
async def get_api_address_bulk(request_address_df: pd.DataFrame, update_queue=True):
    """
    Sends multiple addresses to EarthDefine, returns DataFrame of parsed responses
    Args:
        request_address_df: DataFrame of addresses to geocode
            DataFrame Columns:
                REQUEST_STREET_ADDRESS: the street address component of the requested addresses
                REQUEST_CITY: the city component of the requested addresses
                REQUEST_STATE: the 2-letter abbreviated state component of the requested addresses
                REQUEST_ZIPCODE: the 5-digit zipcode component of the requested addresses
        update_queue: boolean, if True then update the queue upon receiving an error

    Returns: DataFrame of parsed EarthDefine responses

    """
    response_df_columns = [
        "ID",
        "REQUEST_STREET_ADDRESS",
        "REQUEST_CITY",
        "REQUEST_STATE",
        "REQUEST_ZIPCODE",
        "RESPONSE_STREET_ADDRESS",
        "RESPONSE_CITY",
        "RESPONSE_STATE",
        "RESPONSE_ZIPCODE",
        "RESPONSE_COUNTY",
        "FORMATTED_ADDRESS",
        "RESPONSE_COUNT",
        "SOURCE_DATE",
        "SOURCE",
        "AREA_SQFT",
        "BLD_UUID",
        "STR_UUID",
        "ACCURACY",
        "GEOMETRY",
        "LATITUDE",
        "LONGITUDE",
        "RAW_RESPONSE",
    ]
    response_df = pd.DataFrame(columns=response_df_columns)
    error_403_counter = 0
    max_403 = 1
    async with aiohttp.ClientSession() as session:
        for index, row in request_address_df.iterrows():
            address_string = get_address_string(row["REQUEST_STREET_ADDRESS"],
                                                row["REQUEST_CITY"],
                                                row["REQUEST_STATE"],
                                                row["REQUEST_ZIPCODE"])
            params = {"street_address": row["REQUEST_STREET_ADDRESS"],
                      "city": row["REQUEST_CITY"],
                      "state": row["REQUEST_STATE"],
                      "zipcode": row["REQUEST_ZIPCODE"],
                      "token": EARTH_DEFINE_API_TOKEN}
            async with session.get(
                    url=EARTH_DEFINE_BUILDINGS_ENDPOINT, params=params
            ) as resp:
                if resp is None:
                    logging.warning(
                        f"Null response received from Earth Define for address {address_string}"
                    )
                    break
                resp_json = await resp.json(content_type=None)
                if resp.status == 200:
                    if int(resp.headers["X-Rate-Limit-Remaining"]) < int(
                            os.getenv("CACHE_MIN_REQUEST_LIMIT", 5)
                    ):
                        logging.info("Backing off")
                        time.sleep(int(os.getenv("CACHE_BACKOFF_TIME"), 2))
                    if resp_json and resp_json != []:
                        response_df = pd.concat(
                            [
                                response_df,
                                create_df_from_json(
                                    row["REQUEST_STREET_ADDRESS"],
                                    row["REQUEST_CITY"],
                                    row["REQUEST_STATE"],
                                    row["REQUEST_ZIPCODE"],
                                    resp_json
                                ),
                            ],
                            ignore_index=True,
                        )
                    else:
                        response_df = pd.concat(
                            [
                                response_df,
                                create_df_from_empty(
                                    row["REQUEST_STREET_ADDRESS"],
                                    row["REQUEST_CITY"],
                                    row["REQUEST_STATE"],
                                    row["REQUEST_ZIPCODE"]
                                ),
                            ],
                            ignore_index=True,
                        )

                elif resp.status == 403:
                    error_403_counter += 1
                    error_msg = "Call to earth_define failed with status 403"
                    logging.error(error_msg)
                    logging.info(f"403 REQUEST_ADDRESS: {address_string}")
                    logging.info(
                        f"""X-Rate-Limit-Remaining: {resp.headers["X-Rate-Limit-Remaining"]}"""
                    )
                    if update_queue:
                        set_address_queue(row["REQUEST_STREET_ADDRESS"], row["REQUEST_CITY"], row["REQUEST_STATE"], row["REQUEST_ZIPCODE"], **{"IS_FAILED": True})
                    if error_403_counter >= max_403:
                        logging.error(f"Max 403 errors ({max_403}) reached")
                        raise Exception(error_msg)

                elif resp.status == 400:
                    logging.error(f"Error 400 from Earth Define for request address {address_string}")
                    response_df = pd.concat(
                        [
                            response_df,
                            create_df_from_empty(
                                row["REQUEST_STREET_ADDRESS"],
                                row["REQUEST_CITY"],
                                row["REQUEST_STATE"],
                                row["REQUEST_ZIPCODE"]
                            ),
                        ],
                        ignore_index=True,
                    )
                    if update_queue:
                        set_address_queue(row["REQUEST_STREET_ADDRESS"], row["REQUEST_CITY"], row["REQUEST_STATE"], row["REQUEST_ZIPCODE"], **{"IS_FAILED": True})

                else:
                    if update_queue:
                        set_address_queue(row["REQUEST_STREET_ADDRESS"], row["REQUEST_CITY"], row["REQUEST_STATE"], row["REQUEST_ZIPCODE"], **{"IS_FAILED": True})
                    error_msg = f"Error {resp.status} from Earth Define for request address {address_string}"
                    logging.error(error_msg)
                    raise Exception(error_msg)
    return response_df


def set_addresses(address_df, address_run_id):
    """
    Inserts and updates the cache with new records
    Args:
        address_df: DataFrame of new records to be inserted/updated
            DataFrame columns:
                ID: unique address UUID based on the request address components
                REQUEST_STREET_ADDRESS: the street address component of the requested address
                REQUEST_CITY: the city component of the requested address
                REQUEST_STATE: the 2-letter abbreviated state component of the requested address
                REQUEST_ZIPCODE: the 5-digit zipcode component of the requested address
                RESPONSE_STREET_ADDRESS: the street address component of the response address from Earth Define
                RESPONSE_CITY: the city component of the response address from Earth Define
                RESPONSE_STATE: the 2-letter abbreviated state component of the response address from Earth Define
                RESPONSE_ZIPCODE: the 5-digit zipcode component of the response address from Earth Define
                RESPONSE_COUNTY: the county of the response address from Earth Define
                FORMATTED_ADDRESS: the formatted response address from Earth Define
                RESPONSE_COUNT: the number of buildings included in the response from Earth Define
                SOURCE_DATE: the source date included in the response from Earth Define
                SOURCE: the source included in the response from Earth Define
                AREA_SQFT: the square footage of the first building in the response from Earth Define
                BLD_UUID: the unique identifier for the building in the response from Earth Define
                STR_UUID: the unique identifier for the response address components from Earth Define
                ACCURACY: the accuracy of the geocode according to Earth Define
                GEOMETRY: the geometry of the first building in the response from Earth Define
                LATITUDE: the latitude of the first building in the response from Earth Define
                LONGITUDE: the longitude of the first building in the response from Earth Define
                RAW_RESPONSE: the full response from Earth Define, including all buildings
        address_run_id: the current address run ID

    Returns: None

    """
    logging.info(
        f"Setting {address_df.shape[0]} addresses for Address_Run_ID {address_run_id}"
    )
    table_name = persistence.upload_temp_df(
        address_df,
        "CACHE_TEMP",
        landing_schema_prefix=get_cache_schema(),
        run_id=os.getenv("RUN_ID"),
        add_run_schema=False,
        force_create_table=True,
        database=get_cache_database(),
    )
    set_addresses_from_temp_table(table_name, address_run_id)


def set_addresses_from_temp_table(temp_table_name, address_run_id):
    """
    Merges incoming addresses from transient table to the address cache table
    Args:
        temp_table_name: name of transient table to be merged
        address_run_id: the id of the address run

    Returns: None

    """
    sql = f"""
        MERGE INTO {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE} cache USING (
            SELECT DISTINCT * FROM {get_cache_database()}.{get_cache_schema()}.{temp_table_name}
            QUALIFY ROW_NUMBER() OVER(PARTITION BY UPPER(REQUEST_STREET_ADDRESS), UPPER(REQUEST_CITY), UPPER(REQUEST_STATE), REQUEST_ZIPCODE ORDER BY 1) = 1
        ) temp
        ON cache.ID = temp.ID
        WHEN MATCHED THEN UPDATE SET cache.RESPONSE_STREET_ADDRESS = temp.RESPONSE_STREET_ADDRESS,
                                     cache.RESPONSE_CITY = temp.RESPONSE_CITY,
                                     cache.RESPONSE_STATE = temp.RESPONSE_STATE,
                                     cache.RESPONSE_ZIPCODE = temp.RESPONSE_ZIPCODE,
                                     cache.RESPONSE_COUNTY = temp.RESPONSE_COUNTY,
                                     cache.FORMATTED_ADDRESS = temp.FORMATTED_ADDRESS,
                                     cache.ACCURACY = temp.ACCURACY,
                                     cache.GEOMETRY = try_to_geography(temp.GEOMETRY),
                                     cache.LATITUDE = temp.LATITUDE,
                                     cache.LONGITUDE = temp.LONGITUDE,
                                     cache.RAW_RESPONSE = TRY_PARSE_JSON(temp.RAW_RESPONSE),
                                     cache.RESPONSE_ID = uuid_string('fe971b24-9572-4005-b22f-351e9c09274d', to_char(temp.RAW_RESPONSE)),
                                     cache.ADDRESS_RUN_ID = NULLIF('{address_run_id}', 'None'),
                                     cache.IS_UPDATED_RESPONSE = CASE WHEN cache.RESPONSE_ID <> uuid_string('fe971b24-9572-4005-b22f-351e9c09274d', to_char(temp.RAW_RESPONSE))
                                                            THEN TRUE
                                                            ELSE FALSE
                                                        END,
                                     cache.IS_UPDATED_GEOCODE = 
                                        CASE WHEN (round(CACHE.LATITUDE,10) = round(temp.LATITUDE,10) 
                                                        and round(CACHE.LONGITUDE,10) = round(temp.LONGITUDE,10) )
                                             THEN FALSE
                                             ELSE TRUE
                                        END,
                                     cache.RUN_ID = NULLIF(temp.RUN_ID, ''),
                                     cache.REFRESHED_AT = CURRENT_TIMESTAMP(),
                                     cache.UPDATED_AT = CURRENT_TIMESTAMP()
        WHEN NOT MATCHED THEN INSERT (ID, REQUEST_STREET_ADDRESS, REQUEST_CITY, REQUEST_STATE, REQUEST_ZIPCODE, RESPONSE_STREET_ADDRESS, RESPONSE_CITY, RESPONSE_STATE, RESPONSE_ZIPCODE, RESPONSE_COUNTY, FORMATTED_ADDRESS, RESPONSE_COUNT, SOURCE_DATE, SOURCE, AREA_SQFT, BLD_UUID, STR_UUID, ACCURACY, GEOMETRY, LATITUDE, LONGITUDE, RAW_RESPONSE, RESPONSE_ID, ADDRESS_RUN_ID, RUN_ID)
        VALUES (temp.ID, temp.REQUEST_STREET_ADDRESS, temp.REQUEST_CITY, temp.REQUEST_STATE, temp.REQUEST_ZIPCODE, temp.RESPONSE_STREET_ADDRESS, temp.RESPONSE_CITY, temp.RESPONSE_STATE, temp.RESPONSE_ZIPCODE, temp.RESPONSE_COUNTY, temp.FORMATTED_ADDRESS, temp.RESPONSE_COUNT, temp.SOURCE_DATE, temp.SOURCE, temp.AREA_SQFT, temp.BLD_UUID, temp.STR_UUID, temp.ACCURACY, try_to_geography(temp.GEOMETRY), temp.LATITUDE, temp.LONGITUDE, TRY_PARSE_JSON(temp.RAW_RESPONSE), uuid_string('fe971b24-9572-4005-b22f-351e9c09274d', to_char(temp.RAW_RESPONSE)), NULLIF('{address_run_id}', 'None'), NULLIF(temp.RUN_ID, ''))
    """
    result = None
    retries = 5
    logging.info(
        f"Merging into {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE} from temp table {get_cache_database()}.{get_cache_schema()}.{temp_table_name}"
    )
    while result is None and retries > 0:
        result = persistence.execute_query(sql)
        if result is None:
            retries -= 1
            logging.warning(f"Merge unsuccessful, retries left: {retries}")
            if retries == 0:
                logging.warning(f"Merge unsuccessful, out of retries after 5 attempts")
            time.sleep(randrange(30) + 60)

    drop_sql = f"""DROP TABLE {get_cache_database()}.{get_cache_schema()}.{temp_table_name}"""
    persistence.execute_query(drop_sql)


def set_queue_from_temp_table(temp_table_name):
    """
    Merges incoming queue records from transient table to the address queue table
    Args:
        temp_table_name: name of transient table to be merged

    Returns: None

    """
    logging.info(
        f"Merging into {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE} from temp table {get_cache_database()}.{get_cache_schema()}.{temp_table_name}"
    )
    sql = f"""
          MERGE INTO {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE} queue USING (
              SELECT DISTINCT ID,
                              UPPER(REQUEST_STREET_ADDRESS) as REQUEST_STREET_ADDRESS,
                              UPPER(REQUEST_CITY) as REQUEST_CITY,
                              UPPER(REQUEST_STATE) as REQUEST_STATE,
                              UPPER(REQUEST_ZIPCODE) as REQUEST_ZIPCODE
                FROM {get_cache_database()}.{get_cache_schema()}.{temp_table_name}
        ) temp
        ON queue.ID = temp.ID
        WHEN MATCHED THEN UPDATE SET queue.UPDATED_AT = CURRENT_TIMESTAMP()
        WHEN NOT MATCHED THEN INSERT (ID, REQUEST_STREET_ADDRESS, REQUEST_CITY, REQUEST_STATE, REQUEST_ZIPCODE)
        VALUES (temp.ID, UPPER(temp.REQUEST_STREET_ADDRESS), UPPER(temp.REQUEST_CITY), UPPER(temp.REQUEST_STATE), UPPER(temp.REQUEST_ZIPCODE))"""
    persistence.execute_query(sql)

    drop_sql = f"""DROP TABLE {get_cache_database()}.{get_cache_schema()}.{temp_table_name}"""
    persistence.execute_query(drop_sql)


def get_ttl_addresses_for_processing(batch_size, pod_id):
    """
    Returns stale addresses that require refreshing limited to the batch size, ordered by their updated date
    Args:
        batch_size: max number of records to return
        pod_id: the id of the Airflow pod performing the current address run

    Returns: DataFrame of request address components from the cache

    """
    sql = f"""
        SELECT ID, REQUEST_STREET_ADDRESS, REQUEST_CITY, REQUEST_STATE, REQUEST_ZIPCODE FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE}
            WHERE CURRENT_TIMESTAMP()::DATE - REFRESHED_AT::DATE >= {os.getenv('CACHE_TTL_DAYS', 60)}
              AND POD_ID IS NULL
            LIMIT {batch_size}
    """
    df = persistence.get_df(sql)

    pod_id_sql = f"""UPDATE {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE} SET
                       POD_ID = '{pod_id}'
                     WHERE ID IN ('{"','".join(df["ID"])}')"""
    persistence.execute_query(pod_id_sql)
    return df


def delete_queue_bulk(address_run_id):
    """
    Deletes records from the address queue that have been processed
    Args:
        address_run_id: address_run_id

    Returns: None

    """
    logging.info(
        f"Deleting processed addresses for address_run_id {address_run_id} from {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}"
    )

    sql = f"""
        DELETE FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}
            WHERE (upper(trim(REQUEST_STREET_ADDRESS)), upper(trim(REQUEST_CITY)), upper(trim(REQUEST_STATE)), trim(REQUEST_ZIPCODE)) IN (
            select REQUEST_STREET_ADDRESS, REQUEST_CITY, REQUEST_STATE, REQUEST_ZIPCODE
              from {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE}
             where ADDRESS_RUN_ID = '{address_run_id}')"""
    persistence.execute_query(sql)


def delete_queue_single(request_street_address, request_city, request_state, request_zipcode):
    """
    Deletes a single address from the address queue
    Args:
        request_street_address: Street address to be deleted
        request_city: City of address to be deleted
        request_state: State of address to be deleted
        request_zipcode: Zipcode of address to be deleted

    Returns: None

    """
    logging.info(
        f"Deleting address {get_address_string(request_street_address, request_city, request_state, request_zipcode)} from {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}"
    )
    sql = f"""
        DELETE FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}
            WHERE upper(REQUEST_STREET_ADDRESS) = {request_street_address.upper()}
              AND upper(REQUEST_CITY) = {request_city}
              AND upper(REQUEST_STATE) = {request_state}
              AND upper(REQUEST_ZIPCODE) = {request_zipcode}"""
    persistence.execute_query(sql)


def insert_queue_bulk(request_addresses_df):
    """
    Insert incoming request addresses to the address queue
    Args:
        request_addresses_df: DataFrame of request addresses to be inserted
            DataFrame Columns:
                REQUEST_STREET_ADDRESS: the street address component of the requested addresses
                REQUEST_CITY: the city component of the requested addresses
                REQUEST_STATE: the 2-letter abbreviated state component of the requested addresses
                REQUEST_ZIPCODE: the 5-digit zipcode component of the requested addresses

    Returns: None

    """
    logging.info(
        f"Inserting {request_addresses_df.shape[0]} addresses into {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}"
    )
    request_addresses_df["REQUEST_STREET_ADDRESS"] = request_addresses_df[
        "REQUEST_STREET_ADDRESS"
    ].str.upper()
    request_addresses_df["REQUEST_CITY"] = request_addresses_df[
        "REQUEST_CITY"
    ].str.upper()
    request_addresses_df["REQUEST_STATE"] = request_addresses_df[
        "REQUEST_STATE"
    ].str.upper()
    request_addresses_df["ID"] = request_addresses_df.apply(lambda row: get_address_id(row["REQUEST_STREET_ADDRESS"],
                                                                                       row["REQUEST_CITY"],
                                                                                       row["REQUEST_STATE"],
                                                                                       row["REQUEST_ZIPCODE"]), axis=1)
    table_name = persistence.upload_temp_df(
        request_addresses_df,
        "QUEUE_TEMP",
        landing_schema_prefix=get_cache_schema(),
        add_run_schema=False,
        force_create_table=True,
        database=get_cache_database(),
    )
    set_queue_from_temp_table(table_name)


def insert_queue_single(request_street_address, request_city, request_state, request_zipcode):
    """
    Insert single request address to the address queue
    Args:
        request_street_address: the request street address to be inserted
        request_city: the request city to be inserted
        request_state: the request state to be inserted
        request_zipcode: the request zipcode to be inserted

    Returns: None

    """
    logging.info(
        f"Inserting address {get_address_string(request_street_address.upper(), request_city.upper(), request_state.upper(), request_zipcode)} into {get_cache_database()}.{get_cache_schema()}.{ADDRESS_QUEUE_TABLE}"
    )
    request_address_df = pd.DataFrame(
        [[request_street_address.upper(),
          request_city.upper(),
          request_state.upper(),
          request_zipcode]],
        columns=["REQUEST_STREET_ADDRESS", "REQUEST_CITY", "REQUEST_STATE", "REQUEST_ZIPCODE"]
    )
    insert_queue_bulk(request_address_df)


def create_df_from_json(request_street_address, request_city, request_state, request_zipcode, passed_json):
    """
    Creates a DataFrame from a given request address and a JSON response from Earth Define
    Args:
        request_street_address: the request street address that was sent to Earth Define
        request_city: the request city that was sent to Earth Define
        request_state: the request state that was sent to Earth Define
        request_zipcode: the request zipcode that was sent to Earth Define
        passed_json: the Earth Define JSON response for the sent request address

    Returns: DataFrame from parsed JSON

    """
    first_response = passed_json[0]

    df = pd.DataFrame(
        columns=[
            "ID",
            "REQUEST_STREET_ADDRESS",
            "REQUEST_CITY",
            "REQUEST_STATE",
            "REQUEST_ZIPCODE",
            "RESPONSE_STREET_ADDRESS",
            "RESPONSE_CITY",
            "RESPONSE_STATE",
            "RESPONSE_ZIPCODE",
            "RESPONSE_COUNTY",
            "FORMATTED_ADDRESS",
            "RESPONSE_COUNT",
            "SOURCE_DATE",
            "SOURCE",
            "AREA_SQFT",
            "BLD_UUID",
            "STR_UUID",
            "ACCURACY",
            "GEOMETRY",
            "LATITUDE",
            "LONGITUDE",
            "RAW_RESPONSE",
        ]
    )
    df.loc[len(df.index)] = [
        get_address_id(request_street_address, request_city, request_state, request_zipcode),
        request_street_address.strip().upper(),
        request_city.strip().upper(),
        request_state.strip().upper(),
        request_zipcode.strip(),
        f"""{first_response["Address_No"]} {first_response["Street"]}""",
        first_response["City"],
        first_response["STPostal"],
        first_response["ZIP"],
        first_response["County"],
        first_response["Address"],
        len(passed_json),
        first_response["SourceDate"],
        first_response["Source"],
        first_response["Area_SqFt"],
        first_response["bld_UUID"],
        first_response["str_UUID"],
        first_response["score"],
        str(first_response["geometry"]),
        float(first_response["Latitude"]),
        float(first_response["Longitude"]),
        str(passed_json).replace("None", "'None'"),
    ]

    return df


def get_address_id(street_address, city, state, zipcode):
    """
    Gets the unique UUID for an addresses based on its components
    Args:
        street_address: the street address component of the address
        city: the city component of the address
        state: the 2-letter abbreviated state component of the address
        zipcode: the 5-digit zipcode component of the address

    Returns: the UUID of the address

    """
    return str(uuid.uuid5(uuid.NAMESPACE_DNS,
                          street_address.strip().upper() +
                          city.strip().upper() +
                          state.strip().upper() +
                          zipcode.strip()))


def create_df_from_empty(request_street_address, request_city, request_state, request_zipcode):
    """
    Creates a DataFrame with empty Earth Define data from a given request address
    Args:
        request_street_address: the request street address that was sent to Earth Define
        request_city: the request city that was sent to Earth Define
        request_state: the request state that was sent to Earth Define
        request_zipcode: the request zipcode that was sent to Earth Define

    Returns: DataFrame with request address info

    """
    df = pd.DataFrame(
        columns=[
            "ID",
            "REQUEST_STREET_ADDRESS",
            "REQUEST_CITY",
            "REQUEST_STATE",
            "REQUEST_ZIPCODE",
            "RESPONSE_STREET_ADDRESS",
            "RESPONSE_CITY",
            "RESPONSE_STATE",
            "RESPONSE_ZIPCODE",
            "RESPONSE_COUNTY",
            "FORMATTED_ADDRESS",
            "RESPONSE_COUNT",
            "SOURCE_DATE",
            "SOURCE",
            "AREA_SQFT",
            "BLD_UUID",
            "STR_UUID",
            "ACCURACY",
            "GEOMETRY",
            "LATITUDE",
            "LONGITUDE",
            "RAW_RESPONSE",
        ]
    )

    df.loc[len(df.index)] = [
        str(uuid.uuid5(uuid.NAMESPACE_DNS,
                       request_street_address.strip().upper() + request_city.strip().upper() + request_state.strip().upper() + request_zipcode.strip())),
        request_street_address.strip().upper(),
        request_city.strip().upper(),
        request_state.strip().upper(),
        request_zipcode.strip(),
        None,
        None,
        None,
        None,
        None,
        None,
        0,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
    ]

    return df


def handle_geometry_collection_for_raw(raw):
    handled = copy.deepcopy(raw)
    handled["geometry"] = shape(handled["geometry"]).wkb.hex()
    return handled


def get_address(request_street_address, request_city, request_state, request_zipcode):
    """
    Gets a single address record from the address cache
    Args:
        request_street_address: the street address to get from the cache
        request_city: the city of the address to get from the cache
        request_state: the state of the address to get from the cache
        request_zipcode: the zipcode of the address to get from the cache

    Returns: DataFrame of data from cache

    """
    logging.info(
        f"Attempting to get address {request_street_address.upper()}, {request_city}, {request_state} {request_zipcode} from {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE}"
    )
    request_street_address = request_street_address.replace("'", "\\'")
    request_city = request_city.replace("'", "\\'")
    sql = f"""SELECT * FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE}
              WHERE REQUEST_STREET_ADDRESS = '{request_street_address.upper()}'
                AND REQUEST_CITY = '{request_city.upper()}'
                AND REQUEST_STATE = '{request_state.upper()}'
                AND REQUEST_ZIPCODE = '{request_zipcode}'"""
    response_df = persistence.get_df(sql)
    return response_df


def get_address_bulk(request_address_df):
    """
    Gets multiple address records from the address cache
    Args:
        request_address_df: DataFrame of the addresses to retrieve from the cache
            DataFrame Columns:
                REQUEST_STREET_ADDRESS: the street address component of the requested addresses
                REQUEST_CITY: the city component of the requested addresses
                REQUEST_STATE: the 2-letter abbreviated state component of the requested addresses
                REQUEST_ZIPCODE: the 5-digit zipcode component of the requested addresses

    Returns: DataFrame of address data from the cache

    """
    logging.info(
        f"Attempting to get {request_address_df.shape[0]} addresses from {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE}"
    )
    zipped = list(
        zip(request_address_df["REQUEST_STREET_ADDRESS"].tolist(), request_address_df["REQUEST_CITY"].tolist(),
            request_address_df["REQUEST_STATE"].tolist(), request_address_df["REQUEST_ZIPCODE"].tolist()))
    address_list = [', '.join(address) for address in zipped]
    address_list = [address.replace("'", "\\'") for address in address_list]
    sql = f"""SELECT * FROM {get_cache_database()}.{get_cache_schema()}.{ADDRESS_CACHE_TABLE}
                WHERE REQUEST_STREET_ADDRESS || ', ' || REQUEST_CITY || ', ' || REQUEST_STATE || ', ' || REQUEST_ZIPCODE in ('{"','".join(address_list)}')"""
    response_df = persistence.get_df(sql)
    return response_df


def get_address_string(street_address, city, state, zipcode):
    """
    Formats an address based on its components
    Args:
        street_address: the street address component of the address
        city: the city component of the address
        state: the 2-letter abbreviated state component of the address
        zipcode: the 5-digit zipcode component of the address

    Returns: the formatted address

    """
    return f"{street_address}, {city}, {state} {zipcode}"