"""

wi_taxonomy_utils.py

Functions related to working with the SpeciesNet / Wildlife Insights taxonomy.

"""

#%% Imports and constants

import os
import json

import pandas as pd

from copy import deepcopy
from collections import defaultdict
from tqdm import tqdm

from megadetector.utils.path_utils import \
    insert_before_extension, find_images

from megadetector.utils.ct_utils import (
    split_list_into_n_chunks,
    round_floats_in_nested_dict,
    is_list_sorted,
    invert_dictionary,
    sort_list_of_dicts_by_key,
    sort_dictionary_by_value,
)

from megadetector.postprocessing.validate_batch_results import \
    validate_batch_results, ValidateBatchResultsOptions

from megadetector.detection.run_detector import DEFAULT_DETECTOR_LABEL_MAP

md_category_id_to_name = DEFAULT_DETECTOR_LABEL_MAP
md_category_name_to_id = invert_dictionary(md_category_id_to_name)

blank_prediction_string = \
    'f1856211-cfb7-4a5b-9158-c0f72fd09ee6;;;;;;blank'
no_cv_result_prediction_string = \
    'f2efdae9-efb8-48fb-8a91-eccf79ab4ffb;no cv result;no cv result;no cv result;no cv result;no cv result;no cv result'
animal_prediction_string = \
    '1f689929-883d-4dae-958c-3d57ab5b6c16;;;;;;animal'
human_prediction_string = \
    '990ae9dd-7a59-4344-afcb-1b7b21368000;mammalia;primates;hominidae;homo;sapiens;human'
vehicle_prediction_string = \
    'e2895ed5-780b-48f6-8a11-9e27cb594511;;;;;;vehicle'

non_taxonomic_prediction_strings = [blank_prediction_string,
                                    no_cv_result_prediction_string,
                                    animal_prediction_string,
                                    vehicle_prediction_string]

non_taxonomic_prediction_short_strings = [';'.join(s.split(';')[1:-1]) for s in \
                                          non_taxonomic_prediction_strings]

# Ignore some files when generating instances.json from a folder
default_tokens_to_ignore = ['$RECYCLE.BIN']


#%% Miscellaneous taxonomy support functions

def is_valid_prediction_string(s):
    """
    Determine whether [s] is a valid WI prediction string.  Prediction strings look like:

    '90d950db-2106-4bd9-a4c1-777604c3eada;mammalia;rodentia;;;;rodent'

    Args:
        s (str): the string to be tested for validity

    Returns:
        bool: True if this looks more or less like a WI prediction string
    """

    # Note to self... don't get tempted to remove spaces here; spaces are used
    # to indicate subspecies.
    return isinstance(s,str) and (len(s.split(';')) == 7) and (s == s.lower())


def is_valid_taxonomy_string(s):
    """
    Determine whether [s] is a valid 5-token WI taxonomy string.  Taxonomy strings
    look like:

    'mammalia;rodentia;;;;rodent'
    'mammalia;chordata;canidae;canis;lupus dingo'

    Args:
        s (str): the string to be tested for validity

    Returns:
        bool: True if this looks more or less like a WI taxonomy string
    """
    return isinstance(s,str) and (len(s.split(';')) == 5) and (s == s.lower())


def clean_taxonomy_string(s, truncate_multiple_description_strings=True):
    """
    If [s] is a seven-token prediction string, trim the GUID and common name to produce
    a "clean" taxonomy string.  Else if [s] is a five-token string, return it.  Else error.

    Args:
        s (str): the seven- or five-token taxonomy/prediction string to clean
        truncate_multiple_description_strings (bool, optional): we use | to delimit
            multiple descriptions in the same string; if this is True, clean and
            return just the first, else error.

    Returns:
        str: the five-token taxonomy string
    """

    if truncate_multiple_description_strings:
        tokens = s.split('|')
        s = tokens[0]

    if is_valid_taxonomy_string(s):
        return s
    elif is_valid_prediction_string(s):
        tokens = s.split(';')
        assert len(tokens) == 7
        return ';'.join(tokens[1:-1])
    else:
        raise ValueError('Invalid taxonomy string: {}'.format(s))


taxonomy_level_names = \
    ['non-taxonomic','kingdom','phylum','class','order','family','genus','species','subspecies']


def taxonomy_level_to_string(k):
    """
    Maps taxonomy level indices (0 for kindgom, 1 for phylum, etc.) to strings.

    Args:
        k (int): taxonomy level index

    Returns:
        str: taxonomy level string
    """

    assert k >= 0 and k < len(taxonomy_level_names), \
        'Illegal taxonomy level index {}'.format(k)

    return taxonomy_level_names[k]


def taxonomy_level_string_to_index(s):
    """
    Maps strings ('kingdom', 'species', etc.) to level indices.

    Args:
        s (str): taxonomy level string

    Returns:
        int: taxonomy level index
    """

    assert s in taxonomy_level_names, 'Unrecognized taxonomy level string {}'.format(s)
    return taxonomy_level_names.index(s)


def taxonomy_level_index(s):
    """
    Returns the taxonomy level up to which [s] is defined (0 for non-taxnomic, 1 for kingdom,
    2 for phylum, etc.  Empty strings and non-taxonomic strings are treated as level 0.  1 and 2
    will never be returned; "animal" doesn't look like other taxonomic strings, so here we treat
    it as non-taxonomic.

    Args:
        s (str): 5-token or 7-token taxonomy string

    Returns:
        int: taxonomy level
    """

    if s in non_taxonomic_prediction_strings or s in non_taxonomic_prediction_short_strings:
        return 0

    tokens = s.split(';')
    assert len(tokens) in (5,7)

    if len(tokens) == 7:
        tokens = tokens[1:-1]

    # Anything without a class is considered non-taxonomic
    if len(tokens[0]) == 0:
        return 0

    # WI taxonomy strings start at class, so we'll never return 1 (kingdom) or 2 (phylum)
    elif len(tokens[1]) == 0:
        return 3
    elif len(tokens[2]) == 0:
        return 4
    elif len(tokens[3]) == 0:
        return 5
    elif len(tokens[4]) == 0:
        return 6
    # Subspecies are delimited with a space
    elif ' ' not in tokens[4]:
        return 7
    else:
        return 8


def is_taxonomic_prediction_string(s):
    """
    Determines whether [s] is a classification string that has taxonomic properties; this
    does not include, e.g., blanks/vehicles/no cv result.  It also excludes "animal".

    Args:
        s (str): a five- or seven-token taxonomic string

    Returns:
        bool: whether [s] is a taxonomic category
    """

    return (taxonomy_level_index(s) > 0)


def get_kingdom(prediction_string):
    """
    Return the kingdom field from a WI prediction string

    Args:
        prediction_string (str): a string in the semicolon-delimited prediction string format

    Returns:
        str: the kingdom field from the input string
    """
    tokens = prediction_string.split(';')
    assert is_valid_prediction_string(prediction_string)
    return tokens[1]


def is_human_classification(prediction_string):
    """
    Determines whether the input string represents a human classification, which includes a variety
    of common names (hiker, person, etc.)

    Args:
        prediction_string (str): a string in the semicolon-delimited prediction string format

    Returns:
        bool: whether this string corresponds to a human category
    """
    return prediction_string == human_prediction_string or 'homo;sapiens' in prediction_string


def is_vehicle_classification(prediction_string):
    """
    Determines whether the input string represents a vehicle classification.

    Args:
        prediction_string (str): a string in the semicolon-delimited prediction string format

    Returns:
        bool: whether this string corresponds to the vehicle category
    """
    return prediction_string == vehicle_prediction_string


def is_animal_classification(prediction_string):
    """
    Determines whether the input string represents an animal classification, which excludes, e.g.,
    humans, blanks, vehicles, unknowns

    Args:
        prediction_string (str): a string in the semicolon-delimited prediction string format

    Returns:
        bool: whether this string corresponds to an animal category
    """

    if prediction_string == animal_prediction_string:
        return True
    if prediction_string == human_prediction_string or 'homo;sapiens' in prediction_string:
        return False
    if prediction_string == blank_prediction_string:
        return False
    if prediction_string == no_cv_result_prediction_string:
        return False
    if len(get_kingdom(prediction_string)) == 0:
        return False
    return True


def taxonomy_info_to_taxonomy_string(taxonomy_info, include_taxon_id_and_common_name=False):
    """
    Convert a taxonomy record in dict format to a five- or seven-token semicolon-delimited string

    Args:
        taxonomy_info (dict): dict in the format stored in, e.g., taxonomy_string_to_taxonomy_info
        include_taxon_id_and_common_name (bool, optional): by default, this function returns a
            five-token string of latin names; if this argument is True, it includes the leading
            (GUID) and trailing (common name) tokens

    Returns:
        str: string in the format used as keys in, e.g., taxonomy_string_to_taxonomy_info
    """
    s = taxonomy_info['class'] + ';' + \
        taxonomy_info['order'] + ';'  + \
        taxonomy_info['family'] + ';'  + \
        taxonomy_info['genus'] + ';'  + \
        taxonomy_info['species']

    if include_taxon_id_and_common_name:
        s = taxonomy_info['taxon_id'] + ';' + s + ';' + taxonomy_info['common_name']

    return s


#%% Functions used to manipulate results files

def generate_whole_image_detections_for_classifications(classifications_json_file,
                                                        detections_json_file,
                                                        ensemble_json_file=None,
                                                        ignore_blank_classifications=True,
                                                        verbose=True):
    """
    Given a set of classification results in SpeciesNet format that were likely run on
    already-cropped images, generate a file of [fake] detections in SpeciesNet format in which each
    image is covered in a single whole-image detection.

    Args:
        classifications_json_file (str): SpeciesNet-formatted file containing classifications
        detections_json_file (str): SpeciesNet-formatted file to write with detections
        ensemble_json_file (str, optional): SpeciesNet-formatted file to write with detections
            and classfications
        ignore_blank_classifications (bool, optional): use non-top classifications when
            the top classification is "blank" or "no CV result"
        verbose (bool, optional): enable additional debug output

    Returns:
        dict: the contents of [detections_json_file]
    """

    with open(classifications_json_file,'r') as f:
        classification_results = json.load(f)
    predictions = classification_results['predictions']

    output_predictions = []
    ensemble_predictions = []

    # i_prediction = 0; prediction = predictions[i_prediction]
    for i_prediction,prediction in enumerate(predictions):

        output_prediction = {}
        output_prediction['filepath'] = prediction['filepath']
        i_score = 0

        if ignore_blank_classifications:

            while (prediction['classifications']['classes'][i_score] in \
                   (blank_prediction_string,no_cv_result_prediction_string)):

                i_score += 1
                if (i_score >= len(prediction['classifications']['classes'])):

                    if verbose:

                        print('Ignoring blank classifications, but ' + \
                              'image {} has no non-blank values'.format(
                                i_prediction))

                    # Just use the first one
                    i_score = 0
                    break

                # ...if we passed the last prediction

            # ...iterate over classes within this prediction

        # ...if we're supposed to ignore blank classifications

        top_classification = prediction['classifications']['classes'][i_score]
        top_classification_score = prediction['classifications']['scores'][i_score]
        if is_animal_classification(top_classification):
            category_name = 'animal'
        elif is_human_classification(top_classification):
            category_name = 'human'
        else:
            category_name = 'vehicle'

        if category_name == 'human':
            md_category_name = 'person'
        else:
            md_category_name = category_name

        output_detection = {}
        output_detection['label'] = category_name
        output_detection['category'] = md_category_name_to_id[md_category_name]
        output_detection['conf'] = 1.0
        output_detection['bbox'] = [0.0, 0.0, 1.0, 1.0]
        output_prediction['detections'] = [output_detection]
        output_predictions.append(output_prediction)

        ensemble_prediction = {}
        ensemble_prediction['filepath'] = prediction['filepath']
        ensemble_prediction['detections'] = [output_detection]
        ensemble_prediction['prediction'] = top_classification
        ensemble_prediction['prediction_score'] = top_classification_score
        ensemble_prediction['prediction_source'] = 'fake_ensemble_file_utility'
        ensemble_prediction['classifications'] = prediction['classifications']
        ensemble_predictions.append(ensemble_prediction)

    # ...for each image

    ## Write output

    if ensemble_json_file is not None:

        ensemble_output_data = {'predictions':ensemble_predictions}
        with open(ensemble_json_file,'w') as f:
            json.dump(ensemble_output_data,f,indent=1)
        _ = validate_predictions_file(ensemble_json_file)

    output_data = {'predictions':output_predictions}
    with open(detections_json_file,'w') as f:
        json.dump(output_data,f,indent=1)
    return validate_predictions_file(detections_json_file)

# ...def generate_whole_image_detections_for_classifications(...)


def generate_md_results_from_predictions_json(predictions_json_file,
                                              md_results_file=None,
                                              base_folder=None,
                                              max_decimals=5,
                                              convert_human_to_person=True,
                                              convert_homo_species_to_human=True,
                                              verbose=False):
    """
    Generate an MD-formatted .json file from a predictions.json file, generated by the
    SpeciesNet ensemble.  Typically, MD results files use relative paths, and predictions.json
    files use absolute paths, so this function optionally removes the leading string
    [base_folder] from all file names.

    Uses the classification from the "prediction" field if it's available, otherwise
    uses the "classifications" field.

    When using the "prediction" field, records the top class in the "classifications" field to
    a field in each image called "top_classification_common_name".  This is often different
    from the value of  the "prediction" field.

    speciesnet_to_md.py is a command-line driver for this function.

    Args:
        predictions_json_file (str): path to a predictions.json file, or a dict
        md_results_file (str, optional): path to which we should write an MD-formatted .json file
        base_folder (str, optional): leading string to remove from each path in the
            predictions.json file
        max_decimals (int, optional): number of decimal places to which we should round
            all values
        convert_human_to_person (bool, optional): WI predictions.json files sometimes use the
            detection category "human"; MD files usually use "person".  If True, switches "human"
            to "person".
        convert_homo_species_to_human (bool, optional): the ensemble often rolls human predictions
            up to "homo species", which isn't wrong, but looks odd.  This forces these back to
            "homo sapiens".
        verbose (bool, optional): enable additional debug output

    Returns:
        dict: results in MD format
    """

    # Read predictions file
    if isinstance(predictions_json_file,str):
        with open(predictions_json_file,'r') as f:
            predictions = json.load(f)
    else:
        assert isinstance(predictions_json_file,dict)
        predictions = predictions_json_file

    # Round floating-point values (confidence scores, coordinates) to a
    # reasonable number of decimal places
    if (max_decimals is not None) and (max_decimals > 0):
        round_floats_in_nested_dict(predictions, decimal_places=max_decimals)

    predictions = predictions['predictions']
    assert isinstance(predictions,list)

    # Convert backslashes to forward slashes in both filenames and the base folder string
    for im in predictions:
        im['filepath'] = im['filepath'].replace('\\','/')
    if base_folder is not None:
        base_folder = base_folder.replace('\\','/')

    detection_category_id_to_name = {}
    classification_category_name_to_id = {}

    # Keep track of detections that don't have an assigned detection category; these
    # are fake detections we create for non-blank images with non-empty detection lists.
    # We need to go back later and give them a legitimate detection category ID.
    all_unknown_detections = []

    # Create the output images list
    images_out = []

    base_folder_replacements = 0

    # im_in = predictions[0]
    for im_in in predictions:

        im_out = {}

        fn = im_in['filepath']
        if base_folder is not None:
            if fn.startswith(base_folder):
                base_folder_replacements += 1
                fn = fn.replace(base_folder,'',1)

        im_out['file'] = fn

        if 'failures' in im_in:

            im_out['failure'] = str(im_in['failures'])
            im_out['detections'] = None

        else:

            im_out['detections'] = []

            if 'detections' in im_in:

                if len(im_in['detections']) == 0:
                    im_out['detections'] = []
                else:
                    # det_in = im_in['detections'][0]
                    for det_in in im_in['detections']:
                        det_out = {}
                        if det_in['category'] in detection_category_id_to_name:
                            assert detection_category_id_to_name[det_in['category']] == det_in['label']
                        else:
                            detection_category_id_to_name[det_in['category']] = det_in['label']
                        det_out = {}
                        for s in ['category','conf','bbox']:
                            det_out[s] = det_in[s]
                        im_out['detections'].append(det_out)

            # ...if detections are present

            class_to_assign = None
            class_confidence = None
            top_classification_common_name = None

            if 'classifications' in im_in:

                classifications = im_in['classifications']
                assert len(classifications['scores']) == len(classifications['classes'])
                assert is_list_sorted(classifications['scores'],reverse=True)
                class_to_assign = classifications['classes'][0]
                class_confidence = classifications['scores'][0]

                tokens = class_to_assign.split(';')
                assert len(tokens) == 7
                top_classification_common_name = tokens[-1]
                if len(top_classification_common_name) == 0:
                    top_classification_common_name = 'undefined'

            if 'prediction' in im_in:

                class_to_assign = None
                im_out['top_classification_common_name'] = top_classification_common_name
                class_to_assign = im_in['prediction']
                if convert_homo_species_to_human and class_to_assign.endswith('homo species'):
                    class_to_assign = human_prediction_string
                class_confidence = im_in['prediction_score']

            if class_to_assign is not None:

                if class_to_assign == blank_prediction_string:

                    # We have a blank prediction with detections present.  For now, don't do anything
                    # special here, just making a note of this, in case I want to handle this differently
                    # later.
                    if len(im_out['detections']) > 0:
                        pass

                else:

                    assert not class_to_assign.endswith('blank')

                    # This is a scenario that's not captured well by the MD format: no detections present,
                    # but a non-blank prediction.  For now, create a fake detection to handle this prediction.
                    if len(im_out['detections']) == 0:

                        if verbose:
                            print('Warning: creating fake detection for non-blank whole-image classification' + \
                                  ' in {}'.format(im_in['file']))
                        det_out = {}
                        all_unknown_detections.append(det_out)

                        # We will change this to a string-int later
                        det_out['category'] = 'unknown'
                        det_out['conf'] = class_confidence
                        det_out['bbox'] = [0,0,1,1]
                        im_out['detections'].append(det_out)

                # ...if this is/isn't a blank classification

                # Attach that classification to each detection

                # Create a new category ID if necessary
                if class_to_assign in classification_category_name_to_id:
                    classification_category_id = classification_category_name_to_id[class_to_assign]
                else:
                    classification_category_id = str(len(classification_category_name_to_id))
                    classification_category_name_to_id[class_to_assign] = classification_category_id

                for det in im_out['detections']:
                    det['classifications'] = []
                    det['classifications'].append([classification_category_id,class_confidence])

            # ...if we have some type of classification for this image

        # ...if this is/isn't a failure

        images_out.append(im_out)

    # ...for each image

    if base_folder is not None:
        if base_folder_replacements == 0:
            print('Warning: you supplied {} as the base folder, but I made zero replacements'.format(
                base_folder))

    # Fix the 'unknown' category
    if len(all_unknown_detections) > 0:

        if len(detection_category_id_to_name) == 0:
            max_detection_category_id = -1
        else:
            max_detection_category_id = max([int(x) for x in detection_category_id_to_name.keys()])
        unknown_category_id = str(max_detection_category_id + 1)
        detection_category_id_to_name[unknown_category_id] = 'unknown'

        for det in all_unknown_detections:
            assert det['category'] == 'unknown'
            det['category'] = unknown_category_id


    # Sort by filename

    images_out = sort_list_of_dicts_by_key(images_out,'file')

    # Prepare friendly classification names

    classification_category_descriptions = \
        invert_dictionary(classification_category_name_to_id)
    classification_categories_out = {}
    for category_id in classification_category_descriptions.keys():
        category_name = classification_category_descriptions[category_id].split(';')[-1]
        classification_categories_out[category_id] = category_name

    # Prepare the output dict

    detection_categories_out = detection_category_id_to_name
    info = {}
    info['format_version'] = 1.4
    info['detector'] = 'converted_from_predictions_json'

    if convert_human_to_person:
        for k in detection_categories_out.keys():
            if detection_categories_out[k] == 'human':
                detection_categories_out[k] = 'person'

    output_dict = {}
    output_dict['info'] = info
    output_dict['detection_categories'] = detection_categories_out
    output_dict['classification_categories'] = classification_categories_out
    output_dict['classification_category_descriptions'] = classification_category_descriptions
    output_dict['images'] = images_out

    if md_results_file is not None:
        with open(md_results_file,'w') as f:
            json.dump(output_dict,f,indent=1)

        validation_options = ValidateBatchResultsOptions()
        validation_options.raise_errors = True
        _ = validate_batch_results(md_results_file, options=validation_options)

    return output_dict

# ...def generate_md_results_from_predictions_json(...)


def generate_predictions_json_from_md_results(md_results_file,
                                              predictions_json_file,
                                              base_folder=None):
    """
    Generate a predictions.json file from the MD-formatted .json file [md_results_file].  Typically,
    MD results files use relative paths, and predictions.json files use absolute paths, so
    this function optionally prepends [base_folder].  Does not handle classification results in
    MD format, since this is intended to prepare data for passing through the WI classifier.

    md_to_wi.py is a command-line driver for this function.

    Args:
        md_results_file (str): path to an MD-formatted .json file
        predictions_json_file (str): path to which we should write a predictions.json file
        base_folder (str, optional): folder name to prepend to each path in md_results_file,
            to convert relative paths to absolute paths.
    """

    # Validate the input file
    validation_options = ValidateBatchResultsOptions()
    validation_options.raise_errors = True
    validation_options.return_data = True
    md_results = validate_batch_results(md_results_file, options=validation_options)
    category_id_to_name = md_results['detection_categories']

    output_dict = {}
    output_dict['predictions'] = []

    # im = md_results['images'][0]
    for im in md_results['images']:

        prediction = {}
        fn = im['file']
        if base_folder is not None:
            fn = os.path.join(base_folder,fn)
        fn = fn.replace('\\','/')
        prediction['filepath'] = fn
        if 'failure' in im and im['failure'] is not None:
            prediction['failures'] = ['DETECTOR']
        else:
            assert 'detections' in im and im['detections'] is not None
            detections = []
            for det in im['detections']:
                output_det = deepcopy(det)
                output_det['label'] = category_id_to_name[det['category']]
                detections.append(output_det)

            # detections *must* be sorted in descending order by confidence
            detections = sort_list_of_dicts_by_key(detections,'conf', reverse=True)
            prediction['detections'] = detections

        assert len(prediction.keys()) >= 2
        output_dict['predictions'].append(prediction)

    # ...for each image

    output_dir = os.path.dirname(predictions_json_file)
    if len(output_dir) > 0:
        os.makedirs(output_dir,exist_ok=True)
    with open(predictions_json_file,'w') as f:
        json.dump(output_dict,f,indent=1)

# ...def generate_predictions_json_from_md_results(...)


def generate_instances_json_from_folder(folder,
                                        country=None,
                                        admin1_region=None,
                                        lat=None,
                                        lon=None,
                                        output_file=None,
                                        filename_replacements=None,
                                        tokens_to_ignore=default_tokens_to_ignore):
    """
    Generate an instances.json record that contains all images in [folder], optionally
    including location information, in a format suitable for run_model.py.  Optionally writes
    the results to [output_file].

    Args:
        folder (str): the folder to recursively search for images
        country (str, optional): a three-letter country code
        admin1_region (str, optional): an administrative region code, typically a two-letter
            US state code
        lat (float, optional): latitude to associate with all images
        lon (float, optional): longitude to associate with all images
        output_file (str, optional): .json file to which we should write instance records
        filename_replacements (dict, optional): str --> str dict indicating filename substrings
            that should be replaced with other strings.  Replacement occurs *after* converting
            backslashes to forward slashes.
        tokens_to_ignore (list, optional): ignore any images with these tokens in their
            names, typically used to avoid $RECYCLE.BIN.  Can be None.

    Returns:
        dict: dict with at least the field "instances"
    """

    assert os.path.isdir(folder)

    print('Enumerating images in {}'.format(folder))
    image_files_abs = find_images(folder,recursive=True,return_relative_paths=False)

    if tokens_to_ignore is not None:
        n_images_before_ignore_tokens = len(image_files_abs)
        for token in tokens_to_ignore:
            image_files_abs = [fn for fn in image_files_abs if token not in fn]
        print('After ignoring {} tokens, kept {} of {} images'.format(
            len(tokens_to_ignore),len(image_files_abs),n_images_before_ignore_tokens))

    instances = []

    # image_fn_abs = image_files_abs[0]
    for image_fn_abs in image_files_abs:
        instance = {}
        instance['filepath'] = image_fn_abs.replace('\\','/')
        if filename_replacements is not None:
            for s in filename_replacements:
                instance['filepath'] = instance['filepath'].replace(s,filename_replacements[s])
        if country is not None:
            instance['country'] = country
        if admin1_region is not None:
            instance['admin1_region'] = admin1_region
        if lat is not None:
            assert lon is not None, 'Latitude provided without longitude'
            instance['latitude'] = lat
        if lon is not None:
            assert lat is not None, 'Longitude provided without latitude'
            instance['longitude'] = lon
        instances.append(instance)

    to_return = {'instances':instances}

    if output_file is not None:
        output_dir = os.path.dirname(output_file)
        if len(output_dir) > 0:
            os.makedirs(output_dir,exist_ok=True)
        with open(output_file,'w') as f:
            json.dump(to_return,f,indent=1)

    return to_return

# ...def generate_instances_json_from_folder(...)


def split_instances_into_n_batches(instances_json,n_batches,output_files=None):
    """
    Given an instances.json file, split it into batches of equal size.

    Args:
        instances_json (str): input .json file in
        n_batches (int): number of new files to generate
        output_files (list, optional): output .json files for each
            batch.  If supplied, should have length [n_batches].  If not
            supplied, filenames will be generated based on [instances_json].

    Returns:
        list: list of output files that were written; identical to [output_files]
        if it was supplied as input.
    """

    with open(instances_json,'r') as f:
        instances = json.load(f)
    assert isinstance(instances,dict) and 'instances' in instances
    instances = instances['instances']

    if output_files is not None:
        assert len(output_files) == n_batches, \
            'Expected {} output files, received {}'.format(
                n_batches,len(output_files))
    else:
        output_files = []
        for i_batch in range(0,n_batches):
            batch_string = 'batch_{}'.format(str(i_batch).zfill(3))
            output_files.append(insert_before_extension(instances_json,batch_string))

    batches = split_list_into_n_chunks(instances, n_batches)

    for i_batch,batch in enumerate(batches):
        batch_dict = {'instances':batch}
        with open(output_files[i_batch],'w') as f:
            json.dump(batch_dict,f,indent=1)

    print('Wrote {} batches to file'.format(n_batches))

    return output_files

# ...def split_instances_into_n_batches(...)


def merge_prediction_json_files(input_prediction_files,output_prediction_file):
    """
    Merge all predictions.json files in [files] into a single .json file.

    Args:
        input_prediction_files (list): list of predictions.json files to merge
        output_prediction_file (str): output .json file
    """

    predictions = []
    image_filenames_processed = set()

    # input_json_fn = input_prediction_files[0]
    for input_json_fn in tqdm(input_prediction_files):

        assert os.path.isfile(input_json_fn), \
            'Could not find prediction file {}'.format(input_json_fn)
        with open(input_json_fn,'r') as f:
            results_this_file = json.load(f)
        assert isinstance(results_this_file,dict)
        predictions_this_file = results_this_file['predictions']
        for prediction in predictions_this_file:
            image_fn = prediction['filepath']
            assert image_fn not in image_filenames_processed
        predictions.extend(predictions_this_file)

    output_dict = {'predictions':predictions}

    output_dir = os.path.dirname(output_prediction_file)
    if len(output_dir) > 0:
        os.makedirs(output_dir,exist_ok=True)
    with open(output_prediction_file,'w') as f:
        json.dump(output_dict,f,indent=1)

# ...def merge_prediction_json_files(...)


def load_md_or_speciesnet_file(fn,verbose=True):
    """
    Load a .json file that may be in MD or SpeciesNet format.  Typically used so
    SpeciesNet files can be supplied to functions originally written to support MD
    format.

    Args:
        fn (str): a .json file in predictions.json (MD or SpeciesNet) format
        verbose (bool, optional): enable additional debug output

    Returns:
        dict: the contents of [fn], in MD format.
    """

    with open(fn,'r') as f:
        detector_output = json.load(f)

    # If this is a SpeciesNet file, convert to MD format
    if 'predictions' in detector_output:

        if verbose:
            print('This appears to be a SpeciesNet output file, converting to MD format')
        detector_output = generate_md_results_from_predictions_json(predictions_json_file=fn,
                                                                    md_results_file=None,
                                                                    base_folder=None)

    # ...if this is a SpeciesNet file

    assert 'images' in detector_output, \
        'Detector output file should be a json file with an "images" field.'

    return detector_output

# ...def load_md_or_speciesnet_file(...)


def validate_predictions_file(fn,instances=None,verbose=True):
    """
    Validate the predictions.json file [fn].

    Args:
        fn (str): a .json file in predictions.json (SpeciesNet) format
        instances (str or list, optional): a folder, instances.json file,
            or dict loaded from an instances.json file.  If supplied, this
            function will verify that [fn] contains the same number of
            images as [instances].
        verbose (bool, optional): enable additional debug output

    Returns:
        dict: the contents of [fn]
    """

    with open(fn,'r') as f:
        d = json.load(f)
    predictions = d['predictions']

    failures = []

    for im in predictions:
        if 'failures' in im:
            failures.append(im)

    if verbose:
        print('Read predictions for {} images, with {} failure(s)'.format(
            len(d['predictions']),len(failures)))

    if instances is not None:

        if isinstance(instances,str):
            if os.path.isdir(instances):
                instances = generate_instances_json_from_folder(folder=instances)
            elif os.path.isfile(instances):
                with open(instances,'r') as f:
                    instances = json.load(f)
            else:
                raise ValueError('Could not find instances file/folder {}'.format(
                    instances))
        assert isinstance(instances,dict)
        assert 'instances' in instances
        instances = instances['instances']
        if verbose:
            print('Expected results for {} files'.format(len(instances)))
        assert len(instances) == len(predictions), \
            '{} instances expected, {} found'.format(
                len(instances),len(predictions))

        expected_files = set([instance['filepath'] for instance in instances])
        found_files = set([prediction['filepath'] for prediction in predictions])
        assert expected_files == found_files

    # ...if a list of instances was supplied

    return d

# ...def validate_predictions_file(...)


#%% Functions related to geofencing

def find_geofence_adjustments(ensemble_json_file,use_latin_names=False):
    """
    Count the number of instances of each unique change made by the geofence.

    Args:
        ensemble_json_file (str): SpeciesNet-formatted .json file produced
            by the full ensemble.
        use_latin_names (bool, optional): return a mapping using binomial names
            rather than common names.

    Returns:
        dict: maps strings that look like "puma,felidae family" to integers,
            where that entry would indicate the number of times that "puma" was
            predicted, but mapped to family level by the geofence.  Sorted in
            descending order by count.
    """

    # Load and validate ensemble results
    ensemble_results = validate_predictions_file(ensemble_json_file)

    assert isinstance(ensemble_results,dict)
    predictions = ensemble_results['predictions']

    # Maps comma-separated pairs of common names (or binomial names) to
    # the number of times that transition (first --> second) happened
    rollup_pair_to_count = defaultdict(int)

    # prediction = predictions[0]
    for prediction in tqdm(predictions):

        if 'failures' in prediction and \
            prediction['failures'] is not None and \
            len(prediction['failures']) > 0:
                continue

        assert 'prediction_source' in prediction, \
            'Prediction present without [prediction_source] field, are you sure this ' + \
            'is an ensemble output file?'

        if 'geofence' in prediction['prediction_source']:

            classification_taxonomy_string = \
                prediction['classifications']['classes'][0]
            prediction_taxonomy_string = prediction['prediction']
            assert is_valid_prediction_string(classification_taxonomy_string)
            assert is_valid_prediction_string(prediction_taxonomy_string)

            # Typical examples:
            # '86f5b978-4f30-40cc-bd08-be9e3fba27a0;mammalia;rodentia;sciuridae;sciurus;carolinensis;eastern gray squirrel'
            # 'e4d1e892-0e4b-475a-a8ac-b5c3502e0d55;mammalia;rodentia;sciuridae;;;sciuridae family'
            classification_common_name = classification_taxonomy_string.split(';')[-1]
            prediction_common_name = prediction_taxonomy_string.split(';')[-1]
            classification_binomial_name = classification_taxonomy_string.split(';')[-2]
            prediction_binomial_name = prediction_taxonomy_string.split(';')[-2]

            input_name = classification_binomial_name if use_latin_names else \
                classification_common_name
            output_name = prediction_binomial_name if use_latin_names else \
                prediction_common_name

            rollup_pair = input_name.strip() + ',' + output_name.strip()
            rollup_pair_to_count[rollup_pair] += 1

        # ...if we made a geofencing change

    # ...for each prediction

    rollup_pair_to_count = sort_dictionary_by_value(rollup_pair_to_count,reverse=True)

    return rollup_pair_to_count

# ...def find_geofence_adjustments(...)


def generate_geofence_adjustment_html_summary(rollup_pair_to_count,min_count=10):
    """
    Given a list of geofence rollups, likely generated by find_geofence_adjustments,
    generate an HTML summary of the changes made by geofencing.  The resulting HTML
    is wrapped in <div>, but not, for example, in <html> or <body>.

    Args:
        rollup_pair_to_count (dict): list of changes made by geofencing, see
            find_geofence_adjustments for details
        min_count (int, optional): minimum number of changes a pair needs in order
            to be included in the report.
    """

    geofence_footer = ''

    # Restrict to the list of taxa that were impacted by geofencing
    rollup_pair_to_count = \
        {key: value for key, value in rollup_pair_to_count.items() if value >= min_count}

    # rollup_pair_to_count is sorted in descending order by count
    assert is_list_sorted(list(rollup_pair_to_count.values()),reverse=True)

    if len(rollup_pair_to_count) > 0:

        geofence_footer = \
            '<h3>Geofence changes that occurred more than {} times</h3>\n'.format(min_count)
        geofence_footer += '<div class="contentdiv">\n'

        print('\nRollup changes with count > {}:'.format(min_count))
        for rollup_pair in rollup_pair_to_count.keys():
            count = rollup_pair_to_count[rollup_pair]
            rollup_pair_s = rollup_pair.replace(',',' --> ')
            print('{}: {}'.format(rollup_pair_s,count))
            rollup_pair_html = rollup_pair.replace(',',' &rarr; ')
            geofence_footer += '{} ({})<br/>\n'.format(rollup_pair_html,count)

        geofence_footer += '</div>\n'

    return geofence_footer

# ...def generate_geofence_adjustment_html_summary(...)


#%% TaxonomyHandler class

class TaxonomyHandler:
    """
    Handler for taxonomy mapping and geofencing operations.
    """

    def __init__(self, taxonomy_file, geofencing_file, country_code_file):
        """
        Initialize TaxonomyHandler with taxonomy information.

        Args:
            taxonomy_file (str): .csv file containing the SpeciesNet (or WI) taxonomy,
                as seven-token taxonomic specifiers.  Distributed with the SpeciesNet model.
            geofencing_file (str): .json file containing the SpeciesNet geofencing rules.
                Distributed with the SpeciesNet model.
            country_code_file: .csv file mapping country codes to names.  Should include columns
                called "name" and "alpha-3".  A compatible file is available at
                https://github.com/lukes/ISO-3166-Countries-with-Regional-Codes
        """

        #: Maps a taxonomy string (e.g. mammalia;cetartiodactyla;cervidae;odocoileus;virginianus) to
        #: a dict with keys taxon_id, common_name, kingdom, phylum, class, order, family, genus, species
        self.taxonomy_string_to_taxonomy_info = None

        #: Maps a binomial name (one, two, or three ws-delimited tokens) to the same dict described above.
        self.binomial_name_to_taxonomy_info = None

        #: Maps a common name to the same dict described above
        self.common_name_to_taxonomy_info = None

        #: Dict mapping 5-token semicolon-delimited taxonomy strings to geofencing rules
        self.taxonomy_string_to_geofencing_rules = None

        #: Maps lower-case country names to upper-case country codes
        self.country_to_country_code = None

        #: Maps upper-case country codes to lower-case country names
        self.country_code_to_country = None

        self._load_taxonomy_info(taxonomy_file=taxonomy_file)
        self._initialize_geofencing(geofencing_file=geofencing_file,
                                    country_code_file=country_code_file)


    def _load_taxonomy_info(self, taxonomy_file):
        """
        Load WI/SpeciesNet taxonomy information from a .csv file.  Stores information in the
        instance dicts [taxonomy_string_to_taxonomy_info], [binomial_name_to_taxonomy_info],
        and [common_name_to_taxonomy_info].

        Args:
            taxonomy_file (str): .csv file containing the SpeciesNet (or WI) taxonomy,
                as seven-token taxonomic specifiers.  Distributed with the SpeciesNet model.
        """

        """
        Taxonomy keys are five-token taxonomy strings, e.g.:

        'mammalia;cetartiodactyla;cervidae;odocoileus;virginianus'

        Taxonomy values are seven-token strings w/Taxon IDs and common names, e.g.:

        '5c7ce479-8a45-40b3-ae21-7c97dfae22f5;mammalia;cetartiodactyla;cervidae;odocoileus;virginianus;white-tailed deer'
        """

        with open(taxonomy_file,'r') as f:
            taxonomy_lines = f.readlines()
        taxonomy_lines = [s.strip() for s in taxonomy_lines]

        self.taxonomy_string_to_taxonomy_info = {}
        self.binomial_name_to_taxonomy_info = {}
        self.common_name_to_taxonomy_info = {}

        five_token_string_to_seven_token_string = {}

        for line in taxonomy_lines:
            tokens = line.split(';')
            assert len(tokens) == 7, 'Illegal line {} in taxonomy file {}'.format(
                line,taxonomy_file)
            five_token_string = ';'.join(tokens[1:-1])
            assert len(five_token_string.split(';')) == 5
            five_token_string_to_seven_token_string[five_token_string] = line

        for taxonomy_string in five_token_string_to_seven_token_string.keys():

            taxonomy_string = taxonomy_string.lower()

            taxon_info = {}
            extended_string = five_token_string_to_seven_token_string[taxonomy_string]
            tokens = extended_string.split(';')
            assert len(tokens) == 7
            taxon_info['taxon_id'] = tokens[0]
            assert len(taxon_info['taxon_id']) == 36
            taxon_info['kingdom'] = 'animal'
            taxon_info['phylum'] = 'chordata'
            taxon_info['class'] = tokens[1]
            taxon_info['order'] = tokens[2]
            taxon_info['family'] = tokens[3]
            taxon_info['genus'] = tokens[4]
            taxon_info['species'] = tokens[5]
            taxon_info['common_name'] = tokens[6]

            if taxon_info['common_name'] != '':
                self.common_name_to_taxonomy_info[taxon_info['common_name']] = taxon_info

            self.taxonomy_string_to_taxonomy_info[taxonomy_string] = taxon_info

            binomial_name = None
            if len(tokens[4]) > 0 and len(tokens[5]) > 0:
                # strip(), but don't remove spaces from the species name;
                # subspecies are separated with a space, e.g. canis;lupus dingo
                binomial_name = tokens[4].strip() + ' ' + tokens[5].strip()
            elif len(tokens[4]) > 0:
                binomial_name = tokens[4].strip()
            elif len(tokens[3]) > 0:
                binomial_name = tokens[3].strip()
            elif len(tokens[2]) > 0:
                binomial_name = tokens[2].strip()
            elif len(tokens[1]) > 0:
                binomial_name = tokens[1].strip()
            if binomial_name is None:
                # print('Warning: no binomial name for {}'.format(taxonomy_string))
                pass
            else:
                self.binomial_name_to_taxonomy_info[binomial_name] = taxon_info

            taxon_info['binomial_name'] = binomial_name

        # ...for each taxonomy string in the file

        print('Created {} records in taxonomy_string_to_taxonomy_info'.format(len(self.taxonomy_string_to_taxonomy_info)))
        print('Created {} records in common_name_to_taxonomy_info'.format(len(self.common_name_to_taxonomy_info)))

    # ...def _load_taxonomy_info(...)


    def _initialize_geofencing(self, geofencing_file, country_code_file):
        """
        Load geofencing information from a .json file, and country code mappings from
        a .csv file.  Stores results in the instance tables [taxonomy_string_to_geofencing_rules],
        [country_to_country_code], and [country_code_to_country].

        Args:
            geofencing_file (str): .json file with geofencing rules
            country_code_file (str): .csv file with country code mappings, in columns
                called "name" and "alpha-3", e.g. from
                https://github.com/lukes/ISO-3166-Countries-with-Regional-Codes/blob/master/all/all.csv
        """

        # Read country code information
        country_code_df = pd.read_csv(country_code_file)
        self.country_to_country_code = {}
        self.country_code_to_country = {}
        for i_row,row in country_code_df.iterrows():
            self.country_to_country_code[row['name'].lower()] = row['alpha-3'].upper()
            self.country_code_to_country[row['alpha-3'].upper()] = row['name'].lower()

        # Read geofencing information
        with open(geofencing_file,'r',encoding='utf-8') as f:
            self.taxonomy_string_to_geofencing_rules = json.load(f)

        """
        Geofencing keys are five-token taxonomy strings, e.g.:

        'mammalia;cetartiodactyla;cervidae;odocoileus;virginianus'

        Geofencing values are tables mapping allow/block to country codes, optionally including region/state codes, e.g.:

        {'allow': {
          'ALA': [],
          'ARG': [],
          ...
          'SUR': [],
          'TTO': [],
          'USA': ['AL',
           'AR',
           'AZ',
           ...
        }
        """

        # Validate

        # species_string = next(iter(taxonomy_string_to_geofencing_rules.keys()))
        for species_string in self.taxonomy_string_to_geofencing_rules.keys():

            species_rules = self.taxonomy_string_to_geofencing_rules[species_string]

            if len(species_rules.keys()) > 1:
                print('Warning: taxon {} has both allow and block rules'.format(species_string))

            for rule_type in species_rules.keys():

                assert rule_type in ('allow','block')
                all_country_rules_this_species = species_rules[rule_type]

                for country_code in all_country_rules_this_species.keys():

                    assert country_code in self.country_code_to_country
                    region_rules = all_country_rules_this_species[country_code]
                    # Right now we only have regional rules for the USA; these may be part of
                    # allow or block rules.
                    if len(region_rules) > 0:
                        assert country_code == 'USA'

                # ...for each country code in this rule set

            # ...for each rule set for this species

        # ...for each species

    # ...def _initialize_geofencing(...)


    def _parse_region_code_list(self, codes):
        """
        Turn a list of country or state codes in string, delimited string, or list format
        into a list.  Also does basic validity checking.
        """

        if not isinstance(codes,list):

            assert isinstance(codes,str)

            codes = codes.strip()

            # This is just a single codes
            if ',' not in codes:
                codes = [codes]
            else:
                codes = codes.split(',')
            codes = [c.strip() for c in codes]

        assert isinstance(codes,list)

        codes = [c.upper().strip() for c in codes]

        for c in codes:
            assert len(c) in (2,3)

        return codes

    # ...def _parse_region_code_list(...)


    def generate_csv_rows_to_block_all_countries_except(self, species_string, block_except_list):
        """
        Generate rows in the format expected by geofence_fixes.csv, representing a list of
        allow and block rules to block all countries currently allowed for this species
        except [allow_countries], and add allow rules these countries.

        Args:
            species_string (str): five-token taxonomy string
            block_except_list (list): list of country codes not to block

        Returns:
            list of str: strings compatible with geofence_fixes.csv
        """

        assert is_valid_taxonomy_string(species_string), \
            '{} is not a valid taxonomy string'.format(species_string)

        assert self.taxonomy_string_to_geofencing_rules is not None, \
            'Initialize geofencing prior to species lookup'
        assert self.taxonomy_string_to_taxonomy_info is not None, \
            'Initialize taxonomy lookup prior to species lookup'

        geofencing_rules_this_species = \
            self.taxonomy_string_to_geofencing_rules[species_string]

        allowed_countries = []
        if 'allow' in geofencing_rules_this_species:
            allowed_countries.extend(geofencing_rules_this_species['allow'])

        blocked_countries = []
        if 'block' in geofencing_rules_this_species:
            blocked_countries.extend(geofencing_rules_this_species['block'])

        block_except_list = self._parse_region_code_list(block_except_list)

        countries_to_block = []
        countries_to_allow = []

        # country = allowed_countries[0]
        for country in allowed_countries:
            if country not in block_except_list and country not in blocked_countries:
                countries_to_block.append(country)

        for country in block_except_list:
            if country in blocked_countries:
                raise ValueError("I can't allow a country that has already been blocked")
            if country not in allowed_countries:
                countries_to_allow.append(country)

        rows = self.generate_csv_rows_for_species(species_string,
                                             allow_countries=countries_to_allow,
                                             block_countries=countries_to_block)

        return rows

    # ...def generate_csv_rows_to_block_all_countries_except(...)


    def generate_csv_rows_for_species(self, species_string,
                                      allow_countries=None,
                                      block_countries=None,
                                      allow_states=None,
                                      block_states=None):
        """
        Generate rows in the format expected by geofence_fixes.csv, representing a list of
        allow and/or block rules for the specified species and countries/states.  Does not check
        that the rules make sense; e.g. nothing will stop you in this function from both allowing
        and blocking a country.

        Args:
            species_string (str): five-token string in semicolon-delimited WI taxonomy format
            allow_countries (list or str, optional): three-letter country codes, list of
                country codes, or comma-separated list of country codes to allow
            block_countries (list or str, optional): three-letter country codes, list of
                country codes, or comma-separated list of country codes to block
            allow_states (list or str, optional): two-letter state codes, list of
                state codes, or comma-separated list of state codes to allow
            block_states (list or str, optional): two-letter state code, list of
                state codes, or comma-separated list of state codes to block

        Returns:
            list of str: lines ready to be pasted into geofence_fixes.csv
        """

        assert is_valid_taxonomy_string(species_string), \
            '{} is not a valid taxonomy string'.format(species_string)

        lines = []

        if allow_countries is not None:
            allow_countries = self._parse_region_code_list(allow_countries)
            for country in allow_countries:
                lines.append(species_string + ',allow,' + country + ',')

        if block_countries is not None:
            block_countries = self._parse_region_code_list(block_countries)
            for country in block_countries:
                lines.append(species_string + ',block,' + country + ',')

        if allow_states is not None:
            allow_states = self._parse_region_code_list(allow_states)
            for state in allow_states:
                lines.append(species_string + ',allow,USA,' + state)

        if block_states is not None:
            block_states = self._parse_region_code_list(block_states)
            for state in block_states:
                lines.append(species_string + ',block,USA,' + state)

        return lines

    # ...def generate_csv_rows_for_species(...)


    def species_string_to_canonical_species_string(self, species):
        """
        Convert a string that may be a 5-token species string, a binomial name,
        or a common name into a 5-token species string, using taxonomic lookup.

        Args:
            species (str): 5-token species string, binomial name, or common name

        Returns:
            str: common name

        Raises:
            ValueError: if [species] is not in our dictionary
        """

        species = species.lower().strip()

        # Turn "species" into a taxonomy string

        # If this is already a taxonomy string...
        if len(species.split(';')) == 5:
            taxonomy_string = species
        # If this is a common name...
        elif species in self.common_name_to_taxonomy_info:
            taxonomy_info = self.common_name_to_taxonomy_info[species]
            taxonomy_string = taxonomy_info_to_taxonomy_string(taxonomy_info)
        # If this is a binomial name...
        elif (species in self.binomial_name_to_taxonomy_info):
            taxonomy_info = self.binomial_name_to_taxonomy_info[species]
            taxonomy_string = taxonomy_info_to_taxonomy_string(taxonomy_info)
        else:
            raise ValueError('Could not find taxonomic information for {}'.format(species))

        return taxonomy_string

    # ...def species_string_to_canonical_species_string(...)


    def species_string_to_taxonomy_info(self,species):
        """
        Convert a string that may be a 5-token species string, a binomial name,
        or a common name into a taxonomic info dictionary, using taxonomic lookup.

        Args:
            species (str): 5-token species string, binomial name, or common name

        Returns:
            dict: taxonomy information

        Raises:
            ValueError: if [species] is not in our dictionary
        """

        species = species.lower().strip()
        canonical_string = self.species_string_to_canonical_species_string(species)
        return self.taxonomy_string_to_taxonomy_info[canonical_string]


    def species_allowed_in_country(self, species, country, state=None, return_status=False):
        """
        Determines whether [species] is allowed in [country], according to
        already-initialized geofencing rules.

        Args:
            species (str): can be a common name, a binomial name, or a species string
            country (str): country name or three-letter code
            state (str, optional): two-letter US state code
            return_status (bool, optional): by default, this function returns a bool;
                if you want to know *why* [species] is allowed/not allowed, settings
                return_status to True will return additional information.

        Returns:
            bool or str: typically returns True if [species] is allowed in [country], else
            False.  Returns a more detailed string if return_status is set.
        """

        assert self.taxonomy_string_to_geofencing_rules is not None, \
            'Initialize geofencing prior to species lookup'
        assert self.taxonomy_string_to_taxonomy_info is not None, \
            'Initialize taxonomy lookup prior to species lookup'

        taxonomy_string = self.species_string_to_canonical_species_string(species)

        # Normalize [state]

        if state is not None:
            state = state.upper()
            assert len(state) == 2

        # Turn "country" into a country code

        if len(country) == 3:
            assert country.upper() in self.country_code_to_country
            country = country.upper()
        else:
            assert country.lower() in self.country_to_country_code
            country = self.country_to_country_code[country.lower()]

        country_code = country.upper()

        # Species with no rules are allowed everywhere
        if taxonomy_string not in self.taxonomy_string_to_geofencing_rules:
            status = 'allow_by_default'
            if return_status:
                return status
            else:
                return True

        geofencing_rules_this_species = self.taxonomy_string_to_geofencing_rules[taxonomy_string]
        allowed_countries = []
        blocked_countries = []

        rule_types_this_species = list(geofencing_rules_this_species.keys())
        for rule_type in rule_types_this_species:
            assert rule_type in ('allow','block')

        if 'block' in rule_types_this_species:
            blocked_countries = list(geofencing_rules_this_species['block'])
        if 'allow' in rule_types_this_species:
            allowed_countries = list(geofencing_rules_this_species['allow'])

        status = None

        # The convention is that block rules win over allow rules
        if country_code in blocked_countries:
            if country_code in allowed_countries:
                status = 'blocked_over_allow'
            else:
                status = 'blocked'
        elif country_code in allowed_countries:
            status = 'allowed'
        elif len(allowed_countries) > 0:
            # The convention is that if allow rules exist, any country not on that list
            # is blocked.
            status = 'block_not_on_country_allow_list'
        else:
            # Only block rules exist for this species, and they don't include this country
            assert len(blocked_countries) > 0
            status = 'allow_not_on_block_list'

        # Now let's see whether we have to deal with any regional rules.
        #
        # Right now regional rules only exist for the US.
        if (country_code == 'USA') and ('USA' in geofencing_rules_this_species[rule_type]):

            if state is None:

                state_list = geofencing_rules_this_species[rule_type][country_code]
                if len(state_list) > 0:
                    assert status.startswith('allow')
                    status = 'allow_no_state'

            else:

                state_list = geofencing_rules_this_species[rule_type][country_code]

                if state in state_list:
                    # If the state is on the list, do what the list says
                    if rule_type == 'allow':
                        status = 'allow_on_state_allow_list'
                    else:
                        status = 'block_on_state_block_list'
                else:
                    # If the state is not on the list, do the opposite of what the list says
                    if rule_type == 'allow':
                        status = 'block_not_on_state_allow_list'
                    else:
                        status = 'allow_not_on_state_block_list'

        if return_status:
            return status
        else:
            if status.startswith('allow'):
                return True
            else:
                assert status.startswith('block')
                return False

    # ...def species_allowed_in_country(...)


    def export_geofence_data_to_csv(self, csv_fn=None, include_common_names=True):
        """
        Converts the geofence .json representation into an equivalent .csv representation,
        with one taxon per row and one region per column.  Empty values indicate non-allowed
        combinations, positive numbers indicate allowed combinations.  Negative values
        are reserved for specific non-allowed combinations.

        Args:
            csv_fn (str): output .csv file
            include_common_names (bool, optional): include a column for common names

        Returns:
            dataframe: the pandas representation of the csv output file
        """

        all_taxa = sorted(list(self.taxonomy_string_to_geofencing_rules.keys()))
        print('Preparing geofencing export for {} taxa'.format(len(all_taxa)))

        all_regions = set()

        # taxon = all_taxa[0]
        for taxon in all_taxa:

            taxon_rules = self.taxonomy_string_to_geofencing_rules[taxon]
            for rule_type in taxon_rules.keys():

                assert rule_type in ('allow','block')
                all_country_rules_this_species = taxon_rules[rule_type]

                for country_code in all_country_rules_this_species.keys():
                    all_regions.add(country_code)
                    assert country_code in self.country_code_to_country
                    assert len(country_code) == 3
                    region_rules = all_country_rules_this_species[country_code]
                    if len(region_rules) > 0:
                        assert country_code == 'USA'
                        for region_name in region_rules:
                            assert len(region_name) == 2
                            assert isinstance(region_name,str)
                            all_regions.add(country_code + ':' + region_name)

        all_regions = sorted(list(all_regions))

        print('Found {} regions'.format(len(all_regions)))

        n_allowed = 0
        df = pd.DataFrame(index=all_taxa,columns=all_regions)
        # df = df.fillna(np.nan)

        for taxon in tqdm(all_taxa):
            for region in all_regions:
                tokens = region.split(':')
                country_code = tokens[0]
                state_code = None
                if len(tokens) > 1:
                    state_code = tokens[1]
                allowed = self.species_allowed_in_country(species=taxon,
                                                     country=country_code,
                                                     state=state_code,
                                                     return_status=False)
                if allowed:
                    n_allowed += 1
                    df.loc[taxon,region] = 1

            # ...for each region

        # ...for each taxon

        print('Allowed {} of {} combinations'.format(n_allowed,len(all_taxa)*len(all_regions)))

        # Before saving, convert columns with numeric values to integers
        for col in df.columns:
            # Check whether each column has any non-NaN values that could be integers
            if df[col].notna().any() and pd.to_numeric(df[col], errors='coerce').notna().any():
                # Convert column to Int64 type (pandas nullable integer type)
                df[col] = pd.to_numeric(df[col], errors='coerce').astype('Int64')

        if include_common_names:
            df.insert(loc=0,column='common_name',value='')
            for taxon in all_taxa:
                if taxon in self.taxonomy_string_to_taxonomy_info:
                    taxonomy_info = self.taxonomy_string_to_taxonomy_info[taxon]
                    common_name = taxonomy_info['common_name']
                    assert isinstance(common_name,str) and len(common_name) < 50
                    df.loc[taxon,'common_name'] = common_name

        if csv_fn is not None:
            df.to_csv(csv_fn,index=True,header=True)

        return df

    # ...def export_geofence_data_to_csv(...)

# ...class TaxonomyHandler
