from dipy.align import resample
from dipy.segment.clustering import QuickBundles
from dipy.segment.metric import AveragePointwiseEuclideanMetric
from dipy.segment.featurespeed import ResampleFeature
from dipy.io.streamline import (
    load_tractogram, save_tractogram, StatefulTractogram, Space)
from dipy.data.fetcher import _make_fetcher
import dipy.data as dpd
from AFQ.utils.path import drop_extension, apply_cmd_to_afq_derivs

import os
import os.path as op
import json
from glob import glob
import shutil

import numpy as np
import pandas as pd
import logging
import time
from tqdm import tqdm

import warnings
import nibabel as nib
import boto3
from botocore import UNSIGNED
from botocore.client import Config


# capture templateflow resource warning and log
default_warning_format = warnings.formatwarning
try:
    warnings.formatwarning = lambda msg, *args, **kwargs: f'{msg}'
    logging.captureWarnings(True)
    pywarnings_logger = logging.getLogger('py.warnings')
    console_handler = logging.StreamHandler()
    console_handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
    pywarnings_logger.addHandler(console_handler)

    warnings.filterwarnings(
        "default", category=ResourceWarning,
        module="templateflow")

    from templateflow import api as tflow
finally:
    logging.captureWarnings(False)
    warnings.formatwarning = default_warning_format


__all__ = ["fetch_callosum_templates", "read_callosum_templates",
           "fetch_or_templates", "read_or_templates",
           "fetch_templates", "read_templates",
           "fetch_stanford_hardi_tractography",
           "read_stanford_hardi_tractography",
           "organize_stanford_data",
           "fetch_stanford_hardi_lv1"]


# Set a user-writeable file-system location to put files:
if 'AFQ_HOME' in os.environ:
    afq_home = os.environ['AFQ_HOME']
else:
    afq_home = op.join(op.expanduser('~'), 'AFQ_data')

baseurl = "https://ndownloader.figshare.com/files/"


def _make_reusable_fetcher(name, folder, baseurl, remote_fnames, local_fnames,
                           doc="", md5_list=None, **make_fetcher_kwargs):
    def fetcher():
        all_files_downloaded = True
        for fname in local_fnames:
            if not op.exists(op.join(folder, fname)):
                all_files_downloaded = False
        if all_files_downloaded:
            files = {}
            for i, (f, n), in enumerate(zip(remote_fnames, local_fnames)):
                files[n] = (baseurl + f, md5_list[i] if
                            md5_list is not None else None)
            return files, folder
        else:
            return _make_fetcher(
                name, folder, baseurl, remote_fnames, local_fnames,
                doc=doc, **make_fetcher_kwargs)()
    fetcher.__name__ = name
    fetcher.__doc__ = doc
    return fetcher


def _fetcher_to_template(fetcher, as_img=False, resample_to=False):
    if isinstance(resample_to, str):
        resample_to = nib.load(resample_to)
    files, folder = fetcher()
    template_dict = {}
    for f in files:
        img = op.join(folder, f)
        if as_img:
            img = nib.load(img)
        if resample_to:
            img = nib.Nifti1Image(resample(img.get_fdata(),
                                           resample_to,
                                           img.affine,
                                           resample_to.affine).get_fdata(),
                                  resample_to.affine)
        template_dict[drop_extension(f)] = img
    return template_dict


callosum_fnames = ["Callosum_midsag.nii.gz",
                   "L_AntFrontal.nii.gz",
                   "L_Motor.nii.gz",
                   "L_Occipital.nii.gz",
                   "L_Orbital.nii.gz",
                   "L_PostParietal.nii.gz",
                   "L_SupFrontal.nii.gz",
                   "L_SupParietal.nii.gz",
                   "L_Temporal.nii.gz",
                   "R_AntFrontal.nii.gz",
                   "R_Motor.nii.gz",
                   "R_Occipital.nii.gz",
                   "R_Orbital.nii.gz",
                   "R_PostParietal.nii.gz",
                   "R_SupFrontal.nii.gz",
                   "R_SupParietal.nii.gz",
                   "R_Temporal.nii.gz"]

callosum_remote_fnames = ["5273794", "5273797", "5273800", "5273803",
                          "5273806", "5273809", "5273812", "5273815",
                          "5273821", "5273818", "5273824", "5273827",
                          "5273830", "5273833", "5273836", "5273839",
                          "5273842"]

callosum_md5_hashes = ["709fa90baadeacd64f1d62b5049a4125",
                       "987c6169de807c4e93dc2cbd7a25d506",
                       "0da114123d0b0097b96fe450a459550b",
                       "6d845bd10504f67f1dc17f9000076d7e",
                       "e16c7873ef4b08d26b77ef746dab8237",
                       "47193fd4df1ea17367817466de798b90",
                       "7e78bf9671e6945f4b2f5e7c30595a3c",
                       "8adbb947377ff7b484c88d8c0ffc2125",
                       "0fd981a4d0847e0642ff96e84fe44e47",
                       "87c4855efa406d8fb004cffb8259180e",
                       "c7969bcf5f2343fd9ce9c49b336cf14c",
                       "bb4372b88991932150205ffb22aa6cb7",
                       "d198d4e7db18ddc7236cf143ecb8342e",
                       "d0f6edef64b0c710c92e634496085dda",
                       "85eaee44665f244db5adae2e259833f6",
                       "25f24eb22879a05d12bda007c81ea55a",
                       "2664e0b8c2d9c59f13649a89bfcce399"]

fetch_callosum_templates = _make_reusable_fetcher(
    "fetch_callosum_templates",
    op.join(afq_home,
            'callosum_templates'),
    baseurl, callosum_remote_fnames,
    callosum_fnames,
    md5_list=callosum_md5_hashes,
    doc="Download AFQ callosum templates")


def read_callosum_templates(as_img=True, resample_to=False):
    """Load AFQ callosum templates from file

    Parameters
    ----------
    as_img : bool, optional
        If True, values are `Nifti1Image`. Otherwise, values are
        paths to Nifti files. Default: True
    resample_to : str or nibabel image class instance, optional
        A template image to resample to. Typically, this should be the
        template to which individual-level data are registered. Defaults to
        the MNI template. Default: False

    Returns
    -------
    dict with: keys: names of template ROIs and values: nibabel Nifti1Image
    objects from each of the ROI nifti files.
    """
    logger = logging.getLogger('AFQ')

    logger.debug('loading callosum templates')
    tic = time.perf_counter()

    template_dict = _fetcher_to_template(
        fetch_callosum_templates,
        as_img=as_img,
        resample_to=resample_to)

    toc = time.perf_counter()
    logger.debug(f'callosum templates loaded in {toc - tic:0.4f} seconds')

    return template_dict


##########################################################################
# Pediatric templates:
# --------------------
#
# Templates include the UNC 0-1-2 Infant Atlases and waypoint ROIs matched
# to the atlas.
#
# Information on:
#
# - atlas: https://www.nitrc.org/projects/pediatricatlas
#
# - pediatric templates: https://github.com/yeatmanlab/AFQ/tree/babyAFQ
#
# Templates downloaded from:
#
# - https://figshare.com/articles/dataset/ROIs_probabilistic_maps_and_transform_data_for_pediatric_automated_fiber_quantification/13027487  # noqa
#

pediatric_fnames = [
    "ATR_roi1_L.nii.gz", "ATR_roi1_R.nii.gz",
    "ATR_roi2_L.nii.gz", "ATR_roi2_R.nii.gz",
    "ATR_roi3_L.nii.gz", "ATR_roi3_R.nii.gz",
    "CGC_roi1_L.nii.gz", "CGC_roi1_R.nii.gz",
    "CGC_roi2_L.nii.gz", "CGC_roi2_R.nii.gz",
    "CGC_roi3_L.nii.gz", "CGC_roi3_R.nii.gz",
    "CST_roi1_L.nii.gz", "CST_roi1_R.nii.gz",
    "CST_roi2_L.nii.gz", "CST_roi2_R.nii.gz",
    "FA_L.nii.gz", "FA_R.nii.gz",
    "FP_L.nii.gz", "FP_R.nii.gz",
    "HCC_roi1_L.nii.gz", "HCC_roi1_R.nii.gz",
    "HCC_roi2_L.nii.gz", "HCC_roi2_R.nii.gz",
    "IFO_roi1_L.nii.gz", "IFO_roi1_R.nii.gz",
    "IFO_roi2_L.nii.gz", "IFO_roi2_R.nii.gz",
    "IFO_roi3_L.nii.gz", "IFO_roi3_R.nii.gz",
    "ILF_roi1_L.nii.gz", "ILF_roi1_R.nii.gz",
    "ILF_roi2_L.nii.gz", "ILF_roi2_R.nii.gz",
    "LH_Parietal.nii.gz", "RH_Parietal.nii.gz",
    "MdLF_roi1_L.nii.gz", "MdLF_roi1_R.nii.gz",
    "SLF_roi1_L.nii.gz", "SLF_roi1_R.nii.gz",
    "SLF_roi2_L.nii.gz", "SLF_roi2_R.nii.gz",
    "SLFt_roi2_L.nii.gz", "SLFt_roi2_R.nii.gz",
    "SLFt_roi3_L.nii.gz", "SLFt_roi3_R.nii.gz",
    "UNC_roi1_L.nii.gz", "UNC_roi1_R.nii.gz",
    "UNC_roi2_L.nii.gz", "UNC_roi2_R.nii.gz",
    "UNC_roi3_L.nii.gz", "UNC_roi3_R.nii.gz",
    "VOF_box_L.nii.gz", "VOF_box_R.nii.gz",
    "UNCNeo-withCerebellum-for-babyAFQ.nii.gz",
    "UNCNeo_JHU_tracts_prob-for-babyAFQ.nii.gz",
    "mid-saggital.nii.gz",
    "OR_rightV1.nii.gz",
    "OR_rightThal.nii.gz",
    "OR_right_roi3.nii.gz",
    "OR_leftV1.nii.gz",
    "OR_leftThal.nii.gz",
    "OR_left_roi3.nii.gz",
    "pARC_R_start.nii.gz",
    "pARC_L_start.nii.gz",
    "VOF_box_small_L.nii.gz",
    "VOF_box_small_R.nii.gz",
    "VOF_L_start.nii.gz",
    "VOF_R_start.nii.gz",
]

pediatric_md5_hashes = [
    "2efe0deb19ac9e175404bf0cb29d9dbd", "c2e07cd50699527bd7b0cbbe88703c56",
    "76b36d8d6759df58131644281ed16bd2", "645102225bad33da30bafd41d24b3ab0",
    "45ec94d42fdc9448afa6760c656920e9", "54e3cb1b8c242be279f1410d8bb3c383",
    "1ee9f7e8b21ef8ceee81d5a7a178ef33", "4f11097f7ae317aa8d612511be79e2f1",
    "1c4c0823c23b676d6d35004d93b9c695", "d4830d558cc8f707ebec912b32d197a5",
    "c405e0dbd9a4091c77b3d1ad200229b4", "ec0aeccc6661d2ee5ed79259383cdcee",
    "2802cd227b550f6e85df0fec1d515c29", "385addb999dc6d76957d2a35c4ee74bb",
    "0dd14c02b272263adbe2246880979c9d", "99dac5a00c10d81d222f020162fd6194",
    "e49ba370edca96734d9376f551d413db", "f59e9e69e06325198f70047cd63c3bdc",
    "ae3bd2931f95adae0280a8f75cd3ca9b", "c409a0036b8c2dd4d03d11fbc6bfbdcd",
    "c2597a474ea5ec9e3126c35fd238f6b2", "67af59c934147c9f9ff6e0b76c4cc6eb",
    "72d0bbc0b6162e9291fdc450c103a1f0", "51f5041be63ad0ac10d1ac09e3bf1a8e",
    "6200f5cdc1402dce46dedd505468b147", "83cb5bf6b9b1eda63c8643871e84a6d4",
    "2a5d8d309b1256d6e48958e112201c2c", "ba24d0915fdff215a403480d0a7745c9",
    "1001e833d1344274d543ffd02a66af80", "03e20c5eebcd4d439c4ffb36d26a10d9",
    "6200f5cdc1402dce46dedd505468b147", "83cb5bf6b9b1eda63c8643871e84a6d4",
    "a6ae325cce2dc4bb21b52ee4c6ca844f", "a96a31df8f96ccf1c5f871a18e4a2d72",
    "65b7378ca85689a5f23b1b84a6ed78f0", "ce0d0ea696ef51c671aa118e11192e2d",
    "ce4918086ca1e829698576bcf95db62b", "96168d2eff74ec2acf9065f499732526",
    "6b20ba319d44472ec21d6ece937605bb", "26b1cf6ec8bd365dde42e3efe9beeac2",
    "0b3ccf06564d973bfcfff9a87e74f8b5", "84f3426033a2b225b0920b2622864375",
    "5351b3cb7efa9aa8e83e266342809ebe", "4e8a34aaba4e0f22a6149f38620dc39d",
    "682c08f66e8c2cf9e4e60f5ce308c76c", "9077affd4f3a8a1d6b44678cde4b3bf4",
    "5adf36f00669cc547d5eb978acf46266", "66a8002688ffdf3943722361da90ec6a",
    "efb5ae138df92019541861f9aa6a4d57", "757ec61078b2e9f9a073871b3216ff7a",
    "ff1e238c52a21f8cc5d44ac614d9627f", "cf16dd2767c6ab2d3fceb2890f6c3e41",
    "6016621e244b60b9c69fd44b055e4a03", "fd495a2c94b6b13bfb4cd63e293d3fc0",
    "bf81a23d80f55e5f1eb0c16717193105",
    "6f8bf8f70216788d14d9a49a3c664b16",
    "19df0297d6a2ac21da5e432645d63174",
    "b4ffb957a2adbb8b76966e4ea28dbdf1",
    "3de1bc5aae4c76164f34515e2e84939c",
    "705ba1cbfc42ce64a54dda2b732f81f2",
    "bc6d4f880d3eb281358af1c764779704",
    "a3b6a7be067aa12af273482baee1498d",
    "f035813314960eb91f3a4dae508a68e5",
    "728461f81fcfa8f02ff3af969ab6499c",
    "5c1c87ee71c512b38a0711b93bb7e9fe",
    "4652bcf7a1f2b7cdeec52956eb884795",
    "e1c4bf76d2c98dcf6ffe00399a6e8b07",
    "0ecabd68fa9c56614e2a694359c0a545",
    "4c93bea7a72ac2b7475201acef5a1fc4",
]

pediatric_remote_fnames = [
    "24880625", "24880628", "24880631", "24880634", "24880637", "24880640",
    "24880643", "24880646", "24880649", "24880652", "24880655", "24880661",
    "24880664", "24880667", "46407571", "46407568", "24880676", "24880679",
    "24880685", "24880688", "24880691", "24880694", "24880697", "24880700",
    "24880703", "24880706", "24880712", "24880715", "24880718", "24880721",
    "24880724", "24880727", "24880730", "24880733", "24880736", "24880748",
    "24880739", "24880742", "24880754", "24880757", "24880760", "24880763",
    "24880769", "24880772", "24880775", "24880778", "24880781", "24880787",
    "24880790", "24880793", "24880796", "24880802", "24880805", "24880808",
    "24880616", "24880613", "24986396",
    "42120480",
    "42120483",
    "42120486",
    "42120489",
    "42120492",
    "42120495",
    "42121791",
    "42121794",
    "42121836",
    "42121839",
    "42121803",
    "42121806",
]

fetch_pediatric_templates = _make_reusable_fetcher(
    'fetch_pediatric_templates',
    op.join(afq_home, 'pediatric_templates'),
    'https://ndownloader.figshare.com/files/',
    pediatric_remote_fnames,
    pediatric_fnames,
    md5_list=pediatric_md5_hashes,
    doc='Download pediatric templates'
)


def read_pediatric_templates(as_img=True, resample_to=False):
    """
    Load pediatric pyAFQ templates.

    Used to create pediatric `bundle_dict`.

    Parameters
    ----------
    as_img : bool, optional
        If True, values are `Nifti1Image`. Otherwise, values are
        paths to Nifti files. Default: True
    resample_to : str or nibabel image class instance, optional
        A template image to resample to. Typically, this should be the
        template to which individual-level data are registered. Defaults to
        the MNI template. Default: False

    Returns
    -------
    dict :
        keys = names of template ROIs, and
        values = `Nifti1Image` from each of the ROI nifti files.
    """

    print('Loading pediatric templates...', flush=True)
    pediatric_templates = _fetcher_to_template(
        fetch_pediatric_templates,
        as_img=as_img,
        resample_to=resample_to)

    # For the arcuate (AF/ARC), reuse the SLF ROIs
    pediatric_templates['ARC_roi1_L'] = pediatric_templates['SLF_roi1_L']
    pediatric_templates['ARC_roi1_R'] = pediatric_templates['SLF_roi1_R']
    pediatric_templates['ARC_roi2_L'] = pediatric_templates['SLFt_roi3_L']
    pediatric_templates['ARC_roi2_R'] = pediatric_templates['SLFt_roi3_R']
    pediatric_templates['ARC_roi3_L'] = pediatric_templates['SLFt_roi2_L']
    pediatric_templates['ARC_roi3_R'] = pediatric_templates['SLFt_roi2_R']

    # For the middle longitudinal fasciculus (MdLF) reuse ILF ROI
    pediatric_templates['MdLF_roi2_L'] = pediatric_templates['ILF_roi2_L']
    pediatric_templates['MdLF_roi2_R'] = pediatric_templates['ILF_roi2_R']

    return pediatric_templates


def read_resample_roi(roi, resample_to=None, threshold=False):
    """
    Reads an roi from file-name/img and resamples it to conform with
    another file-name/img.

    Parameters
    ----------
    roi : str or nibabel image class instance.
        Should contain a binary volume with 1s in the region of interest and
        0s elsewhere.

    resample_to : str or nibabel image class instance, optional
        A template image to resample to. Typically, this should be the
        template to which individual-level data are registered. Defaults to
        the MNI template.

    threshold: bool or float
        If set to False (default), resampled result is returned. Otherwise,
        the resampled result is thresholded at this value and binarized.
        This is not applied if the input ROI is already in the space of the
        output.

    Returns
    -------
    nibabel image class instance that contains the binary ROI resampled into
    the requested space.
    """
    if isinstance(roi, str):
        roi = nib.load(roi)

    if resample_to is None:
        resample_to = read_mni_template()

    if isinstance(resample_to, str):
        resample_to = nib.load(resample_to)

    if resample_to is False or np.allclose(resample_to.affine, roi.affine):
        return roi

    as_array = resample(
        roi.get_fdata(),
        resample_to,
        roi.affine,
        resample_to.affine).get_fdata()
    if threshold:
        as_array = (as_array > threshold).astype(int)

    img = nib.Nifti1Image(
        as_array,
        resample_to.affine)

    return img


template_fnames = ["ATR_roi1_L.nii.gz",
                   "ATR_roi1_R.nii.gz",
                   "ATR_roi2_L.nii.gz",
                   "ATR_roi2_R.nii.gz",
                   "ATR_L_prob_map.nii.gz",
                   "ATR_R_prob_map.nii.gz",
                   "CGC_roi1_L.nii.gz",
                   "CGC_roi1_R.nii.gz",
                   "CGC_roi2_L.nii.gz",
                   "CGC_roi2_R.nii.gz",
                   "CGC_L_prob_map.nii.gz",
                   "CGC_R_prob_map.nii.gz",
                   "CST_roi1_L.nii.gz",
                   "CST_roi1_R.nii.gz",
                   "CST_roi2_L.nii.gz",
                   "CST_roi2_R.nii.gz",
                   "CST_L_prob_map.nii.gz",
                   "CST_R_prob_map.nii.gz",
                   "FA_L.nii.gz",
                   "FA_R.nii.gz",
                   "FA_prob_map.nii.gz",
                   "FP_L.nii.gz",
                   "FP_R.nii.gz",
                   "FP_prob_map.nii.gz",
                   "HCC_roi1_L.nii.gz",
                   "HCC_roi1_R.nii.gz",
                   "HCC_roi2_L.nii.gz",
                   "HCC_roi2_R.nii.gz",
                   "HCC_L_prob_map.nii.gz",
                   "HCC_R_prob_map.nii.gz",
                   "IFO_roi1_L.nii.gz",
                   "IFO_roi1_R.nii.gz",
                   "IFO_roi2_L.nii.gz",
                   "IFO_roi2_R.nii.gz",
                   "IFO_L_prob_map.nii.gz",
                   "IFO_R_prob_map.nii.gz",
                   "ILF_roi1_L.nii.gz",
                   "ILF_roi1_R.nii.gz",
                   "ILF_roi2_L.nii.gz",
                   "ILF_roi2_R.nii.gz",
                   "ILF_L_prob_map.nii.gz",
                   "ILF_R_prob_map.nii.gz",
                   "SLF_roi1_L.nii.gz",
                   "SLF_roi1_R.nii.gz",
                   "SLF_roi2_L.nii.gz",
                   "SLF_roi2_R.nii.gz",
                   "SLFt_roi2_L.nii.gz",
                   "SLFt_roi2_R.nii.gz",
                   "SLF_L_prob_map.nii.gz",
                   "SLF_R_prob_map.nii.gz",
                   "UNC_roi1_L.nii.gz",
                   "UNC_roi1_R.nii.gz",
                   "UNC_roi2_L.nii.gz",
                   "UNC_roi2_R.nii.gz",
                   "UNC_L_prob_map.nii.gz",
                   "UNC_R_prob_map.nii.gz",
                   "ARC_L_prob_map.nii.gz",
                   "ARC_R_prob_map.nii.gz",
                   "VOF_R_end.nii.gz",
                   "VOF_R_start.nii.gz",
                   "VOF_L_end.nii.gz",
                   "VOF_L_start.nii.gz",
                   "pARC_R_start.nii.gz",
                   "pARC_L_start.nii.gz",
                   "ARC_R_end.nii.gz",
                   "ARC_R_start.nii.gz",
                   "ARC_L_end.nii.gz",
                   "ARC_L_start.nii.gz",
                   "UNC_R_end.nii.gz",
                   "UNC_R_start.nii.gz",
                   "UNC_L_end.nii.gz",
                   "UNC_L_start.nii.gz",
                   "SLF_R_end.nii.gz",
                   "SLF_R_start.nii.gz",
                   "SLF_L_end.nii.gz",
                   "SLF_L_start.nii.gz",
                   "ILF_R_end.nii.gz",
                   "ILF_R_start.nii.gz",
                   "ILF_L_end.nii.gz",
                   "ILF_L_start.nii.gz",
                   "IFO_R_end.nii.gz",
                   "IFO_R_start.nii.gz",
                   "IFO_L_end.nii.gz",
                   "IFO_L_start.nii.gz",
                   "FA_end.nii.gz",
                   "FA_start.nii.gz",
                   "FP_end.nii.gz",
                   "FP_start.nii.gz",
                   "CGC_R_start.nii.gz",
                   "CGC_L_start.nii.gz",
                   "CST_R_end.nii.gz",
                   "CST_R_start.nii.gz",
                   "CST_L_end.nii.gz",
                   "CST_L_start.nii.gz",
                   "ATR_R_end.nii.gz",
                   "ATR_R_start.nii.gz",
                   "ATR_L_end.nii.gz",
                   "ATR_L_start.nii.gz"]


template_remote_fnames = ["5273680", "5273683", "5273686", "5273689",
                          "11458274", "11458277",
                          "5273695", "5273692", "5273698", "5273701",
                          "11458268", "11458271",
                          "5273704", "5273707", "46407574", "46407577",
                          "11458262", "11458265",
                          "5273716", "5273719",
                          "11458220",
                          "5273722", "5273725",
                          "11458226",
                          "5273728", "5273731", "5273734", "5273746",
                          "11458259", "11458256",
                          "5273737", "5273740", "5273743", "5273749",
                          "11458250", "11458253",
                          "5273752", "5273755", "5273758", "5273761",
                          "11458244", "11458247",
                          "5273764", "5273767", "5273770", "5273773",
                          "5273776", "5273791",
                          "11458238", "11458241",
                          "5273779", "5273782", "5273785", "5273788",
                          "11458223", "11458229",
                          "11458232", "11458235",
                          "40943957",
                          "40943960",
                          "40943966",
                          "40943969",
                          "40943972",
                          "40943975",
                          "40943978",
                          "40943981",
                          "40943984",
                          "40943987",
                          "40943990",
                          "40943993",
                          "40943996",
                          "40943999",
                          "40944002",
                          "40944005",
                          "40944008",
                          "40944011",
                          "40944014",
                          "40944017",
                          "40944020",
                          "40944023",
                          "40944026",
                          "40944029",
                          "40944032",
                          "40944035",
                          "40944038",
                          "40944041",
                          "40944044",
                          "40944047",
                          "40944050",
                          "40944053",
                          "40944056",
                          "40944059",
                          "40944062",
                          "40944065",
                          "40944068",
                          "40944074",
                          "40944077",
                          "40944080"]


template_md5_hashes = ["6b7aaed1a2982fd0ea436a223133908b",
                       "fd60d46d4e3cbd906c86e4c9e4fd6e2a",
                       "3aba60b169a35c38640de4ec29d362c8",
                       "12716a5688a1809fbaed1d58d2e68b59",
                       "c5637f471df861d9bbb45604db34770b",
                       "850cc4c04d7241747063fe3cd440b2ce",
                       "8e8973bc7838c8744914d402f52d91ca",
                       "c5fa4e6e685e695c006823b6784d2407",
                       "e1fab77f21d5303ed52285f015e24f0b",
                       "5f89defec3753fd75cd688c7bfb20a36",
                       "a4f3cd65b06fb25f63d5dab7592f00f2",
                       "7e73ab02db30a3ad6bd9e82148c2486e",
                       "f9db3154955a20b67c2dda758800d14c",
                       "73941510c798c1ed1b03e2bd481cd5c7",
                       "b20e0caa54cf35002cd06cf6033b964f",
                       "e751306df304af32c3ce7617913bbd30",
                       "fd012bc89f6bed7bd54530195496bac4",
                       "3406906a86e633cc102127cf210a1063",
                       "9040a7953dcbbf131d135c866182d8ef",
                       "a72e17194824fcd838a594a2eb50c72e",
                       "627d7bb2e6d55f8243da815a36d9ff1a",
                       "55adbe9b8279185eedbe342149e1ff90",
                       "5a7412a3cf0fb185eec53d1989df2f7c",
                       "1aa36e83ae7b5555bb19d776ede9c18d",
                       "ba453196ff179b0e31172806e313b52c",
                       "d85c6574526b296935f34bf4f65cd493",
                       "9b81646317f59c7db087f27e2f85679e",
                       "9806e82c250e4604534b96917f87b7e8",
                       "213d3fb1ccd756d878f9b50b765b1c8f",
                       "f1e7e6bc51aa0aa279c54fb3805fb5e3",
                       "0e68a9feaaddcc9b4d667c2f15903368",
                       "d45020a87ee4bb496edd350631d91f6a",
                       "75c2c911826ec4b23159f9bd80e3c039",
                       "55d616ea9e0c646adc1aafa0f5fbe625",
                       "dee83fa6b03cfa5e0f5c965953aa6778",
                       "a13eef7059c98568adfefbab660e434e",
                       "045b7d5c6341997f3f0120c3a4212ad8",
                       "d174b1359ba982b03840436c93b7bbb4",
                       "fff9753f394fc4c73fb2ae40b3b4dde0",
                       "cd278b4dd6ff77481ea9ac16485a5ae2",
                       "7bdf5111265107091c7a2fca9215de30",
                       "7d4a43714504e6e930f922c9bc2a13d5",
                       "af2bcedf47e193686af329b9a8e259da",
                       "9a1122943579d11ba169d3ad87a75625",
                       "627903f7a06627bfd4153dc9245fa390",
                       "1714cd7f989c3435bdd5a2076e6272a0",
                       "1fa2114049707a4e05b53f9d95730375",
                       "b6663067d5ea53c70cb8803948f8adf7",
                       "d3e068997ebc60407bd6e9576e47dede",
                       "27ecfbd1d2f98213e52d73b7d70fe0e7",
                       "fa141bb2d951bec486916acda3652d95",
                       "d391d073e86e28588be9a6d01b2e7a82",
                       "a3e085562e6b8111c7ebc358f9450c8b",
                       "d65c67910807504735e034f7ea92d590",
                       "93cb24a9128db1a6c34a09eaf79fe7f0",
                       "71a7455cb4062dc39b1677c118c7b5a5",
                       "19590c712f1776da1fdba64d4eb7f1f6",
                       "04d5af0feb2c1b5b52a87ccbbf148e4b",
                       "53c277be990d00f7de04f2ea35e74d73",
                       "d37d815fd1bdaaf3a9d2dcfc3ccb1345",
                       "95ed3189d8ac152945e6be1eb24381a3",
                       "a9007e6f2d6ae13ef182f65057c06573",
                       "c6eb9ee33b7caf691749e266f89e8ec4",
                       "a06b2e2e52c09a601f683dc39859a7f1",
                       "bee876a34fdb03e69a418b791f90975a",
                       "680749c9e4565bc02492019d57d8e7d7",
                       "ffc157e9f73a43eff23821f2cfca614a",
                       "a92beacd59ff2c90408edc7407a571c4",
                       "1c0b570bb2d622718b01ee2c429a5d15",
                       "4d3db603f7cc86b5696b1514c5b43e0e",
                       "96c644616e8724b4d071c64981363e6c",
                       "9be681028226aa9b70fac46f8e376282",
                       "e19274bd1ee58906c03f5f9a7c4f01c5",
                       "1d33ac68aeb8eb444c07b5a853414727",
                       "ca3c698154d77217c9480123a49c73dd",
                       "b17d876c0ca51f0e3ec6177382aa4ecf",
                       "6ff7e2dd357affc7c4e4d79f5f5866e9",
                       "96c644616e8724b4d071c64981363e6c",
                       "de045095d914d4929409e062fc1a09b6",
                       "e19274bd1ee58906c03f5f9a7c4f01c5",
                       "69009da4aa52607962ec99b9e248a39f",
                       "2b106da3ed88e4cc9a5b4c391cf3445d",
                       "de045095d914d4929409e062fc1a09b6",
                       "0d28a8e367646f9f0036ba7a100e49e5",
                       "69009da4aa52607962ec99b9e248a39f",
                       "1c0b570bb2d622718b01ee2c429a5d15",
                       "ffc157e9f73a43eff23821f2cfca614a",
                       "69009da4aa52607962ec99b9e248a39f",
                       "de045095d914d4929409e062fc1a09b6",
                       "9d897a3ab960931c5119a9bbfc6d7838",
                       "0046be49e39cf8c639418c805de8b9e2",
                       "67d8f24bd749d8a9094ccb9d8e63fa25",
                       "5736d28551296253f879c4b0d450b70e",
                       "67d8f24bd749d8a9094ccb9d8e63fa25",
                       "5736d28551296253f879c4b0d450b70e",
                       "5f2c0c0f1b7b32ba0a8474a460a2b980",
                       "ffc157e9f73a43eff23821f2cfca614a",
                       "a8d308a93b26242c04b878c733cb252f",
                       "1c0b570bb2d622718b01ee2c429a5d15"]

fetch_templates = _make_reusable_fetcher(
    "fetch_templates",
    op.join(afq_home, 'templates'),
    baseurl, template_remote_fnames,
    template_fnames, md5_list=template_md5_hashes,
    doc="Download AFQ templates")


def read_templates(as_img=True, resample_to=False):
    """Load AFQ templates from file

    Parameters
    ----------
    as_img : bool, optional
        If True, values are `Nifti1Image`. Otherwise, values are
        paths to Nifti files. Default: True
    resample_to : str or nibabel image class instance, optional
        A template image to resample to. Typically, this should be the
        template to which individual-level data are registered. Defaults to
        the MNI template. Default: False

    Returns
    -------
    dict with: keys: names of template ROIs and values: nibabel Nifti1Image
    objects from each of the ROI nifti files.
    """
    logger = logging.getLogger('AFQ')
    logger.debug('loading AFQ templates')
    tic = time.perf_counter()

    template_dict = _fetcher_to_template(
        fetch_templates,
        as_img=as_img,
        resample_to=resample_to)

    toc = time.perf_counter()
    logger.debug(f'AFQ templates loaded in {toc - tic:0.4f} seconds')

    return template_dict


cp_fnames = [
    "ICP_L_inferior_prob.nii.gz",
    "ICP_L_superior_prob.nii.gz",
    "ICP_R_inferior_prob.nii.gz",
    "ICP_R_superior_prob.nii.gz",
    "MCP_L_inferior_prob.nii.gz",
    "MCP_L_superior_prob.nii.gz",
    "MCP_R_inferior_prob.nii.gz",
    "MCP_R_superior_prob.nii.gz",
    "SCP_L_inferior_prob.nii.gz",
    "SCP_L_inter_prob.nii.gz",
    "SCP_L_superior_prob.nii.gz",
    "SCP_R_inferior_prob.nii.gz",
    "SCP_R_inter_prob.nii.gz",
    "SCP_R_superior_prob.nii.gz"]

cp_remote_fnames = [
    "40897772",
    "40897787",
    "40897805",
    "40897769",
    "40897778",
    "40897799",
    "40897784",
    "40897808",
    "40897772",
    "40897781",
    "40897793",
    "40897790",
    "40897775",
    "40897796"]

cp_md5_hashes = [
    "7ec3a78f30aefe8c7e2a99773941b8f4",
    "fc6b088f359201aff480d8e2f7b17eef",
    "e9688ff4554b768f79a64731705107f9",
    "f1959600d6aaae58d7ba4ce5e63c5e3b",
    "e68c669d58bd2cbb3490bd952957009e",
    "eb7643fd54f6046cd71b7ac085679594",
    "746da79d0639630c1eb872b2895814ee",
    "3f69b20a8a7dec4839cef461667d18a5",
    "4d95b3d3804352eae6e66fe024f1baf5",
    "a9410ab5fe3240cc8ef929062d54cc9e",
    "531babda0dd284bda4a5b4b1303f8266",
    "dde4907d7914dfe71ed07436b073bd75",
    "c02f2fcb48c33fe4b2988087075e9566",
    "62e32e090cd326790c3fdbc6acb8eb75"]

fetch_cp_templates = _make_reusable_fetcher(
    "fetch_cb_templates",
    op.join(afq_home,
            'cp_templates'),
    baseurl, cp_remote_fnames,
    cp_fnames,
    md5_list=cp_md5_hashes,
    doc="Download AFQ cerebellar penducles templates")


def read_cp_templates(as_img=True, resample_to=False):
    """Load AFQ Cerebellar penducles templates from file

    Parameters
    ----------
    as_img : bool, optional
        If True, values are `Nifti1Image`. Otherwise, values are
        paths to Nifti files. Default: True
    resample_to : str or nibabel image class instance, optional
        A template image to resample to. Typically, this should be the
        template to which individual-level data are registered. Defaults to
        the MNI template. Default: False

    Returns
    -------
    dict with: keys: names of template ROIs and values: nibabel Nifti1Image
    objects from each of the ROI nifti files.
    """
    logger = logging.getLogger('AFQ')

    logger.debug('loading or templates')
    tic = time.perf_counter()

    template_dict = _fetcher_to_template(
        fetch_cp_templates,
        as_img=as_img,
        resample_to=resample_to)

    toc = time.perf_counter()
    logger.debug(
        f'Cerebellar peduncles templates loaded in {toc - tic:0.4f} seconds')

    return template_dict


or_fnames = [
    "left_thal_MNI.nii.gz",
    "left_V1_MNI.nii.gz",
    "right_thal_MNI.nii.gz",
    "right_V1_MNI.nii.gz",
    "left_OP_MNI.nii.gz",
    "left_OR_1.nii.gz",
    "left_OR_2.nii.gz",
    "left_pos_thal_MNI.nii.gz",
    "left_TP_MNI.nii.gz",
    "right_OP_MNI.nii.gz",
    "right_OR_1.nii.gz",
    "right_OR_2.nii.gz",
    "right_pos_thal_MNI.nii.gz",
    "right_TP_MNI.nii.gz",
]

or_remote_fnames = [
    "36514170",
    "26831633",
    "36514173",
    "26831639",
    "26831642",
    "26831645",
    "26831648",
    "26831651",
    "26831654",
    "26831657",
    "26831660",
    "26831663",
    "26831666",
    "26831669",
]

or_md5_hashes = [
    "a45126a727c4b5d843b2f7aae181825f",
    "ad996c67bf5cc59fc3a7b60255873b67",
    "7a75c3ddd25335277a099626dbc946ac",
    "cc88fb4671311404eb9dfa8fa11a59e0",
    "9cff03af586d9dd880750cef3e0bf63f",
    "ff728ba3ffa5d1600bcd19fdef8182c4",
    "4f1978e418a3169609375c28b3eba0fd",
    "fd163893081b520f4594171aeea04f39",
    "bf795d197912b5e074d248d2763c6930",
    "13efde1efe0de52683cbf352ecba457e",
    "75c7bd2092950578e599a2dcb218909f",
    "8f3890fa8c26a568503226757f7e7d6c",
    "f239aa3140809152da8884ff879dde1b",
    "60a748567e4dd81b40ad8967a14cb09e",
]

ar_fnames = [
    "AAL_Thal_R.nii.gz",
    "AAL_Thal_L.nii.gz",
    "AAL_TempSup_L.nii.gz",
    "AAL_TempSup_R.nii.gz",
]

ar_remote_fnames = [
    "43817367",
    "43817370",
    "43817373",
    "43817376",
]

ar_md5_hashes = [
    "5f2c0c0f1b7b32ba0a8474a460a2b980",
    "a8d308a93b26242c04b878c733cb252f",
    "92a255f706dd4e09268c5bc6bf2876e4",
    "ec0a3ef3cb3a4c549dd783dc8aeceeee",
]


fetch_ar_templates = _make_reusable_fetcher(
    "fetch_ar_templates",
    op.join(afq_home,
            'ar_templates'),
    baseurl, ar_remote_fnames,
    ar_fnames,
    md5_list=ar_md5_hashes,
    doc="Download AFQ or templates")


def read_ar_templates(as_img=True, resample_to=False):
    """Load AFQ AR templates from file

    Parameters
    ----------
    as_img : bool, optional
        If True, values are `Nifti1Image`. Otherwise, values are
        paths to Nifti files. Default: True
    resample_to : str or nibabel image class instance, optional
        A template image to resample to. Typically, this should be the
        template to which individual-level data are registered. Defaults to
        the MNI template. Default: False

    Returns
    -------
    dict with: keys: names of template ROIs and values: nibabel Nifti1Image
    objects from each of the ROI nifti files.
    """
    logger = logging.getLogger('AFQ')

    logger.debug('loading ar templates')
    tic = time.perf_counter()

    template_dict = _fetcher_to_template(
        fetch_ar_templates,
        as_img=as_img,
        resample_to=resample_to)

    toc = time.perf_counter()
    logger.debug(f'or templates loaded in {toc - tic:0.4f} seconds')

    return template_dict


fetch_or_templates = _make_reusable_fetcher(
    "fetch_or_templates",
    op.join(afq_home,
            'or_templates'),
    baseurl, or_remote_fnames,
    or_fnames,
    md5_list=or_md5_hashes,
    doc="Download AFQ or templates")


def read_or_templates(as_img=True, resample_to=False):
    """Load AFQ OR templates from file

    Parameters
    ----------
    as_img : bool, optional
        If True, values are `Nifti1Image`. Otherwise, values are
        paths to Nifti files. Default: True
    resample_to : str or nibabel image class instance, optional
        A template image to resample to. Typically, this should be the
        template to which individual-level data are registered. Defaults to
        the MNI template. Default: False

    Returns
    -------
    dict with: keys: names of template ROIs and values: nibabel Nifti1Image
    objects from each of the ROI nifti files.
    """
    logger = logging.getLogger('AFQ')

    logger.debug('loading or templates')
    tic = time.perf_counter()

    template_dict = _fetcher_to_template(
        fetch_or_templates,
        as_img=as_img,
        resample_to=resample_to)

    toc = time.perf_counter()
    logger.debug(f'or templates loaded in {toc - tic:0.4f} seconds')

    return template_dict


stanford_hardi_tractography_remote_fnames = ["5325715", "5325718", "25289735"]
stanford_hardi_tractography_hashes = ['6f4bdae702031a48d1cd3811e7a42ef9',
                                      'f20854b4f710577c58bd01072cfb4de6',
                                      '294bfd1831861e8635eef8834ff18892']
stanford_hardi_tractography_fnames = [
    'mapping.nii.gz',
    'tractography_subsampled.trk',
    'full_segmented_cleaned_tractography.trk']

fetch_stanford_hardi_tractography = _make_reusable_fetcher(
    "fetch_stanford_hardi_tractography",
    op.join(afq_home,
            'stanford_hardi_tractography'),
    baseurl,
    stanford_hardi_tractography_remote_fnames,
    stanford_hardi_tractography_fnames,
    md5_list=stanford_hardi_tractography_hashes,
    doc="""Download Stanford HARDI tractography and mapping. For testing
           purposes""")


def read_stanford_hardi_tractography():
    """
    Reads a minimal tractography from the Stanford dataset.
    """
    files, folder = fetch_stanford_hardi_tractography()
    files_dict = {}
    files_dict['mapping.nii.gz'] = nib.load(
        op.join(afq_home,
                'stanford_hardi_tractography',
                'mapping.nii.gz'))

    # We need the original data as reference
    dwi_img, gtab = dpd.read_stanford_hardi()

    files_dict['tractography_subsampled.trk'] = load_tractogram(
        op.join(afq_home,
                'stanford_hardi_tractography',
                'tractography_subsampled.trk'),
        dwi_img,
        bbox_valid_check=False,
        trk_header_check=False).streamlines

    files_dict['full_segmented_cleaned_tractography.trk'] = load_tractogram(
        op.join(
            afq_home,
            'stanford_hardi_tractography',
            'full_segmented_cleaned_tractography.trk'),
        dwi_img).streamlines

    return files_dict


def to_bids_description(path, fname='dataset_description.json',
                        BIDSVersion="1.4.0", **kwargs):
    """Dumps a dict into a bids description at the given location"""
    kwargs.update({"BIDSVersion": BIDSVersion})
    desc_file = op.join(path, fname)
    with open(desc_file, 'w') as outfile:
        json.dump(kwargs, outfile)


def organize_cfin_data(path=None):
    """
    Create the expected file-system structure for the
    CFIN multi b-value diffusion data-set.
    """

    dpd.fetch_cfin_multib()
    if path is None:
        os.makedirs(afq_home, exist_ok=True)
        path = afq_home

    bids_path = op.join(path, 'cfin_multib',)
    derivatives_path = op.join(bids_path, 'derivatives')
    dmriprep_folder = op.join(derivatives_path, 'dmriprep')

    if not op.exists(derivatives_path):
        anat_folder = op.join(dmriprep_folder, 'sub-01', 'ses-01', 'anat')
        os.makedirs(anat_folder, exist_ok=True)
        dwi_folder = op.join(dmriprep_folder, 'sub-01', 'ses-01', 'dwi')
        os.makedirs(dwi_folder, exist_ok=True)
        t1_img = dpd.read_cfin_t1()
        nib.save(t1_img, op.join(anat_folder, 'sub-01_ses-01_T1w.nii.gz'))
        dwi_img, gtab = dpd.read_cfin_dwi()
        nib.save(dwi_img, op.join(dwi_folder, 'sub-01_ses-01_dwi.nii.gz'))
        np.savetxt(op.join(dwi_folder, 'sub-01_ses-01_dwi.bvec'), gtab.bvecs)
        np.savetxt(op.join(dwi_folder, 'sub-01_ses-01_dwi.bval'), gtab.bvals)

    to_bids_description(
        bids_path,
        **{"BIDSVersion": "1.0.0",
           "Name": "CFIN",
           "Subjects": ["sub-01"]})

    to_bids_description(
        dmriprep_folder,
        **{"Name": "CFIN",
           "PipelineDescription": {"Name": "dipy"},
           "GeneratedBy": [{"Name": "dipy"}]})


def organize_stanford_data(path=None, clear_previous_afq=None):
    """
    If necessary, downloads the Stanford HARDI dataset into DIPY directory and
    creates a BIDS compliant file-system structure in AFQ data directory:


    ~/AFQ_data/
    └── stanford_hardi
    ├── dataset_description.json
    └── derivatives
        ├── freesurfer
        │   ├── dataset_description.json
        │   └── sub-01
        │       └── ses-01
        │           └── anat
        │               ├── sub-01_ses-01_T1w.nii.gz
        │               └── sub-01_ses-01_seg.nii.gz
        └── vistasoft
            ├── dataset_description.json
            └── sub-01
                └── ses-01
                    └── dwi
                        ├── sub-01_ses-01_dwi.bval
                        ├── sub-01_ses-01_dwi.bvec
                        └── sub-01_ses-01_dwi.nii.gz

    Parameters
    ----------
    path : str or None
        Path to download dataset to, by default it is ~/AFQ_data/.
    clear_previous_afq : str or None
        Whether to clear previous afq results in the stanford
        hardi dataset. If not None, can be "all", "track", "recog", "prof".
        Default: None
    """
    logger = logging.getLogger('AFQ')

    # fetches data for first subject and session
    logger.info('fetching Stanford HARDI data')
    dpd.fetch_stanford_hardi()

    if path is None:
        if not op.exists(afq_home):
            logger.info(f'creating AFQ home directory: {afq_home}')
        os.makedirs(afq_home, exist_ok=True)
        path = afq_home

    bids_path = op.join(path, 'stanford_hardi',)
    derivatives_path = op.join(bids_path, 'derivatives')
    dmriprep_folder = op.join(derivatives_path, 'vistasoft')
    freesurfer_folder = op.join(derivatives_path, 'freesurfer')

    if clear_previous_afq is not None and op.exists(derivatives_path):
        afq_folder = op.join(derivatives_path, 'afq')
        if clear_previous_afq == "all":
            if op.exists(afq_folder):
                shutil.rmtree(afq_folder)
        else:
            apply_cmd_to_afq_derivs(
                op.join(afq_folder, "sub-01/ses-01"),
                op.join(afq_folder, "sub-01/ses-01/sub-01_ses-01"),
                dependent_on=clear_previous_afq)

    if not op.exists(derivatives_path):
        logger.info(f'creating derivatives directory: {derivatives_path}')

        # anatomical data
        anat_folder = op.join(freesurfer_folder, 'sub-01', 'ses-01', 'anat')
        os.makedirs(anat_folder, exist_ok=True)

        t1_img = dpd.read_stanford_t1()
        nib.save(t1_img, op.join(anat_folder, 'sub-01_ses-01_T1w.nii.gz'))

        seg_img = dpd.read_stanford_labels()[-1]
        nib.save(seg_img, op.join(anat_folder,
                                  'sub-01_ses-01_seg.nii.gz'))

        # diffusion-weighted imaging data
        dwi_folder = op.join(dmriprep_folder, 'sub-01', 'ses-01', 'dwi')
        os.makedirs(dwi_folder, exist_ok=True)

        dwi_img, gtab = dpd.read_stanford_hardi()
        nib.save(dwi_img, op.join(dwi_folder, 'sub-01_ses-01_dwi.nii.gz'))
        np.savetxt(op.join(dwi_folder, 'sub-01_ses-01_dwi.bvec'), gtab.bvecs)
        np.savetxt(op.join(dwi_folder, 'sub-01_ses-01_dwi.bval'), gtab.bvals)
    else:
        logger.info('Dataset is already in place. If you want to fetch it '
                    + 'again please first remove the folder '
                    + derivatives_path)

    # Dump out the description of the dataset
    to_bids_description(bids_path,
                        **{"Name": "Stanford HARDI", "Subjects": ["sub-01"]})

    # And descriptions of the pipelines in the derivatives:
    to_bids_description(dmriprep_folder,
                        **{"Name": "Stanford HARDI",
                           "PipelineDescription": {"Name": "vistasoft"},
                           "GeneratedBy": [{"Name": "vistasoft"}]})
    to_bids_description(freesurfer_folder,
                        **{"Name": "Stanford HARDI",
                           "PipelineDescription": {"Name": "freesurfer"},
                           "GeneratedBy": [{"Name": "freesurfer"}]})


fetch_stanford_hardi_lv1 = _make_reusable_fetcher(
    "fetch_stanford_hardi_lv1",
    op.join(afq_home,
            'stanford_hardi',
            'derivatives/freesurfer/sub-01/ses-01/anat'),
    'https://stacks.stanford.edu/file/druid:ng782rw8378/',
    ["SUB1_LV1.nii.gz"],
    ["sub-01_ses-01_desc-LV1_anat.nii.gz"],
    md5_list=["e403c602e53e5491414f86af5f08a913"],
    doc="Download the LV1 segmentation for the Standord Hardi subject",
    unzip=False)


fetch_hcp_atlas_16_bundles = _make_reusable_fetcher(
    "fetch_hcp_atlas_16_bundles",
    op.join(afq_home,
            'hcp_atlas_16_bundles'),
    'https://ndownloader.figshare.com/files/',
    ["11921522"],
    ["atlas_16_bundles.zip"],
    md5_list=["b071f3e851f21ba1749c02fc6beb3118"],
    doc="Download minimal Recobundles atlas",
    unzip=True)


fetch_hcp_atlas_80_bundles = _make_reusable_fetcher(
    "fetch_hcp_atlas_80_bundles",
    op.join(afq_home,
            'hcp_atlas_80_bundles'),
    'https://ndownloader.figshare.com/files/',
    ["13638644"],
    ["Atlas_80_Bundles.zip"],
    md5_list=["78331d527a10ec000d4f33bac472e099"],
    doc="Download 80-bundle Recobundles atlas",
    unzip=True)


def read_hcp_atlas(n_bundles=16, as_file=False):
    """
    as_file : bool, optional
        If True, values are paths to sls. Otherwise, the sl
        are located and the centroids calculated. Default: False

    n_bundles : int
        16 or 80, which selects among the two different
        atlases:

        https://figshare.com/articles/Simple_model_bundle_atlas_for_RecoBundles/6483614  # noqa

        https://figshare.com/articles/Advanced_Atlas_of_80_Bundles_in_MNI_space/7375883  # noqa
    """
    bundle_dict = {}
    if n_bundles == 16:
        _, folder = fetch_hcp_atlas_16_bundles()
        atlas_folder = "Atlas_in_MNI_Space_16_bundles"
    elif n_bundles == 80:
        _, folder = fetch_hcp_atlas_80_bundles()
        atlas_folder = "Atlas_80_Bundles"

    bundle_files = glob(
        op.join(
            folder,
            atlas_folder,
            "bundles", "*.trk"))
    centroid_folder = op.join(
        folder,
        atlas_folder,
        "centroid")
    os.makedirs(centroid_folder, exist_ok=True)
    for bundle_file in bundle_files:
        bundle = drop_extension(op.split(bundle_file)[-1])
        centroid_file = op.join(centroid_folder, f"{bundle}.trk")
        bundle_dict[bundle] = {"recobundles": {}}
        if not op.exists(centroid_file):
            bundle_sl = load_tractogram(
                bundle_file,
                'same',
                bbox_valid_check=False)
            feature = ResampleFeature(nb_points=100)
            metric = AveragePointwiseEuclideanMetric(feature)
            qb = QuickBundles(np.inf, metric=metric)
            cluster = [qb.cluster(bundle_sl.streamlines).centroids[0]]
            save_tractogram(
                StatefulTractogram(
                    cluster, bundle_sl, Space.RASMM),
                centroid_file,
                bbox_valid_check=False)
        if not as_file:
            bundle_dict[bundle]["recobundles"]['sl'] = load_tractogram(
                bundle_file,
                'same',
                bbox_valid_check=False).streamlines
            bundle_dict[bundle]["recobundles"]['centroid'] = load_tractogram(
                centroid_file,
                "same", bbox_valid_check=False).streamlines
        else:
            bundle_dict[bundle]["recobundles"]['sl'] = bundle_file
            bundle_dict[bundle]["recobundles"]['centroid'] = centroid_file

    # For some reason, this file-name has a 0 in it, instead of an O:
    bundle_dict["IFOF_R"] = bundle_dict["IF0F_R"]
    # In the 80-bundle case, there are two files, and both have identical
    # content, so this is fine:
    del bundle_dict["IF0F_R"]
    return bundle_dict


fetch_aal_atlas = _make_reusable_fetcher(
    "fetch_aal_atlas",
    op.join(afq_home,
            'aal_atlas'),
    'https://ndownloader.figshare.com/files/',
    ["28416852",
     "28416855"],
    ["MNI_AAL_AndMore.nii.gz",
     "MNI_AAL.txt"],
    md5_list=["69395b75a16f00294a80eb9428bf7855",
              "59fd3284b17de2fbe411ca1c7afe8c65"],
    doc="Download the AAL atlas",
    unzip=False)


def read_aal_atlas(resample_to=None):
    """
    Reads the AAL atlas [1]_.

    Parameters
    ----------
    template : nib.Nifti1Image class instance, optional
        If provided, this is the template used and AAL atlas should be
        registered and aligned to this template


    .. [1] Tzourio-Mazoyer N, Landeau B, Papathanassiou D, Crivello F, Etard O,
           Delcroix N, Mazoyer B, Joliot M. (2002). Automated anatomical
           labeling of activations in SPM using a macroscopic anatomical
           parcellation of the MNI MRI single-subject brain. Neuroimage. 2002;
           15(1):273-89.
    """
    file_dict, folder = fetch_aal_atlas()
    out_dict = {}
    for f in file_dict:
        if f.endswith('.txt'):
            out_dict['labels'] = pd.read_csv(op.join(folder, f))
        else:
            out_dict['atlas'] = nib.load(op.join(folder, f))
    if resample_to is not None:
        data = out_dict['atlas'].get_fdata()
        oo = []
        for ii in range(data.shape[-1]):
            oo.append(resample(
                data[..., ii],
                resample_to,
                out_dict['atlas'].affine,
                resample_to.affine).get_fdata())
        out_dict['atlas'] = nib.Nifti1Image(np.stack(oo, -1),
                                            resample_to.affine)
    return out_dict


def _apply_mask(template_img, resolution=1):
    """
    Helper function, gets MNI brain mask and applies it to template_img.

    Parameters
    ----------
    template_img : nib.Nifti1Image
        Unmasked template
    resolution : int, optional
        Resolution of mask. Default: 1

    Returns
    -------
    Masked template as nib.Nifti1Image
    """
    mask_img = nib.load(str(tflow.get('MNI152NLin2009cAsym',
                                      resolution=resolution,
                                      desc='brain',
                                      suffix='mask')))

    template_data = template_img.get_fdata()
    mask_data = mask_img.get_fdata()

    if mask_data.shape != template_data.shape:
        mask_img = nib.Nifti1Image(
            resample(
                mask_data,
                template_data,
                mask_img.affine,
                template_img.affine).get_fdata(),
            template_img.affine)
        mask_data = mask_img.get_fdata()

    out_data = template_data * mask_data
    return nib.Nifti1Image(out_data, template_img.affine)


def read_mni_template(resolution=1, mask=True, weight="T2w"):
    """

    Reads the MNI T1w or T2w template

    Parameters
    ----------
    resolution : int, optional.
        Either 1 or 2, the resolution in mm of the voxels. Default: 1.

    mask : bool, optional
        Whether to mask the data with a brain-mask before returning the image.
        Default : True

    weight: str, optional
        Which relaxation technique to use.
        Should be either "T2w" or "T1w".
        Default : "T2w"

    Returns
    -------
    nib.Nifti1Image class instance
    containing masked or unmasked T1w or template.

    """
    template_img = nib.load(str(tflow.get('MNI152NLin2009cAsym',
                                          desc=None,
                                          resolution=resolution,
                                          suffix=weight,
                                          extension='nii.gz')))
    if not mask:
        return template_img
    else:
        return _apply_mask(template_img, resolution)


fetch_biobank_templates = \
    _make_reusable_fetcher(
        "fetch_biobank_templates",
        op.join(afq_home,
                'biobank_templates'),
        "http://biobank.ctsu.ox.ac.uk/showcase/showcase/docs/",
        ["bmri_group_means.zip"],
        ["bmri_group_means.zip"],
        data_size="1.1 GB",
        doc="Download UK Biobank templates",
        unzip=True)


def read_ukbb_fa_template(mask=True):
    """
    Reads the UK Biobank FA template

    Parameters
    ----------
    mask : bool, optional
        Whether to mask the data with a brain-mask before returning the image.
        Default : True

    Returns
    -------
    nib.Nifti1Image class instance containing the FA template.

    """
    fa_folder = op.join(
        afq_home,
        'biobank_templates',
        'UKBiobank_BrainImaging_GroupMeanTemplates'
    )
    fa_path = op.join(
        fa_folder,
        'dti_FA.nii.gz'
    )

    if not op.exists(fa_path):
        logger = logging.getLogger('AFQ')
        logger.warning(
            "Downloading brain MRI group mean statistics from UK Biobank. "
            + "This download is approximately 1.1 GB. "
            + "It is currently necessary to access the FA template.")

        files, folder = fetch_biobank_templates()

        # remove zip
        for filename in files:
            os.remove(op.join(folder, filename))

        # remove non-FA related directories
        for filename in os.listdir(fa_folder):
            full_path = op.join(fa_folder, filename)
            if full_path != fa_path:
                if os.path.isfile(full_path):
                    os.remove(full_path)
                else:
                    shutil.rmtree(full_path)

    template_img = nib.load(fa_path)

    if not mask:
        return template_img
    else:
        return _apply_mask(template_img, 1)


def fetch_hcp(subjects,
              hcp_bucket='hcp-openaccess',
              profile_name="hcp",
              path=None,
              study='HCP_1200',
              aws_access_key_id=None,
              aws_secret_access_key=None):
    """
    Fetch HCP diffusion data and arrange it in a manner that resembles the
    BIDS [1]_ specification.

    Parameters
    ----------
    subjects : list
        Each item is an integer, identifying one of the HCP subjects
    hcp_bucket : string, optional
        The name of the HCP S3 bucket. Default: "hcp-openaccess"
    profile_name : string, optional
        The name of the AWS profile used for access. Default: "hcp"
    path : string, optional
        Path to save files into. Default: '~/AFQ_data'
    study : string, optional
        Which HCP study to grab. Default: 'HCP_1200'
    aws_access_key_id : string, optional
        AWS credentials to HCP AWS S3. Will only be used if `profile_name` is
        set to False.
    aws_secret_access_key : string, optional
        AWS credentials to HCP AWS S3. Will only be used if `profile_name` is
        set to False.

    Returns
    -------
    dict with remote and local names of these files,
    path to BIDS derivative dataset

    Notes
    -----
    To use this function with its default setting, you need to have a
    file '~/.aws/credentials', that includes a section:

    [hcp]
    AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXX
    AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXX

    The keys are credentials that you can get from HCP
    (see https://wiki.humanconnectome.org/display/PublicData/How+To+Connect+to+Connectome+Data+via+AWS)  # noqa

    Local filenames are changed to match our expected conventions.

    .. [1] Gorgolewski et al. (2016). The brain imaging data structure,
           a format for organizing and describing outputs of neuroimaging
           experiments. Scientific Data, 3::160044. DOI: 10.1038/sdata.2016.44.
    """
    if profile_name:
        boto3.setup_default_session(profile_name=profile_name)
    elif aws_access_key_id is not None and aws_secret_access_key is not None:
        boto3.setup_default_session(
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key)
    else:
        raise ValueError("Must provide either a `profile_name` or ",
                         "both `aws_access_key_id` and ",
                         "`aws_secret_access_key` as input to 'fetch_hcp'")

    s3 = boto3.resource('s3')
    bucket = s3.Bucket(hcp_bucket)

    if path is None:
        if not op.exists(afq_home):
            os.mkdir(afq_home)
        my_path = afq_home
    else:
        my_path = path

    base_dir = op.join(my_path, study, 'derivatives', 'dmriprep')

    if not os.path.exists(base_dir):
        os.makedirs(base_dir, exist_ok=True)

    data_files = {}
    for subject in subjects:
        # We make a single session folder per subject for this case, because
        # AFQ api expects session structure:
        sub_dir = op.join(base_dir, f'sub-{subject}')
        sess_dir = op.join(sub_dir, "ses-01")
        if not os.path.exists(sub_dir):
            os.makedirs(os.path.join(sess_dir, 'dwi'), exist_ok=True)
            os.makedirs(os.path.join(sess_dir, 'anat'), exist_ok=True)
        data_files[op.join(sess_dir, 'dwi', f'sub-{subject}_dwi.bval')] =\
            f'{study}/{subject}/T1w/Diffusion/bvals'
        data_files[op.join(sess_dir, 'dwi', f'sub-{subject}_dwi.bvec')] =\
            f'{study}/{subject}/T1w/Diffusion/bvecs'
        data_files[op.join(sess_dir, 'dwi', f'sub-{subject}_dwi.nii.gz')] =\
            f'{study}/{subject}/T1w/Diffusion/data.nii.gz'
        data_files[op.join(sess_dir, 'anat', f'sub-{subject}_T1w.nii.gz')] =\
            f'{study}/{subject}/T1w/T1w_acpc_dc.nii.gz'
        data_files[op.join(sess_dir, 'anat',
                           f'sub-{subject}_aparc+aseg_seg.nii.gz')] =\
            f'{study}/{subject}/T1w/aparc+aseg.nii.gz'

    download_files = {}
    for k in data_files.keys():
        if not op.exists(k):
            download_files[k] = data_files[k]
    if len(download_files.keys()):
        with tqdm(total=len(download_files.keys())) as pbar:
            for k in download_files.keys():
                pbar.set_description_str(f"Downloading {k}")
                bucket.download_file(data_files[k], k)
                pbar.update()

    # Create the BIDS dataset description file text
    hcp_acknowledgements = """Data were provided by the Human Connectome Project, WU-Minn Consortium (Principal Investigators: David Van Essen and Kamil Ugurbil; 1U54MH091657) funded by the 16 NIH Institutes and Centers that support the NIH Blueprint for Neuroscience Research; and by the McDonnell Center for Systems Neuroscience at Washington University.""",  # noqa
    to_bids_description(op.join(my_path, study),
                        **{"Name": study,
                           "Acknowledgements": hcp_acknowledgements,
                           "Subjects": subjects})

    # Create the BIDS derivatives description file text
    to_bids_description(base_dir,
                        **{"Name": study,
                           "Acknowledgements": hcp_acknowledgements,
                           "PipelineDescription": {'Name': 'dmriprep'},
                           "GeneratedBy": [{'Name': 'dmriprep'}]})

    return data_files, op.join(my_path, study)


def fetch_hbn_preproc(subjects, path=None):
    """
    Fetches data from the Healthy Brain Network POD2 study [1, 2]_.

    Parameters
    ----------
    subjects : list
        Identifiers of the subjects to download.
        For example: ["NDARAA112DMH", "NDARAA117NEJ"].
    path : string, optional
        Path to save files into. Default: '~/AFQ_data'

    Returns
    -------
    dict with remote and local names of these files,
    path to BIDS derivative dataset

    Notes
    -----

    .. [1] Alexander LM, Escalera J, Ai L, et al. An open resource for
        transdiagnostic research in pediatric mental health and learning
        disorders. Sci Data. 2017;4:170181.

    .. [2] Richie-Halford A, Cieslak M, Ai L, et al. An analysis-ready and
        quality controlled resource for pediatric brain white-matter research.
        Scientific Data. 2022;9(1):1-27.

    """
    # Anonymous access:
    client = boto3.client('s3', config=Config(signature_version=UNSIGNED))

    if path is None:
        if not op.exists(afq_home):
            os.mkdir(afq_home)
        my_path = afq_home
    else:
        my_path = path

    base_dir = op.join(my_path, "HBN", 'derivatives', 'qsiprep')

    if not os.path.exists(base_dir):
        os.makedirs(base_dir, exist_ok=True)

    data_files = {}

    for subject in subjects:
        initial_query = client.list_objects(
            Bucket="fcp-indi",
            Prefix=f"data/Projects/HBN/BIDS_curated/sub-{subject}/")
        ses = initial_query['Contents'][0]["Key"].split('/')[5]
        query = client.list_objects(
            Bucket="fcp-indi",
            Prefix=f"data/Projects/HBN/BIDS_curated/derivatives/qsiprep/sub-{subject}/")  # noqa
        file_list = [kk["Key"] for kk in query["Contents"]]
        sub_dir = op.join(base_dir, f'sub-{subject}')
        ses_dir = op.join(sub_dir, ses)
        if not os.path.exists(sub_dir):
            os.makedirs(os.path.join(sub_dir, 'anat'), exist_ok=True)
            os.makedirs(os.path.join(sub_dir, 'figures'), exist_ok=True)
            os.makedirs(os.path.join(ses_dir, 'dwi'), exist_ok=True)
            os.makedirs(os.path.join(ses_dir, 'anat'), exist_ok=True)
        for remote in file_list:
            full = remote.split("Projects")[-1][1:].replace("/BIDS_curated", "")
            local = op.join(my_path, full)
            data_files[local] = remote

    download_files = {}
    for k in data_files.keys():
        if not op.exists(k):
            download_files[k] = data_files[k]
    if len(download_files.keys()):
        with tqdm(total=len(download_files.keys())) as pbar:
            for k in download_files.keys():
                pbar.set_description_str(f"Downloading {k}")
                client.download_file("fcp-indi", download_files[k], k)
                pbar.update()

    # Create the BIDS dataset description file text
    hbn_acknowledgements = """XXX""",  # noqa
    to_bids_description(op.join(my_path, "HBN"),
                        **{"Name": "HBN",
                           "Acknowledgements": hbn_acknowledgements,
                           "Subjects": subjects})

    # Create the BIDS derivatives description file text
    to_bids_description(base_dir,
                        **{"Name": "HBN",
                           "Acknowledgements": hbn_acknowledgements,
                           "PipelineDescription": {'Name': 'qsiprep'},
                           "GeneratedBy": [{'Name': 'qsiprep'}]})

    return data_files, op.join(my_path, "HBN")


def fetch_hbn_afq(subjects, path=None):
    """
    Fetches AFQ derivatives for Healthy Brain Network POD2 study [1, 2]_.

    Parameters
    ----------
    subjects : list
        Identifiers of the subjects to download.
        For example: ["NDARAA112DMH", "NDARAA117NEJ"].
    path : string, optional
        Path to save files into. Default: '~/AFQ_data'

    Returns
    -------
    dict with remote and local names of these files,
    path to BIDS derivative dataset

    Notes
    -----

    .. [1] Alexander LM, Escalera J, Ai L, et al. An open resource for
        transdiagnostic research in pediatric mental health and learning
        disorders. Sci Data. 2017;4:170181.

    .. [2] Richie-Halford A, Cieslak M, Ai L, et al. An analysis-ready and
        quality controlled resource for pediatric brain white-matter research.
        Scientific Data. 2022;9(1):1-27.

    """
    # Anonymous access:
    client = boto3.client('s3', config=Config(signature_version=UNSIGNED))

    if path is None:
        if not op.exists(afq_home):
            os.mkdir(afq_home)
        my_path = afq_home
    else:
        my_path = path

    base_dir = op.join(my_path, "HBN", 'derivatives', 'afq')

    if not os.path.exists(base_dir):
        os.makedirs(base_dir, exist_ok=True)

    data_files = {}

    for subject in subjects:
        initial_query = client.list_objects(
            Bucket="fcp-indi",
            Prefix=f"data/Projects/HBN/BIDS_curated/sub-{subject}/")
        ses = initial_query['Contents'][0]["Key"].split('/')[5]
        query = client.list_objects(
            Bucket="fcp-indi",
            Prefix=f"data/Projects/HBN/BIDS_curated/derivatives/afq/sub-{subject}/")  # noqa
        file_list = [kk["Key"] for kk in query["Contents"]]
        sub_dir = op.join(base_dir, f'sub-{subject}')
        ses_dir = op.join(sub_dir, ses)

        for deriv_dir in ["bundles",
                          "clean_bundles",
                          "ROIs",
                          "tract_profile_plots",
                          "viz_bundles"]:
            this_deriv = os.path.join(ses_dir, deriv_dir)
            if not os.path.exists(this_deriv):
                os.makedirs(this_deriv, exist_ok=True)
        for remote in file_list:
            full = remote.split("Projects")[-1][1:].replace("/BIDS_curated", "")
            local = op.join(afq_home, full)
            data_files[local] = remote

    download_files = {}
    for k in data_files.keys():
        if not op.exists(k):
            download_files[k] = data_files[k]
    if len(download_files.keys()):
        with tqdm(total=len(download_files.keys())) as pbar:
            for k in download_files.keys():
                pbar.set_description_str(f"Downloading {k}")
                client.download_file("fcp-indi", download_files[k], k)
                pbar.update()

    # Create the BIDS dataset description file text
    hbn_acknowledgements = """XXX""",  # noqa
    to_bids_description(op.join(my_path, "HBN"),
                        **{"Name": "HBN",
                           "Acknowledgements": hbn_acknowledgements,
                           "Subjects": subjects})

    # Create the BIDS derivatives description file text
    to_bids_description(base_dir,
                        **{"Name": "HBN",
                           "PipelineDescription": {'Name': 'afq'},
                           "GeneratedBy": [{'Name': 'afq'}]})

    return data_files, op.join(my_path, "HBN")
