###############################################################################
# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
#                                                                             #
# This software is distributed under the terms of the GNU General Public      #
# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".   #
#                                                                             #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization  #
# or submit itself to any jurisdiction.                                       #
###############################################################################
"""Methods for scanning the popularity table."""
import os
import time
from datetime import datetime, timedelta

import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers, resolveSEGroup

from LHCbDIRAC.DataManagementSystem.Client.StorageUsageClient import StorageUsageClient
from LHCbDIRAC.DataManagementSystem.Client.DataUsageClient import DataUsageClient
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
from LHCbDIRAC.BookkeepingSystem.Client.BKQuery import BKQuery, makeBKPath
from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient

# Code
dmsHelper = DMSHelpers()

# Dictionary containing the BK path for each directory
bkPathForDir = {}
# Set of invisible BK paths
cachedInvisible = set()
# Set of productions for each BK path
prodForBKPath = {}
# Storage usage for each BK path
bkPathUsage = {}
# Processing pass for each BK path
processingPass = {}
# PFN #files and size for each BK path
physicalStorageUsage = {}

# Dictionary with weekly/dayly usage for each BK path
bkPathPopularity = {}

storageTypes = ("Disk", "Tape", "Archived", "All", "LFN")
# Set of BK paths for each storage types (above)
datasetStorage = {}

# We are only interested in CERN and Tier1s
storageSites = dmsHelper.getTiers(tier=(0, 1))
# Caching the site at which an SE is
cachedSESites = {}

# Instantiate clients
duClient = DataUsageClient()
suClient = StorageUsageClient()
bkClient = BookkeepingClient()
transClient = TransformationClient()


def getTimeBin(date):
    """get the bin number as week number."""
    year, week, _day = date.isocalendar()
    week += 52 * year
    return week


def cacheDirectories(directories):
    """Get directories information from popularity and from storageUsage and
    cache the information."""

    # Ignore already cached directories
    dirSet = directories - set(bkPathForDir) - cachedInvisible
    if not dirSet:
        return
    startTime = time.time()
    gLogger.always(f"\nCaching {len(directories)} directories, {len(dirSet)} not cached yet")

    # Some DBs refer to "long" directory names (i.e. with additional number, see below)
    #     while others refer to the unnumbered directory.
    # Make conversion tables and keep only the first directory when numbered
    # Short is for example /lhcb/LHCb/Collision18/RDST/00074101/
    # Long is for example /lhcb/LHCb/Collision18/RDST/00074101/0008/
    dirLong2Short = {}
    dirShort2Long = {}
    for dirLfn in sorted(dirSet):
        splitDir = dirLfn.split("/")
        if len(splitDir[-2]) == 4 and splitDir[-2].isdigit():
            short = "/".join(splitDir[:-2])
        else:
            short = dirLfn
        dirLong2Short[dirLfn] = short
        dirShort2Long.setdefault(short, dirLfn)

    # # Cache BK path for directories
    # First try cached information in Data Usage DB
    chunkSize = 1000
    gLogger.always(f"Getting cache metadata for {len(dirShort2Long)} directories")
    dirMetadata = {}
    for longDirs in breakListIntoChunks(dirShort2Long.values(), chunkSize):
        res = duClient.getDirMetadata(longDirs)
        if not res["OK"]:
            gLogger.error("\nError getting metadata from DataUsage", res["Message"])
        else:
            dirMetadata.update(res["Value"])

    invisible = set()
    lfnsFromBK = set()
    for dirLfn in dirSet:
        longDir = dirShort2Long[dirLong2Short[dirLfn]]
        metadata = dirMetadata.get(longDir)
        if metadata:
            if len(metadata) < 9 or metadata[8] != "N":
                bkPathForDir[dirLfn] = os.path.join(
                    "/", metadata[1], metadata[2], metadata[3], metadata[4][1:], metadata[5], metadata[6]
                )
                processingPass[bkPathForDir[dirLfn]] = metadata[4]
                prodForBKPath.setdefault(bkPathForDir[dirLfn], set()).add(metadata[7])
            else:
                invisible.add(dirLfn)
        else:
            lfnsFromBK.add(longDir)

    # For those not available ask the BK
    chunkSize = 100
    if lfnsFromBK:
        gLogger.always(f"Getting BK metadata for {len(lfnsFromBK)} directories")
        success = {}
        for lfns in breakListIntoChunks(lfnsFromBK, chunkSize):
            for trial in range(10, -1, -1):
                res = bkClient.getDirectoryMetadata(lfns)
                if not res["OK"]:
                    if not trial:
                        gLogger.fatal("\nError getting BK metadata", res["Message"])
                        DIRAC.exit(1)
                    else:
                        gLogger.warn("Error getting BK metadata, retrying", res["Message"])
                else:
                    break
            success.update(res["Value"].get("Successful", {}))
        gLogger.always(f"Successfully obtained BK metadata for {len(success)} directories")
        for dirLfn in dirSet - set(bkPathForDir):
            longDir = dirShort2Long[dirLong2Short[dirLfn]]
            metadata = success.get(longDir, [{}])[0]
            if metadata:
                if metadata.get("VisibilityFlag", metadata.get("Visibility", "Y")) == "Y":
                    bkDict = BKQuery(metadata).getQueryDict()
                    bkPath = makeBKPath(bkDict)
                    processingPass[bkPath] = metadata["ProcessingPass"]
                    prodForBKPath.setdefault(bkPath, set()).add(metadata["Production"])
                else:
                    invisible.add(dirLfn)
            else:
                bkPath = "Unknown-" + dirLfn
                prodForBKPath.setdefault(bkPath, set())
            if dirLfn not in invisible:
                bkPathForDir[dirLfn] = bkPath
                bkPathUsage.setdefault(bkPath, {}).setdefault("LFN", [0, 0])

    # Ignore invisible directories
    if invisible:
        gLogger.always(f"{len(invisible)} paths are ignored as invisible")
        cachedInvisible.update(invisible)
        dirSet -= invisible

    # For remaining directories, get the LFN information and the PFN information SE by SE
    if dirSet:
        missingSU = {dirLong2Short[dirLfn] for dirLfn in dirSet}
        gLogger.always(f"Get LFN Storage Usage for {len(missingSU)} directories")
        for dirLfn in missingSU:
            # LFN usage
            for trial in range(10, -1, -1):
                res = suClient.getSummary(dirLfn)
                if res["OK"]:
                    break
                if not trial:
                    gLogger.fatal(f"Error getting LFN storage usage {dirLfn}", res["Message"])
                    DIRAC.exit(1)
            bkPath = bkPathForDir[dirShort2Long[dirLfn]]
            infoType = "LFN"
            gLogger.verbose(f"Directory {dirLfn}: {str(res['Value'])}")
            bkPathUsage.setdefault(bkPath, {}).setdefault(infoType, [0, 0])
            bkPathUsage[bkPath][infoType][0] += sum(val.get("Files", 0) for val in res["Value"].values())
            bkPathUsage[bkPath][infoType][1] += sum(val.get("Size", 0) for val in res["Value"].values())

        # # get the PFN usage per storage type
        # Storage type is Disk, Archived, Tape and All
        gLogger.always(f"Check storage type and PFN usage for {len(dirSet)} directories")
        for dirLfn in dirSet:
            for trial in range(10, -1, -1):
                res = suClient.getDirectorySummaryPerSE(dirLfn)
                if not res["OK"]:
                    if not trial:
                        gLogger.fatal("Error getting storage usage per SE", f"{dirLfn} {res['Message']}")
                        DIRAC.exit(1)
                    else:
                        gLogger.warn("Error getting storage usage per SE, retrying", f"{dirLfn} {res['Message']}")
                else:
                    break
            info = physicalStorageUsage.setdefault(dirLfn, {})
            for infoType in storageTypes:
                # Active type will be recorded in Disk, just a special flag
                if infoType != "LFN" and infoType not in info:
                    nf = sum(val["Files"] for se, val in res["Value"].items() if isType(se, infoType))
                    size = sum(val["Size"] for se, val in res["Value"].items() if isType(se, infoType))
                    info[infoType] = {"Files": nf, "Size": size}
            for site in storageSites:
                if site not in info:
                    nf = sum(
                        val["Files"] for se, val in res["Value"].items() if isAtSite(se, site) and isType(se, "Disk")
                    )
                    size = sum(
                        val["Size"] for se, val in res["Value"].items() if isAtSite(se, site) and isType(se, "Disk")
                    )
                    info[site] = {"Files": nf, "Size": size}
            bkPath = bkPathForDir[dirLfn]
            for infoType in info:
                bkPathUsage.setdefault(bkPath, {}).setdefault(infoType, [0, 0])
                bkPathUsage[bkPath][infoType][0] += info[infoType].get("Files", 0)
                bkPathUsage[bkPath][infoType][1] += info[infoType].get("Size", 0)
            datasetStorage[storageType(res["Value"])].add(bkPath)

    gLogger.always(
        f"Obtained BK path and storage usage of {len(dirSet)} directories in {time.time() - startTime:.1f} seconds"
    )


def isType(se, infoType):
    """check if an SE is of a given type."""
    if infoType == "All":
        return True
    if infoType == "LFN":
        return False
    return storageType([se]) == infoType


def isAtSite(se, site):
    """checks if an SE is at a given site."""
    seSite = cachedSESites.get(se)
    if seSite is None:
        seSite = dmsHelper.getLocalSiteForSE(se)
        if seSite["OK"]:
            cachedSESites[se] = seSite["Value"]
    return site == seSite


def prBinNumber(binNumber):
    """Return bin number as a string."""
    week = binNumber
    year = week / 52
    week = week % 52
    return "y%4d/w%02d" % (year, week)


def prSize(size):
    """Return size as a string with sensible unit."""
    units = ("Bytes", "kB", "MB", "GB", "TB", "PB")
    for unit in units:
        if size < 1000.0:
            return f"{size:.1f} {unit}"
        size /= 1000.0
    return "???"


def getPhysicalUsage(baseDir):
    """Extract information about storage usage from the StorageusageDB."""
    for trial in range(10, -1, -1):
        res = suClient.getStorageDirectoryData(baseDir, None, None, None, timeout=3600)
        if not res["OK"]:
            if not trial:
                gLogger.fatal(f"Error getting list of directories for {baseDir}", res["Message"])
                DIRAC.exit(1)
        else:
            break
    # The returned value is a dictionary of all directory leaves
    info = res["Value"]
    dirSet = set()
    nf = 0
    size = 0
    gLogger.info(f"Storage usage OK for {len(info)} directories in {baseDir}")
    for directory in info:
        nodes = directory.split("/")
        try:
            prod = int(nodes[5])
            # FIXME
            # Don't count the certification productions
            if prod < 2000 and nodes[2] == "validation":
                continue
        except (IndexError, ValueError):
            pass
        if len(nodes) < 5 or nodes[4] not in ("LOG", "HIST", "BRUNELHIST", "DAVINCIHIST", "GAUSSHIST"):
            physicalStorageUsage.setdefault(directory, {}).setdefault("All", info[directory])
            dirSet.add(directory)
            nf += info[directory]["Files"]
            size += info[directory]["Size"]
    # In case the base directory was not a leave, cache its physical storage data
    physicalStorageUsage.setdefault(baseDir, {}).setdefault("All", {"Files": nf, "Size": size})
    return dirSet


def storageType(seList):
    """Return the storage type: disk, tape or archive."""
    if not (
        {se for se in seList if not dmsHelper.isSEArchive(se)} - {"CERN-SWTEST", "CERN-FREEZER-EOS", "CERN-FREEZER"}
    ) and {se for se in seList if dmsHelper.isSEArchive(se)}:
        # Only -ARCHIVE
        return "Archived"
    tapeSEs = resolveSEGroup("Tier1-RAW") + resolveSEGroup("Tier1-RDST")
    if [se for se in seList if se in tapeSEs]:
        # Any file on Tape
        return "Tape"
    # Others
    return "Disk"


def scanPopularity(since, getAllDatasets, topDirectory="/lhcb", csvFile=None):
    """That function does the job to cache the directories, get the corresponding
    datasets and join with the popularity."""
    # Reset global variables

    bkPathForDir.clear()
    cachedInvisible.clear()
    prodForBKPath.clear()
    bkPathUsage.clear()
    processingPass.clear()
    bkPathPopularity.clear()
    physicalStorageUsage.clear()
    datasetStorage.clear()
    for infoType in storageTypes:
        datasetStorage[infoType] = set()

    # set of used directories
    usedDirectories = set()
    usedSEs = {}
    binSize = "week"
    nbBins = int((since + 6) / 7)
    since = 7 * nbBins

    from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations

    ignoreDirectories = Operations().getValue(
        "DataManagement/PopularityIgnoreDirectories",
        ["user", "test", "debug", "dataquality", "software", "database", "swtest", "certification", "validation"],
    )
    nowBin = getTimeBin(datetime.now() - timedelta(days=1))
    notCached = set()

    if getAllDatasets:
        # Get list of directories
        startTime = time.time()
        res = FileCatalog().listDirectory(topDirectory)
        if not res["OK"]:
            gLogger.fatal("Cannot get list of directories", res["Message"])
            DIRAC.exit(1)
        directories = {
            subDir
            for subDir in res["Value"]["Successful"][topDirectory]["SubDirs"]
            if subDir.split("/")[2] not in ignoreDirectories
            and "RAW" not in subDir
            and "RDST" not in subDir
            and "SDST" not in subDir
        }
        allDirectoriesSet = set()
        for baseDir in directories:
            allDirectoriesSet.update(getPhysicalUsage(baseDir))
        gLogger.always(
            "Obtained %d directories storage usage information in %.1f seconds)"
            % (len(allDirectoriesSet), time.time() - startTime)
        )
        cacheDirectories(allDirectoriesSet)

    # Get the popularity raw information for the specified number of days
    if since:
        entries = 0
        now = datetime.now()
        endTime = datetime(now.year, now.month, now.day, 0, 0, 0)
        startTime = endTime
        gLogger.always("Get popularity day-by-day")
        stTime = time.time()
        for _i in range(since):
            endTime = startTime
            startTime = endTime - timedelta(days=1)
            endTimeQuery = endTime.isoformat()
            startTimeQuery = startTime.isoformat()
            status = "Used"
            # Getting the popularity with 10 retries for the given day
            for _i in range(10):
                res = duClient.getDataUsageSummary(startTimeQuery, endTimeQuery, status, timeout=7200)
                if res["OK"]:
                    break
                gLogger.error("Error getting popularity entries, retrying...", res["Message"])
            if not res["OK"]:
                gLogger.fatal("Error getting popularity entries", res["Message"])
                DIRAC.exit(1)
            val = res["Value"]
            entries += len(val)

            # Get information on useful directories
            directories = {row[1] for row in val if row[1].split("/")[2] not in ignoreDirectories}
            usedDirectories.update(directories)
            cacheDirectories(directories)

            # Get information in bins (day or week)
            for _rowId, dirLfn, se, count, insertTime in val:
                if dirLfn not in directories:
                    # print rowId, dirLfn, count, insertTime, 'ignored'
                    continue
                # get the binNumber (day or week)
                binNumber = getTimeBin(insertTime)
                bkPath = bkPathForDir.get(dirLfn)
                if not bkPath:
                    if dirLfn not in notCached:
                        notCached.add(dirLfn)
                        gLogger.error(f"Directory {dirLfn} was not cached")
                    bkPath = "Unknown-" + dirLfn
                bkPathPopularity[bkPath][binNumber] = (
                    bkPathPopularity.setdefault(bkPath, {}).setdefault(binNumber, 0) + count
                )
                usedSEs.setdefault(bkPath, set()).add(se)

        gLogger.always("\n=============================================================")
        gLogger.always("Retrieved %d entries from Popularity table in %.1f seconds" % (entries, time.time() - stTime))
        gLogger.always("Found %d datasets used since %d days" % (len(bkPathPopularity), since))
        counters = {}
        strangeBKPaths = {
            bkPath for bkPath in bkPathPopularity if not bkPathUsage.get(bkPath, {}).get("LFN", (0, 0))[0]
        }
        if strangeBKPaths:
            gLogger.always(f"{len(strangeBKPaths)} used datasets do not have an LFN count:")
            gLogger.always("\n".join(f"{bkPath} : {str(bkPathUsage.get(bkPath, {}))}" for bkPath in strangeBKPaths))
        gLogger.always(f"\nDataset usage for {len(bkPathPopularity)} datasets")
        for infoType in ("All", "LFN"):
            for i in range(2):
                counters.setdefault(infoType, []).append(
                    sum(bkPathUsage.get(bkPath, {}).get(infoType, (0, 0))[i] for bkPath in bkPathPopularity)
                )
        for bkPath in sorted(bkPathPopularity):
            if bkPath not in datasetStorage["Disk"] | datasetStorage["Archived"] | datasetStorage["Tape"]:
                ses = usedSEs.get(bkPath)
                if ses is None:
                    gLogger.error("BK path not in usedSEs", bkPath)
                else:
                    datasetStorage[storageType(ses)].add(bkPath)
            nLfns, lfnSize = bkPathUsage.get(bkPath, {}).get("LFN", (0, 0))
            nPfns, pfnSize = bkPathUsage.get(bkPath, {}).get("All", (0, 0))
            gLogger.always(
                "%s (%d LFNs, %s), (%d PFNs, %s, %.1f replicas)"
                % (
                    bkPath,
                    nLfns,
                    prSize(lfnSize),
                    nPfns,
                    prSize(pfnSize),
                    float(nPfns) / float(nLfns) if nLfns else 0.0,
                )
            )
            bins = sorted(bkPathPopularity[bkPath])
            lastBin = bins[-1]
            accesses = sum(bkPathPopularity[bkPath][binNumber] for binNumber in bins)
            gLogger.always(
                "\tUsed first in %s, %d accesses (%.1f%%), %d accesses during last %s %s"
                % (
                    prBinNumber(bins[0]),
                    accesses,
                    accesses * 100.0 / nLfns if nLfns else 0.0,
                    bkPathPopularity[bkPath][lastBin],
                    binSize,
                    prBinNumber(lastBin),
                )
            )
        gLogger.always(
            "\nA total of %d LFNs (%s), %d PFNs (%s) have been used"
            % (counters["LFN"][0], prSize(counters["LFN"][1]), counters["All"][0], prSize(counters["All"][1]))
        )

    if getAllDatasets:
        # Consider only unused directories
        unusedDirectories = allDirectoriesSet - usedDirectories
        if unusedDirectories:
            gLogger.always("\n=============================================================")
            gLogger.always(f"{len(unusedDirectories)} directories have not been used")
            # Remove the used datasets (from other directories)
            unusedBKPaths = {bkPathForDir[dirLfn] for dirLfn in unusedDirectories if dirLfn in bkPathForDir} - set(
                bkPathPopularity
            )
            # Remove empty datasets
            strangeBKPaths = {
                bkPath for bkPath in unusedBKPaths if not bkPathUsage.get(bkPath, {}).get("LFN", (0, 0))[0]
            }
            if strangeBKPaths:
                gLogger.always(f"{len(strangeBKPaths)} unused datasets do not have an LFN count:")
                gLogger.always("\n".join(f"{bkPath} : {str(bkPathUsage.get(bkPath, {}))}" for bkPath in strangeBKPaths))
            unusedBKPaths = {bkPath for bkPath in unusedBKPaths if bkPathUsage.get(bkPath, {}).get("LFN", (0, 0))[0]}

            # In case there are datasets both on tape and disk, priviledge tape
            datasetStorage["Disk"] -= datasetStorage["Tape"]
            gLogger.always("\nThe following %d BK paths were not used since %d days" % (len(unusedBKPaths), since))
            for infoType in storageTypes[0:3]:
                gLogger.always(f"\n=========== {infoType} datasets ===========")
                unusedPaths = unusedBKPaths & datasetStorage[infoType]
                counters = {}
                for ty in ("All", "LFN"):
                    for i in range(2):
                        counters.setdefault(ty, []).append(
                            sum(bkPathUsage.get(bkPath, {}).get(ty, (0, 0))[i] for bkPath in unusedPaths)
                        )
                for bkPath in sorted(unusedPaths):
                    nLfns, lfnSize = bkPathUsage.get(bkPath, {}).get("LFN", (0, 0))
                    nPfns, pfnSize = bkPathUsage.get(bkPath, {}).get("All", (0, 0))
                    gLogger.always(
                        "\t%s (%d LFNs, %s), (%d PFNs, %s, %.1f replicas)"
                        % (bkPath, nLfns, prSize(lfnSize), nPfns, prSize(pfnSize), float(nPfns) / float(nLfns))
                    )
                gLogger.always(
                    "\nA total of %d %s LFNs (%s), %d PFNs (%s) were not used"
                    % (
                        counters["LFN"][0],
                        infoType,
                        prSize(counters["LFN"][1]),
                        counters["All"][0],
                        prSize(counters["All"][1]),
                    )
                )
    else:
        unusedBKPaths = set()
        datasetStorage["Disk"] -= datasetStorage["Tape"]

    # Now create a CSV file with all dataset information
    # Name, ProcessingPass, #files, size, SE type, each week's usage (before now)
    csvFile = "popularity-%ddays.csv" % since if csvFile is None else csvFile
    gLogger.always("\n=============================================================")
    gLogger.always("Creating %s file with %d datasets" % (csvFile, len(bkPathPopularity) + len(unusedBKPaths)))
    with open(csvFile, "w") as fd:
        title = (
            f"Name;Configuration;ProcessingPass;FileType;Type;Creation-{binSize};"
            + "NbLFN;LFNSize;NbDisk;DiskSize;NbTape;TapeSize;NbArchived;ArchivedSize;"
            + ";".join(site.split(".")[1] for site in storageSites)
            + ";Nb Replicas;Nb ArchReps;Storage;FirstUsage;LastUsage;Now"
        )
        for binNumber in range(nbBins):
            title += ";%d" % (1 + binNumber)
        fd.write(title + "\n")
        teraByte = 1000.0 * 1000.0 * 1000.0 * 1000.0
        for bkPath in sorted(bkPathPopularity) + sorted(unusedBKPaths):
            # Skip unknown datasets
            if bkPath.startswith("Unknown-"):
                continue
            # Not interested in histograms
            splitBKPath = bkPath.split("/")
            fileType = splitBKPath[-1]
            if "HIST" in fileType:
                continue
            # Only RAW for partition LHCb may be of interest (and even...)
            if fileType == "RAW" and not bkPath.startswith("/LHCb"):
                continue
            info = bkPathUsage.get(bkPath, {})
            # check if the production is still active
            prods = prodForBKPath[bkPath]
            res = transClient.getTransformations({"TransformationID": list(prods)})
            creationTime = datetime.now()
            active = []
            for prodDict in res.get("Value", []):
                creationTime = min(creationTime, prodDict["CreationDate"])
                if prodDict["Status"] in ("Active", "Idle", "Completed"):
                    active.append(str(prodDict["TransformationID"]))
            if active:
                gLogger.always(f"Active productions {','.join(sorted(active))} found in {bkPath}")
            if info["LFN"][0] == 0:
                continue
            for infoType in info:
                info[infoType][1] /= teraByte
            # Some BK paths contain a , to be replaces by a . for the CSV file!!
            config = "/".join(splitBKPath[0:3])
            if "," in bkPath:
                gLogger.always("BK path found with ',':", bkPath)
            # Name,Configuration,ProcessingPass, FileType
            row = "{};{};{};{}".format(
                bkPath.replace("Real Data", "RealData"),
                config,
                processingPass.get(bkPath, "Unknown").replace("Real Data", "RealData"),
                fileType,
            )
            # Type
            configTypes = {"/MC/Dev": 2, "/MC/Upgrade": 3}
            configType = configTypes.get(config, 0 if bkPath.startswith("/MC") else 1)
            row += ";%d" % configType
            # CreationTime
            row += ";%d" % (getTimeBin(creationTime))
            # NbLFN,LFNSize,NbDisk,DiskSize,NbTape,TapeSize, NbArchived,ArchivedSize
            for infoType in ("LFN", "Disk", "Tape", "Archived"):
                row += ";%d;%f" % tuple(info[infoType])
            for site in storageSites:
                row += f";{info[site][1]:f}"
            row += ";{:f};{:f}".format(
                float(info["Disk"][0]) / float(info["LFN"][0]),
                float(info["Archived"][0]) / float(info["LFN"][0]),
            )
            if active:
                dsType = "Active"
            else:
                dsType = "Unknown"
                for infoType in storageTypes[0:3]:
                    if bkPath in datasetStorage[infoType]:
                        dsType = infoType
                        break
            row += f";{dsType}"
            bins = sorted(bkPathPopularity.get(bkPath, {}))
            if not bins:
                bins = [0]
            row += ";%d;%d;%d" % (bins[0], bins[-1], nowBin)
            usage = 0
            for binNumber in range(nbBins):
                usage += bkPathPopularity.get(bkPath, {}).get(nowBin - binNumber, 0)
                row += ";%d" % usage
            fd.write(row + "\n")
    gLogger.always(f"\nSuccessfully wrote CSV file {csvFile}")
