#!/usr/bin/env python

# BSD 3-Clause License; see https://github.com/scikit-hep/aghast/blob/master/LICENSE

import math
import numbers
try:
    from inspect import signature
except ImportError:
    try:
        from funcsigs import signature
    except ImportError:
        raise ImportError("Install funcsigs package with:\n    pip install funcsigs\nor\n    conda install funcsigs\n(or just use Python >= 3.3).")

import numpy

import aghast
import aghast.checktype
import aghast.interface

prologue = u"""= Aghast Specification

== Introduction

Aghast encodes aggregated, histogram-like statistics in a way that can be shared among many libraries that fill, fit, or plot such things. It represents reduced data, such as the output of a histogram-filling (map) and combining (reduce) process, or the output of an SQL-like group-by procedure.

Aghast is not intended to be used directly by data analysts. It is horrifically verbose. Instead, it is a general ontology of aggregated data types with conversion routines to translate among libraries. With the appropriate translations behind the scenes, data analysts can reduce large datasets with one library, model/fit the aggregation with another, and plot it with a third.

This role is inspired by two projects: https://arrow.apache.org[Apache Arrow], which acts as an efficient memory representation for nested, columnar data, and http://dmg.org/pmml/v4-3/GeneralStructure.html[PMML], which is an ontology of data mining models in XML. Like Arrow, aghast is an in-memory format to pass data in a random-access and zero-copy way (though an in-memory format can also be saved to disk). Like PMML, aghast expresses domain-specific statistical concepts, such as histograms, moments, quantiles, covariances, fit functions, and ntuples, rather than generic data types.

Aghast is specified using https://google.github.io/flatbuffers[Google Flatbuffers]. Flatbuffers allows for lazy, random-access interpretation: any nested property can be accessed without reading or deserializing the others. Thus, a "`ghast`" (a sharable object) could contain thousands of histograms and accessing the bin contents of a particular one would not only leave the other histograms untouched, it would leave the histogram's own metadata untouched. Flatbuffers minimizes its serialized size (because it is intended as a network protocol) and its serialization/deserialization time (because it is intended for computer games).

Aghast is designed to scale. Modern particle physics analyses deal in thousands of histograms generated by viewing the same quantities with many different "`cuts`" (selection filters) and variations (to test senstivity to systematic effects). This usually results in duplicated histogram axis descriptions: every histogram in a group of a hundred must be binned the same way. Duplication inflates serialized size, but it also introduces the possibility that some binnings might disagree (due to a copy-paste error) and the need for extra validity checks. Aghast is designed around the idea of "`superhistograms,`" representing groups of histograms that have identical binnings as a binning in a larger histogram. Thus, there are axis types for <<PredicateBinning>> (defined by if-then predicates) and <<VariationBinning>> (defined by systematic variations), and the data filling these bins are contiguous across all histograms in the group.

What this specification does not define are any methods for filling, fitting, or plotting. From a data analyst's perspective, this is everything one does with histograms or other statistical objects. This is because aghast is firmly behind-the-scenes, a helper to other fine analysis tools, such as https://root.cern[ROOT], https://github.com/boostorg/histogram[Boost.Histogram], https://physt.readthedocs.io/en/latest/index.html[Physt], and https://pandas.pydata.org[Pandas].

== Data Types

Aghast is specified as a Flatbuffers format in link:flatbuffers/aghast.fbs[]. Flatbuffers provides a standard suite of types that can be translated into https://google.github.io/flatbuffers/flatbuffers_support.html[many languages]. However, the code the Flatbuffers code generator produces is too low-level even for applications to use as a backend, so we describe the interfaces and types of wrapper classes here.

These class descriptions are sufficiently constrained to fit into any static type system, and while they include heterogeneous lists (lists of an enumerated union type), they can be decomposed into homogeneous lists with an additional level of nesting. In fact, the Flatbuffers specification doesn't allow heterogeneous lists, so it provides an example of this decomposition. (For example, the heterogeneous *objects* list in a <<Collection>>, which can contain <<Histogram>>, <<ParameterizedFunction>>, <<BinnedEvaluatedFunction>>, and <<Ntuple>>, is encoded in Flatbuffers as a homogeneous list of `Object`, which contains a _single_ union of `ObjectData`.)

Basic types, like booleans, integers, floating point numbers, and strings, are passed through without modification (though strings are explicitly encoded as `utf-8`; Flatbuffers strings are not encoding-aware). Integers and floating point numbers may have a constrained range, such as [0, \u221e) for non-negative numbers excluding \u221e or (0, 2\u03c0] for positive values less than or equal to 2\u03c0. (A square bracket includes the endpoint; a round bracket excludes it.) Empty strings and missing strings (null) are distinct.

Lists may contain basic types or class instances, and there is no distinction between empty lists and missing lists (an artifact of Flatbuffers).

In some cases, we want a mapping type, such as str \u2192 X, so that objects are retrievable by name, rather than index. Flatbuffers does not have such a type, so we build it by decomposing the high-level mapping into a low-level pair of lists with equal length. (For example, the *objects* mapping in a <<Collection>> is encoded in Flatbuffers as a list `objects` and a list `lookup`.)

Class objects may be missing (null) if they are not required. Required properties are a Flatbuffers feature: it doesn't generate code that would allow the serialized object to be missing. The class schemas can evolve to include more properties (with full forward and backward compatibility), but properties cannot be removed and required properties cannot become non-required.

Any properties that are not required have a default value (usually null).

In addition to the type constraints, which are tighter in the wrapper classes than they are in the Flatbuffers serialization, we list invariants (conditions that must be true) that depend on multiple properties or multiple class objects. To avoid unnecessary Flatbuffers deserialization, these are not automatically checked, but a check can be invoked and any behavior when those invariants are not satisfied is undefined.

A sharable ghast may have one of the following types: <<Collection>>, <<Histogram>>, <<ParameterizedFunction>>, <<BinnedEvaluatedFunction>>, and <<Ntuple>>."""

epilogue = u""

classes = [
    aghast.Collection,
    aghast.Histogram,
    aghast.Axis,
    aghast.IntegerBinning,
    aghast.RegularBinning,
    aghast.RealInterval,
    aghast.RealOverflow,
    aghast.HexagonalBinning,
    aghast.EdgesBinning,
    aghast.IrregularBinning,
    aghast.CategoryBinning,
    aghast.SparseRegularBinning,
    aghast.FractionBinning,
    aghast.PredicateBinning,
    aghast.VariationBinning,
    aghast.Variation,
    aghast.Assignment,
    aghast.UnweightedCounts,
    aghast.WeightedCounts,
    aghast.InterpretedInlineBuffer,
    aghast.InterpretedInlineInt64Buffer,
    aghast.InterpretedInlineFloat64Buffer,
    aghast.InterpretedExternalBuffer,
    aghast.Profile,
    aghast.Statistics,
    aghast.Moments,
    aghast.Quantiles,
    aghast.Modes,
    aghast.Extremes,
    aghast.StatisticFilter,
    aghast.Covariance,
    aghast.ParameterizedFunction,
    aghast.Parameter,
    aghast.EvaluatedFunction,
    aghast.BinnedEvaluatedFunction,
    aghast.Ntuple,
    aghast.Column,
    aghast.NtupleInstance,
    aghast.Chunk,
    aghast.ColumnChunk,
    aghast.Page,
    aghast.RawInlineBuffer,
    aghast.RawExternalBuffer,
    aghast.Metadata,
    aghast.Decoration,
    ]

unions = {
    aghast.interface.RawBuffer: [aghast.RawInlineBuffer, aghast.RawExternalBuffer],
    aghast.interface.InterpretedBuffer: [aghast.InterpretedInlineBuffer, aghast.InterpretedInlineInt64Buffer, aghast.InterpretedInlineFloat64Buffer, aghast.InterpretedExternalBuffer],
    aghast.interface.Binning: [aghast.IntegerBinning, aghast.RegularBinning, aghast.HexagonalBinning, aghast.EdgesBinning, aghast.IrregularBinning, aghast.CategoryBinning, aghast.SparseRegularBinning, aghast.FractionBinning, aghast.PredicateBinning, aghast.VariationBinning],
    aghast.interface.Counts: [aghast.UnweightedCounts, aghast.WeightedCounts],
    aghast.interface.Function: [aghast.ParameterizedFunction, aghast.EvaluatedFunction],
    aghast.interface.FunctionObject: [aghast.ParameterizedFunction, aghast.BinnedEvaluatedFunction],
    aghast.interface.Object: [aghast.Histogram, aghast.Ntuple, aghast.ParameterizedFunction, aghast.BinnedEvaluatedFunction, aghast.Collection],
    }

def num(x):
    if not isinstance(x, (bool, numpy.bool, numpy.bool)) and isinstance(x, (numbers.Real, numpy.integer, numpy.floating)):
        if x == float("-inf"):
            return u"\u2012\u221e"
        elif x == float("inf"):
            return u"\u221e"
        elif x == -0.5*math.pi:
            return u"\u2012\u03c0/2"
        elif x == 0.5*math.pi:
            return u"\u03c0/2"
        elif x == aghast.interface.MININT64:
            return u"\u20122\u2076\u00b3"  # -2**63
        elif x == aghast.interface.MAXINT64:
            return u"2\u2076\u00b3 \u2012 1"  # 2**63 - 1
        elif x == 0.5:
            return "1/2"
        elif x < 0:
            return u"\u2012" + repr(abs(x))
        else:
            return repr(abs(x))
    elif x is None:
        return "null"
    elif x is True:
        return "true"
    elif x is False:
        return "false"
    elif x == []:
        return "null/empty"
    else:
        return "`+{0}+`".format(repr(x))

def formatted(cls, end="\n"):
    out = [end + end + "== {0}{1}".format(cls.__name__, end), "*" + cls.description.strip() + "*"]

    out.append(end + "[%hardbreaks]")
    for name, param in signature(cls.__init__).parameters.items():
        if name != "self":
            check = cls._params[name]
            hasdefault = param.default is not param.empty

            islist = False
            if isinstance(check, aghast.checktype.CheckBool):
                typestring = "bool"
            elif isinstance(check, aghast.checktype.CheckString):
                typestring = "str"
            elif isinstance(check, aghast.checktype.CheckNumber):
                typestring = "float in {0}{1}, {2}{3}".format(
                    "[" if check.min_inclusive else "(",
                    num(check.min),
                    num(check.max),
                    "]" if check.max_inclusive else ")")
            elif isinstance(check, aghast.checktype.CheckInteger):
                typestring = "int in {0}{1}, {2}{3}".format(
                    "(" if check.min == float("-inf") else "[",
                    num(check.min),
                    num(check.max),
                    ")" if check.max == float("inf") else "]")
            elif isinstance(check, aghast.checktype.CheckEnum):
                typestring = "one of {" + ", ".join("`+" + str(x) + "+`" for x in check.choices) + "}"
            elif isinstance(check, aghast.checktype.CheckClass):
                if check.type in unions:
                    typestring = " or ".join("<<{0}>>".format(x.__name__) for x in unions[check.type])
                else:
                    typestring = "<<{0}>>".format(check.type.__name__)
            elif isinstance(check, aghast.checktype.CheckKey) and check.type is str:
                typestring = "unique str"
            elif isinstance(check, (aghast.checktype.CheckVector, aghast.checktype.CheckLookup)):
                islist = True
                if check.type is str:
                    subtype = "str"
                elif check.type is int:
                    subtype = "int"
                elif check.type is float:
                    subtype = "float"
                elif isinstance(check.type, list):
                    subtype = "{" + ", ".join("`+" + str(x) + "+`" for x in check.type) + "}"
                else:
                    if check.type in unions:
                        subtype = " or ".join("<<{0}>>".format(x.__name__) for x in unions[check.type])
                    else:
                        subtype = "<<{0}>>".format(check.type.__name__)
                if check.minlen != 0 or check.maxlen != float("inf"):
                    withlength = " with length in [{0}, {1}{2}".format(
                        num(check.minlen),
                        num(check.maxlen),
                        ")" if check.maxlen == float("inf") else "]")
                else:
                    withlength = ""
                if isinstance(check, aghast.checktype.CheckVector):
                    typestring = "list of {0}{1}".format(subtype, withlength)
                else:
                    typestring = u"str \u2192 {0}{1}".format(subtype, withlength)
            elif isinstance(check, aghast.checktype.CheckBuffer):
                typestring = "buffer"
            elif isinstance(check, aghast.checktype.CheckSlice):
                typestring = "slice (start:stop:step)"
            else:
                raise AssertionError(type(check))

            linebreak = " +" + end if len(name) + len(typestring) > 75 else " "
            if check.required:
                defaultstring = "(required)"
            elif hasdefault:
                defaultstring = "(default: {0})".format(num([] if islist and param.default is None else param.default))
            else:
                raise AssertionError

            out.append(u"\u2022{nbsp}" + " *{0}*: {1}{2}{3}".format(name, typestring, linebreak, defaultstring))

    if len(cls.validity_rules) != 0:
        out.append((" +" + end).join(cls.validity_rules))

    if cls.long_description is not None:
        out.append(end + "*Details:*" + end)
        out.append(cls.long_description.strip())

    return end.join(out)

if __name__ == "__main__":
    with open("../specification.adoc", "w") as file:
        file.write(prologue)

        for cls in classes:
            file.write(formatted(cls))

        file.write(epilogue)
