// Copyright 2020 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

namespace tflite;

// TFLite metadata contains both human readable and machine readable information
// about what the model does and how to use the model. It can be used as a
// README file, which elaborates the details of the model, each input/ouput
// tensor, and each associated file.
//
// An important use case of TFLite metadata is the TFLite codegen tool, which
// automatically generates the model interface based on the properties of the
// model and the tensors. The model interface provides high-level APIs to
// interact with the model, such as preprocessing the input data and running
// inferences.
//
// Entries marked with "<Codegen usage>" are used in TFLite codegen tool to
// generate the model interface. It is recommended to fill in at least those
// enties to boost the codegen performance.

// This corresponds to the schema version.
file_identifier "M001";
// File extension of any written files.
file_extension "tflitemeta";

enum AssociatedFileType : byte {
  UNKNOWN = 0,
  // Files such as readme.txt
  DESCRIPTIONS = 1,

  // Contains labels that annotate certain axis of the tensor. For example,
  // the label file in image classification. Those labels annotate the
  // the output tensor, such that each value in the output tensor is the
  // probability of that corresponding category specified by the label.
  //
  // <Codegen usage>:
  // If an output tensor has an associated file as TENSOR_AXIS_LABELS, return
  // the output as a mapping between the labels and probability in the model
  // interface.
  // If multiple files of the same type are present, the first one is used by
  // default; additional ones are to be distinguished from one another by their
  // specified locale.
  TENSOR_AXIS_LABELS = 2,

  // Contains labels that tensor values correspond to. For example, in
  // the object detection model, one of the output tensors is the detected
  // classes. And each value in the tensor refers to the index of label in the
  // category label file.
  //
  // <Codegen usage>:
  // If an output tensor has an associated file as TENSOR_VALUE_LABELS, convert
  // the tensor values into labels, and return a list of string as the output.
  // If multiple files of the same type are present, the first one is used by
  // default; additional ones are to be distinguished from one another by their
  // specified locale.
  TENSOR_VALUE_LABELS = 3,

  // Contains sigmoid-based score calibration parameters, formatted as CSV.
  // Lines contain for each index of an output tensor the scale, slope, offset
  // and min_score parameters to be used for sigmoid fitting (in this order and
  // in `strtof`-compatible [1] format).
  // A line may be left empty to default calibrated scores for this index to
  // default_score. See documentation for ScoreCalibrationOptions for details.
  //
  // [1]: https://en.cppreference.com/w/c/string/byte/strtof
  TENSOR_AXIS_SCORE_CALIBRATION = 4,
}

table AssociatedFile {
  // Name of this file. Need to be exact the same as the name of the actual file
  // packed into the TFLite model as a zip file.
  //
  // <Codegen usage>:
  // Locates to the actual file in the TFLite model.
  name:string;

  // A description of what the file is.
  description:string;

  // Type of the associated file. There may be special pre/post processing for
  // some types. For example in image classification, a label file of the output
  // will be used to convert object index into string.
  //
  // <Codegen usage>:
  // Determines how to process the corresponding tensor.
  type:AssociatedFileType;

  // An optional locale for this associated file (if applicable). It is
  // recommended to use an ISO 639-1 letter code (e.g. "en" for English),
  // optionally completed by a two letter region code (e.g. "en-US" for US
  // English and "en-CA" for Canadian English).
  // Leverage this in order to specify e.g multiple label files translated in
  // different languages.
  locale:string;
}

// The basic content type for all tensors.
//
// <Codegen usage>:
// Input feature tensors:
// 1. Generates the method to load data from a TensorBuffer.
// 2. Creates the preprocessing logic. The default processing pipeline is:
// [NormalizeOp, QuantizeOp].
// Output feature tensors:
// 1. Generates the method to return the output data to a TensorBuffer.
// 2. Creates the post-processing logic. The default processing pipeline is:
// [DeQuantizeOp].
table FeatureProperties {
}

// The type of color space of an image.
enum ColorSpaceType : byte {
  UNKNOWN = 0,
  RGB = 1,
  GRAYSCALE = 2,
}

table ImageSize {
  width:uint;
  height:uint;
}

// The properties for image tensors.
//
// <Codegen usage>:
// Input image tensors:
// 1. Generates the method to load an image from a TensorImage.
// 2. Creates the preprocessing logic. The default processing pipeline is:
// [ResizeOp, NormalizeOp, QuantizeOp].
// Output image tensors:
// 1. Generates the method to return the output data to a TensorImage.
// 2. Creates the post-processing logic. The default processing pipeline is:
// [DeQuantizeOp].
table ImageProperties {
  // The color space of the image.
  //
  // <Codegen usage>:
  // Determines how to convert the color space of a given image from users.
  color_space:ColorSpaceType;

  // Indicates the default value of image width and height if the tensor shape
  // is dynamic. For fixed-size tensor, this size will be consistent with the
  // expected size.
  default_size:ImageSize;
}

// The properties for tensors representing bounding boxes.
//
// <Codegen usage>:
// Input image tensors: NA.
// Output image tensors: parses the values into a data stucture that represents
// bounding boxes. For example, in the generated wrapper for Android, it returns
// the output as android.graphics.Rect objects.
enum BoundingBoxType : byte {
  UNKNOWN = 0,
  // Represents the bounding box by using the combination of boundaries,
  // {left, top, right, bottom}.
  // The default order is {left, top, right, bottom}. Other orders can be
  // indicated by BoundingBoxProperties.index.
  BOUNDARIES = 1,

  // Represents the bounding box by using the upper_left corner, width and
  // height.
  // The default order is {upper_left_x, upper_left_y, width, height}. Other
  // orders can be indicated by BoundingBoxProperties.index.
  UPPER_LEFT = 2,

  // Represents the bounding box by using the center of the box, width and
  // height. The default order is {center_x, center_y, width, height}. Other
  // orders can be indicated by BoundingBoxProperties.index.
  CENTER = 3,

}

enum CoordinateType : byte {
  // The coordinates are float values from 0 to 1.
  RATIO = 0,
  // The coordinates are integers.
  PIXEL = 1,
}

table BoundingBoxProperties {
  // Denotes the order of the elements defined in each bounding box type. An
  // empty index array represent the defualt order of each bounding box type.
  // For example, to denote the default order of BOUNDARIES, {left, top, right,
  // bottom}, the index should be {0, 1, 2, 3}. To denote the order {left,
  // right, top, bottom}, the order should be {0, 2, 1, 3}.
  //
  // The index array can be applied to all bounding box types to adjust the
  // order of their corresponding underlying elements.
  //
  // <Codegen usage>:
  // Indicates how to parse the bounding box values.
  index:[uint];

  // <Codegen usage>:
  // Indicates how to parse the bounding box values.
  type:BoundingBoxType;

  // <Codegen usage>:
  // Indicates how to convert the bounding box back to the original image in
  // pixels.
  coordinate_type:CoordinateType;
}

union ContentProperties {
  FeatureProperties,
  ImageProperties,
  BoundingBoxProperties,
}

table ValueRange {
  min:int;
  max:int;
}

table Content {
  // The properties that the content may have, indicating the type of the
  // Content.
  //
  // <Codegen usage>:
  // Indicates how to process the tensor.
  content_properties:ContentProperties;

  // The range of dimensions that the content corresponds to. A NULL
  // "range" indicates that the content uses up all dimensions,
  // except the batch axis if applied.
  //
  // Here are all the possible situations of how a tensor is composed.
  // Case 1: The tensor is a single object, such as an image.
  // For example, the input of an image classifier
  // (https://www.tensorflow.org/lite/models/image_classification/overview),
  // a tensor of shape [1, 224, 224, 3]. Dimensions 1 to 3 correspond to the
  // image. Since dimension 0 is a batch axis, which can be ignored,
  // "range" can be left as NULL.
  //
  // Case 2: The tensor contains multiple instances of the same object.
  // For example, the output tensor of detected bounding boxes of an object
  // detection model
  // (https://www.tensorflow.org/lite/models/object_detection/overview).
  // The tensor shape is [1, 10, 4]. Here is the what the three dimensions
  // represent for:
  // dimension 0: the batch axis.
  // dimension 1: the 10 objects detected with the highest confidence.
  // dimension 2: the bounding boxes of the 10 detected objects.
  // The tensor is essentially 10 bounding boxes. In this case,
  // "range" should be {min=2; max=2;}.
  // Another example is the pose estimation model
  // (https://www.tensorflow.org/lite/models/pose_estimation/overview).
  // The output tensor of heatmaps is in the shape of [1, 9, 9, 17].
  // Here is the what the four dimensions represent for:
  // dimension 0: the batch axis.
  // dimension 1/2: the heatmap image.
  // dimension 3: 17 body parts of a person.
  // Even though the last axis is body part, the real content of this tensor is
  // the heatmap. "range" should be [min=2; max=3].
  //
  // Case 3: The tensor contains multiple different objects. (Not supported by
  // Content at this point).
  // Sometimes a tensor may contain multiple different objects, thus different
  // contents. It is very common for regression models. For example, a model
  // to predict the fuel efficiency
  // (https://www.tensorflow.org/tutorials/keras/regression).
  // The input tensor has shape [1, 9], consisting of 9 features, such as
  // "Cylinders", "Displacement", "Weight", etc. In this case, dimension 1
  // contains 9 different contents. However, since these sub-dimension objects
  // barely need to be specifically processed, their contents are not recorded
  // in the metadata. Through, the name of each dimension can be set through
  // TensorMetadata.dimension_names.
  //
  // Note that if it is not case 3, a tensor can only have one content type.
  //
  // <Codegen usage>:
  // Case 1: return a processed single object of certain content type.
  // Case 2: return a list of processed objects of certain content type. The
  // generated model interface have API to random access those objects from
  // the output.
  range:ValueRange;
}

// Parameters that are used when normalizing the tensor.
table NormalizationOptions{
  // mean and std are normalization parameters. Tensor values are normailzed
  // per-channelly by,
  //   (x - mean) / std.
  // For example, a float MobileNet model will have
  //   mean = 127.5f and std = 127.5f.
  // A quantized MobileNet model will have
  //   mean = 0.0f and std = 1.0f.
  // If there is only one value in mean or std, we'll propogate the value to
  // all channels.

  // Per-channel mean of the possible values used in normalization.
  //
  // <Codegen usage>:
  // Apply normalization to input tensors accordingly.
  mean:[float];

  // Per-channel standard dev. of the possible values used in normalization.
  //
  // <Codegen usage>:
  // Apply normalization to input tensors accordingly.
  std:[float];
}

// The different possible score transforms to apply to uncalibrated scores
// before applying score calibration.
enum ScoreTransformationType : byte {
  // Identity function: g(x) = x.
  IDENTITY = 0,
  // Log function: g(x) = log(x).
  LOG = 1,
  // Inverse logistic function: g(x) = log(x) - log(1-x).
  INVERSE_LOGISTIC = 2,
}

// Options to perform score calibration on an output tensor through sigmoid
// functions. One of the main purposes of score calibration is to make scores
// across classes comparable, so that a common threshold can be used for all
// output classes. This is meant for models producing class predictions as
// output, e.g. image classification or detection models.
//
// For each index in the output tensor, this applies:
// * `f(x) = scale / (1 + e^-(slope*g(x)+offset))` if `x > min_score`,
// * `f(x) = default_score` otherwise or if no scale, slope, offset and
//    min_score have been specified.
// Where:
// * scale, slope, offset and min_score are index-specific parameters
// * g(x) is an index-independent transform among those defined in
//   ScoreTransformationType
// * default_score is an index-independent parameter.
// An AssociatedFile with type TANSOR_AXIS_SCORE_CALIBRATION specifying the
// index-specific parameters must be associated with the corresponding
// TensorMetadata for score calibration be applied.
table ScoreCalibrationOptions {
  // The function to use for transforming the uncalibrated score before
  // applying score calibration.
  score_transformation:ScoreTransformationType;

  // The default calibrated score to apply if the uncalibrated score is
  // below min_score or if no parameters were specified for a given index.
  default_score:float;
}

// Performs thresholding on output tensor values, in order to filter out
// low-confidence results.
table ScoreThresholdingOptions {
  // The recommended global threshold below which results are considered
  // low-confidence and should be filtered out.
  global_score_threshold:float;
}

// Options that are used when processing the tensor.
union ProcessUnitOptions {
  NormalizationOptions,
  ScoreCalibrationOptions,
  ScoreThresholdingOptions,
}

// A process unit that is used to process the tensor out-of-graph.
table ProcessUnit {
  options:ProcessUnitOptions;
}


// Statistics to describe a tensor.
table Stats {
  // Max and min are not currently used in tflite.support codegen. They mainly
  // serve as references for users to better understand the model. They can also
  // be used to validate model pre/post processing results.
  // If there is only one value in max or min, we'll propogate the value to
  // all channels.

  // Per-channel maximum value of the tensor.
  max:[float];

  // Per-channel minimum value of the tensor.
  min:[float];
}

// Detailed information of an input or output tensor.
table TensorMetadata {
  // Name of the tensor.
  //
  // <Codegen usage>:
  // The name of this tensor in the generated model interface.
  name:string;

  // A description of the tensor.
  description:string;

  // A list of names of the dimensions in this tentor. The length of
  // dimension_names need to match the number of dimensions in this tensor.
  //
  // <Codegen usage>:
  // The name of each dimension in the generated model interface. See "Case 2"
  // in the comments of Content.range.
  dimension_names:[string];

  // The content that represents this tensor.
  //
  // <Codegen usage>:
  // Determines how to process this tensor. See each item in ContentProperties
  // for the default process units that will be applied to the tensor.
  content:Content;

  // The process units that are used to process the tensor out-of-graph.
  //
  // <Codegen usage>:
  // Contains the parameters of the default processing pipeline for each content
  // type, such as the normalization parameters in all content types. See the
  // items under ContentProperties for the details of the default processing
  // pipeline.
  process_units:[ProcessUnit];

  // The statistics of the tensor values.
  stats:Stats;

  // A list of associated files of this tensor.
  //
  // <Codegen usage>:
  // Contains processing parameters of this tensor, such as normalization.
  associated_files:[AssociatedFile];
}

table SubGraphMetadata {
  // Name of the subgraph.
  //
  // Note that, since TFLite only support one subgraph at this moment, the
  // Codegen tool will use the name in ModelMetadata in the generated model
  // interface.
  name:string;

  // A description explains details about what the subgraph does.
  description:string;

  // Metadata of all input tensors used in this subgraph.
  //
  // <Codegen usage>:
  // Determines how to process the inputs.
  input_tensor_metadata:[TensorMetadata];

  // Metadata of all output tensors used in this subgraph.
  //
  // <Codegen usage>:
  // Determines how to process the outputs.
  output_tensor_metadata:[TensorMetadata];

  // A list of associated files of this subgraph.
  associated_files:[AssociatedFile];
}

table ModelMetadata {
  // Name of the model.
  //
  // <Codegen usage>:
  // The name of the model in the generated model interface.
  name:string;

  // Model description in schema.
  description:string;

  // Version of the model that specified by model creators.
  version:string;

  // Noted that, the minimum required TFLite runtime version that the model is
  // compatible with, has already been added as a metadata entry in tflite
  // schema. We'll decide later if we want to move it here, and keep it with
  // other metadata entries.

  // Metadata of all the subgraphs of the model. The 0th is assumed to be the
  // main subgraph.
  //
  // <Codegen usage>:
  // Determines how to process the inputs and outputs.
  subgraph_metadata:[SubGraphMetadata];

  // The person who creates this model.
  author:string;

  // Licenses that may apply to this model.
  license:string;

  // A list of associated files of this model.
  associated_files:[AssociatedFile];
}

root_type ModelMetadata;
