// If you make changes to this file, you'll need to run protoc to updated the
// generated files by running the following command:
//
// python -m grpc_tools.protoc -I=semantic_model_generator/protos/ --python_out=semantic_model_generator/protos/ --pyi_out=semantic_model_generator/protos/ semantic_model_generator/protos/semantic_model.proto

syntax = "proto3";

import "google/protobuf/descriptor.proto";

package semantic_model_generator;

// We are using proto FieldOptions to validate the YAMLs match the expected
// schema. Any fields with the `optional` option, are not required during YAML
// validation.
extend google.protobuf.FieldOptions {
  optional bool optional = 51234;
  optional bool sql_expression = 51235;
  optional bool id_field = 51236;
}

// AggregationType defines a list of various aggregations.
enum AggregationType {
  aggregation_type_unknown = 0;
  sum = 1;
  avg = 2;
  median = 7;
  min = 3;
  max = 4;
  count = 5;
  count_distinct = 6;
}

// ColumnKind defines various kinds of columns, mainly categorized into
// dimensions and measures.
enum ColumnKind {
  column_kind_unknown = 0;
  // A column containing categorical values such as names, countries, dates.
  dimension = 1;
  // A column containing numerical values such as revenue, impressions, salary.
  // TODO: migrate to fact.
  measure = 2;
  // A column containing date/time data.
  time_dimension = 3;
  // A "column" containing calculations about an entity such as sum_revenue,
  // cvr.
  metric = 4;
}

message RetrievalResult {
  string value = 1;
  float score = 2;
}

// Column is analogous to a database column and defines various semantic
// properties of a column.  A column can either simply be a column in the base
// database schema or it can be an arbitrary expression over the base schema,
// e.g. `base_column1 + base_column2`.
message Column {
  // A descriptive name for this column.
  string name = 1 [ (id_field) = true ];
  // A list of other terms/phrases used to refer to this column.
  repeated string synonyms = 2 [ (optional) = true ];
  // A brief description about this column, including things like what data this
  // column has.
  string description = 3 [ (optional) = true ];
  // The SQL expression for this column. Could simply be a base table column
  // name or an arbitrary SQL expression over one or more columns of the base
  // table.
  string expr = 4 [ (sql_expression) = true ];
  // The data type of this column.
  // TODO(nsehrawat): Consider creating an enum instead, with all ClickZetta-
  // supported data types.
  string data_type = 5;
  // The kind of this column - dimension or fact, metric.
  ColumnKind kind = 6;
  // If true, assume that this column has unique values.
  bool unique = 7 [ (optional) = true ];
  // If no aggregation is specified, then this is the default aggregation
  // applied to this column in contxt of a grouping.
  AggregationType default_aggregation = 8 [ (optional) = true, deprecated = true ];
  // Sample values of this column.
  repeated string sample_values = 9 [ (optional) = true ];
  // Whether to index the values and retrieve them based on the question.
  // If False, all sample values will be used as input to the model.
  bool index_and_retrieve_values = 10 [ (optional) = true ];
  // Retrieved literals of this column.
  repeated RetrievalResult retrieved_literals = 11 [ (optional) = true ];

  // A Cortex Search Service configured on this column to retrieve literals.
  string cortex_search_service_name = 12
      [ (optional) = true, deprecated = true ];
  CortexSearchService cortex_search_service = 13 [ (optional) = true ];
  // If true, this column has limited possible values, all of which are in
  // the sample_values field.
  bool is_enum = 14 [ (optional) = true ];
}

// Dimension columns contain categorical values (e.g. state, user_type,
// platform). NOTE: If modifying this protobuf, make appropriate changes in
// context_to_column_format() of clickzetta/semantic_context/protos/schema.py.
message Dimension {
  // A descriptive name for this dimension.
  string name = 1 [ (id_field) = true ];
  // A list of other terms/phrases used to refer to this dimension.
  repeated string synonyms = 2 [ (optional) = true ];
  // A brief description about this dimension, including things like
  // what data this dimension has.
  string description = 3 [ (optional) = true ];
  // The SQL expression defining this dimension. Could simply be a physical
  // column name or an arbitrary SQL expression over one or more columns of the
  // physical table.
  string expr = 4 [ (sql_expression) = true ];
  // The data type of this dimension.
  // TODO(nsehrawat): Consider creating an enum instead with all ClickZetta-
  // supported data types.
  string data_type = 5;
  // If true, assume that this dimension has unique values.
  bool unique = 6 [ (optional) = true ];
  // Sample values of this column.
  repeated string sample_values = 7 [ (optional) = true ];
  // A Cortex Search Service configured on this column to retrieve literals.
  CortexSearchService cortex_search_service = 8 [ (optional) = true ];
  string cortex_search_service_name = 9
      [ (optional) = true, deprecated = true ];
  // If true, this column has limited possible values, all of which are in
  // the sample_values field.
  bool is_enum = 10 [ (optional) = true ];
}

// Fully qualified Cortex Search Service name.
message CortexSearchService {
  string database = 1 [ (optional) = true ];
  string schema = 2 [ (optional) = true ];
  string service = 3;
  string literal_column = 4 [ (optional) = true ];
}

// Time dimension columns contain time values (e.g. sale_date, created_at,
// year). NOTE: If modifying this protobuf, make appropriate changes in
// to_column_format() of clickzetta/semantic_context/utils/utils.py.
message TimeDimension {
  // A descriptive name for this time dimension.
  string name = 1 [ (id_field) = true ];
  // A list of other terms/phrases used to refer to this time dimension.
  repeated string synonyms = 2 [ (optional) = true ];
  // A brief description about this time dimension, including things like
  // what data it has, the timezone of values, etc.
  string description = 3 [ (optional) = true ];
  // The SQL expression defining this time dimension. Could simply be a physical
  // column name or an arbitrary SQL expression over one or more columns of the
  // physical table.
  string expr = 4 [ (sql_expression) = true ];
  // The data type of this time dimension.
  // TODO(nsehrawat): Consider creating an enum instead, with all ClickZetta-
  // supported data types.
  string data_type = 5;
  // If true, assume that this time dimension has unique values.
  bool unique = 6 [ (optional) = true ];
  // Sample values of this time dimension.
  repeated string sample_values = 7 [ (optional) = true ];
}

// Measure columns contain numerical values (e.g. revenue, impressions, salary).
// NOTE: If modifying this protobuf, make appropriate changes in
// to_column_format() of clickzetta/semantic_context/utils/utils.py.
message Fact {
  // A descriptive name for this measure.
  string name = 1 [ (id_field) = true ];
  // A list of other terms/phrases used to refer to this measure.
  repeated string synonyms = 2 [ (optional) = true ];
  // A brief description about this measure, including things like what data
  // it has.
  string description = 3 [ (optional) = true ];
  // The SQL expression defining this measure. Could simply be a physical column
  // name or an arbitrary SQL expression over one or more physical columns of
  // the underlying physical table.
  string expr = 4 [ (sql_expression) = true ];
  // The data type of this measure.
  // TODO(nsehrawat): Consider creating an enum instead, with all ClickZetta-
  // supported data types.
  string data_type = 5;
  // If no aggregation is specified, then this is the default aggregation
  // applied to this measure in contxt of a grouping.
  AggregationType default_aggregation = 6 [ (optional) = true , deprecated = true ];
  // Sample values of this measure.
  repeated string sample_values = 7 [ (optional) = true ];
}

// Filter represents a named SQL expression that's used for filtering.
// TODO: add validation. we should only support where clause style filter (no
// aggregations) and reject having clauses.
message NamedFilter {
  // A descriptive name for this filter.
  string name = 1;
  // A list of other term/phrases used to refer to this column.
  repeated string synonyms = 2 [ (optional) = true ];
  // A brief description about this column, including details of what this
  // filter is typically used for.
  string description = 3 [ (optional) = true ];
  // The SQL expression of this filter.
  string expr = 4 [ (sql_expression) = true ];
}

// FullyQualifiedTable is used to represent three part table names -
// (database, schema, table).
message FullyQualifiedTable {
  string database = 1;
  string schema = 2;
  string table = 3;
}

// Defines a primary key of a table. In the general case, primary keys
// are a collection of columns of the table.
// For discussion: PK FK are potentially duplicative to join path in a semantic
// model. However, it implies uniqueness which can be informative for getting
// right aggregation level. For that reason, we are exposing only the PrimaryKey
// currently. Join paths seem more extensible than foreign keys for supporting
// join. Further experimentation is needed to see if JoinPath and ForeignKey can
// yield similar results.
message PrimaryKey {
  // Base column names that constitute the primary key.
  repeated string columns = 1;
}

// Defines a foreign key that references the primary key of another table.
message ForeignKey {
  // Base column names of the foreign key table.
  repeated string fkey_columns = 1;
  // The primary key table that this foreign key references.
  FullyQualifiedTable pkey_table = 2;
  // Base column names of the primary key table.
  repeated string pkey_columns = 3;
}

// Table is analogous to a database table and provides a simple view over an
// existing database table.  A table can leave out some columns from the base
// table and/or introduce new derived columns.
message Table {
  // A descriptive name for this table.
  string name = 1 [ (id_field) = true ];
  // A list of other term/phrases used to refer to this table.
  repeated string synonyms = 2 [ (optional) = true ];
  // A brief description of this table, including details of what kinds of
  // analysis is it typically used for.
  string description = 3 [ (optional) = true ];
  // Fully qualified name of the underlying base table.
  FullyQualifiedTable base_table = 4;

  // We allow two formats for specifying logical columns of a table:
  // 1. As a list of columns.
  // 2. As three separate list of dimensions, time dimensions, and measures.
  // For the external facing yaml specification, we have chosen to go with (2).
  // However, for the time being we'll support both (1) and (2) and continue
  // using (1) as the internal representation.
  repeated Column columns = 5 [ (optional) = true ];
  repeated Dimension dimensions = 9 [ (optional) = true ];
  repeated TimeDimension time_dimensions = 10 [ (optional) = true ];
  repeated Fact measures = 11 [ (optional) = true, deprecated = true ];
  repeated Fact facts = 12 [ (optional) = true ];
  repeated Metric metrics = 13 [ (optional) = true ];

  // Primary key of the table, if any.
  PrimaryKey primary_key = 6 [ (optional) = true ];
  // Foreign keys of the table, if any.
  repeated ForeignKey foreign_keys = 7 [ (optional) = true ];
  // Predefined filters on this table, if any.
  repeated NamedFilter filters = 8 [ (optional) = true ];
  // NEXT_TAG: 14.
}

// Metric are named computation over a collection of columns.  For now, we
// only allow a metric to be defined over columns from a single table. In
// future, we'll expand to allowing metrics that refer to columns from multiple
// tables.
message Metric {
  // A descriptive name of the metric.
  string name = 1 [ (id_field) = true ];
  // A list of other term/phrases used to refer to this metric.
  repeated string synonyms = 2 [ (optional) = true ];
  // A brief description of this metric, including details of what it computes.
  string description = 3 [ (optional) = true ];
  // The SQL expression to compute this metric.
  // All columns used must be fully qualified with the logical table name.
  // Expression must be an aggregate
  string expr = 4 [ (sql_expression) = true ];
  // The filter associated with this metric.
  // Do not expose this for now.
  MetricsFilter filter = 5 [ (optional) = true ];
}

message MetricsFilter { string expr = 1 [ (sql_expression) = true ]; }

// Type of the join - inner, left outer, etc.
enum JoinType {
  join_type_unknown = 0;
  inner = 1;
  left_outer = 2;
  full_outer = 3 [ deprecated = true ];
  cross = 4 [ deprecated = true ];
  right_outer = 5 [ deprecated = true ];
}

// Type of the relationship - one-to-one, many-to-one, etc.
enum RelationshipType {
  relationship_type_unknown = 0;
  one_to_one = 1;
  many_to_one = 2;
  one_to_many = 3 [ deprecated = true ];
  many_to_many = 4 [ deprecated = true ];
}

message RelationKey {
  // Only support equi-join relationship for now.
  string left_column = 1;
  string right_column = 2;
}

// Relationship represents a join between two tables.
message Relationship {
  // A unique name of the join.
  string name = 1;
  // The left hand side table of the join.
  string left_table = 2;
  // The right hand side table of the join.
  string right_table = 3;
  // The expression used to join left and right tables. Only used internally.
  string expr = 4 [ (sql_expression) = true, (optional) = true ];
  // Keys directly represent the join relationship.
  repeated RelationKey relationship_columns = 7 [ (optional) = true ];
  // Type of the join.
  JoinType join_type = 5;
  // Type of the relationship.
  RelationshipType relationship_type = 6;
}

// A message that encapsulates custom instructions for each module.
message ModuleCustomInstructions {
  // Custom instructions for SQL Generation.
  string sql_generation = 1 [ (optional) = true ];
  // Custom instructions for Question Categorization.
  string question_categorization = 2 [ (optional) = true ];
}

// The semantic context relevant to generating SQL for answering a data
// question.
message SemanticModel {
  // A descriptive name of the project.
  string name = 1;
  // A brief description of this project, including details of what kind of
  // analysis does this project enable.
  string description = 2 [ (optional) = true ];
  // List of tables in this project.
  repeated Table tables = 3;
  // Derived metrics scoped to the entire semantic model.
  repeated Metric metrics = 4 [ (optional) = true ];
  // List of relationships in this project.
  repeated Relationship relationships = 5 [ (optional) = true ];
  // List of verified queries for this semantic model.
  repeated VerifiedQuery verified_queries = 6 [ (optional) = true ];
  // Custom instructions that will be applied to the final SQL generation.
  string custom_instructions = 7 [ (optional) = true ];
  // Module-specific custom instructions. The SQL generation instruction here
  // will take precedence over the legacy custom_instructions if it exists.
  ModuleCustomInstructions module_custom_instructions = 8 [ (optional) = true ];
}

// VerifiedQuery represents a (question, sql) pair that has been manually
// verified (e.g. by an analyst) to be correct.
message VerifiedQuery {
  // A name for this verified query. Mainly used for display purposes.
  string name = 1;
  // The name of the semantic model on which this verified query is based off.
  string semantic_model_name = 2 [ (optional) = true ];
  // The question being answered.
  string question = 3;
  // The correct SQL query for answering the question.
  string sql = 4 [ (sql_expression) = true ];
  // Timestamp at which the query was last verified - measures in seconds since
  // epoch, in UTC.
  int64 verified_at = 5 [ (optional) = true ];
  // Name of the person who verified this query.
  string verified_by = 6 [ (optional) = true ];
  // Whether to always include in this question in the suggested questions
  // module
  bool use_as_onboarding_question = 7 [ (optional) = true ];
}

// VerifiedQueryRepository is a simply a collection of verified queries.
message VerifiedQueryRepository { repeated VerifiedQuery verified_queries = 1; }
