syntax = "proto3";
import "google/protobuf/timestamp.proto";
import "google/protobuf/empty.proto";

package source.v1;

service Source {
  // Read returns a stream of datum responses.
  // The size of the returned ReadResponse is less than or equal to the num_records specified in ReadRequest.
  // If the request timeout is reached on server side, the returned ReadResponse will contain all the datum that have been read (which could be an empty list).
  rpc ReadFn(ReadRequest) returns (stream ReadResponse);

  // AckFn acknowledges a list of datum offsets.
  // When AckFn is called, it implicitly indicates that the datum stream has been processed by the source vertex.
  // The caller (numa) expects the AckFn to be successful, and it does not expect any errors.
  // If there are some irrecoverable errors when the callee (UDSource) is processing the AckFn request,
  // then it is best to crash because there are no other retry mechanisms possible. 
  rpc AckFn(AckRequest) returns (AckResponse);

  // PendingFn returns the number of pending records at the user defined source.
  rpc PendingFn(google.protobuf.Empty) returns (PendingResponse);

  // IsReady is the heartbeat endpoint for user defined source gRPC.
  rpc IsReady(google.protobuf.Empty) returns (ReadyResponse);
}

/*
 * ReadRequest is the request for reading datum stream from user defined source.
 */
message ReadRequest {
  message Request {
    // Required field indicating the number of records to read.
    uint64 num_records = 1;
    // Required field indicating the request timeout in milliseconds.
    // uint32 can represent 2^32 milliseconds, which is about 49 days.
    // We don't use uint64 because time.Duration takes int64 as nano seconds. Using uint64 for milli will cause overflow.
    uint32 timeout_in_ms = 2;
  }
  // Required field indicating the request.
  Request request = 1;
}

/*
 * ReadResponse is the response for reading datum stream from user defined source.
 */
message ReadResponse {
  message Result {
    // Required field holding the payload of the datum.
    bytes payload = 1;
    // Required field indicating the offset information of the datum.
    Offset offset = 2;
    // Required field representing the time associated with each datum. It is used for watermarking.
    google.protobuf.Timestamp event_time = 3;
    // Optional list of keys associated with the datum.
    // Key is the "key" attribute in (key,value) as in the map-reduce paradigm.
    // We add this optional field to support the use case where the user defined source can provide keys for the datum.
    // e.g. Kafka and Redis Stream message usually include information about the keys.
    repeated string keys = 4;
  }
  // Required field holding the result.
  Result result = 1;
}

/*
 * AckRequest is the request for acknowledging datum.
 * It takes a list of offsets to be acknowledged.
 */
message AckRequest {
  message Request {
    // Required field holding a list of offsets to be acknowledged.
    // The offsets must be strictly corresponding to the previously read batch,
    // meaning the offsets must be in the same order as the datum responses in the ReadResponse.
    // By enforcing ordering, we can save deserialization effort on the server side, assuming the server keeps a local copy of the raw/un-serialized offsets.
    repeated Offset offsets = 1;
  }
  // Required field holding the request. The list will be ordered and will have the same order as the original Read response.
  Request request = 1;
}

/*
 * AckResponse is the response for acknowledging datum. It contains one empty field confirming
 * the batch of offsets that have been successfully acknowledged. The contract between client and server
 * is that the server will only return the AckResponse if the ack request is successful.
 * If the server hangs during the ack request, the client can decide to timeout and error out the data forwarder.
 * The reason why we define such contract is that we always expect the server to be able to process the ack request.
 * Client is expected to send the AckRequest to the server with offsets that are strictly
 * corresponding to the previously read batch. If the client sends the AckRequest with offsets that are not,
 * it is considered as a client error and the server will not return the AckResponse.
 */
message AckResponse {
  message Result {
    // Required field indicating the ack request is successful.
    google.protobuf.Empty success = 1;
  }
  // Required field holding the result.
  Result result = 1;
}

/*
 * ReadyResponse is the health check result for user defined source.
 */
message ReadyResponse {
  // Required field holding the health check result.
  bool ready = 1;
}

/*
 * PendingResponse is the response for the pending request.
 */
message PendingResponse {
  message Result {
    // Required field holding the number of pending records at the user defined source.
    // A negative count indicates that the pending information is not available.
    int64 count = 1;
  }
  // Required field holding the result.
  Result result = 1;
}

/*
 * Offset is the offset of the datum.
 */
message Offset {
  // offset is the offset of the datum. This field is required.
  // We define Offset as a byte array because different input data sources can have different representations for Offset.
  // The only way to generalize it is to define it as a byte array,
  // Such that we can let the UDSource to de-serialize the offset using its own interpretation logics.
  bytes offset = 1;
  // Optional partition_id indicates which partition of the source the datum belongs to.
  // It is useful for sources that have multiple partitions. e.g. Kafka.
  // If the partition_id is not specified, it is assumed that the source has a single partition.
  string partition_id = 2;
}