5.5.2.8 Versioning

// Proposed contract, version 1. Status: proposal for review, not confirmed design.
//
// The farm data clock and dataset exports.
// Callers: ai-model-platform (GetWatermark before a build; StartExport and GetExport to build a dataset version at a
// write number into the datasets bucket; req 61, 71, 72, 73). Data for models travels as files, never as database access.
// Deadline: GetWatermark 500 ms, StartExport 5 s, GetExport 2 s; the export itself is a job.
// The callers listed are authentication only (decided 2026-11-11): each RPC below names who may call it and what
// it sees, and refuses anyone else with PERMISSION_DENIED.
// x-moonitoring-callers: ai-model-platform
// x-moonitoring-requirements: 61, 71, 72, 73
syntax = "proto3";

package moonitoring.farm.versioning.v1;

import "google/protobuf/timestamp.proto";

service WatermarkService {
  // The highest write number such that no write at or below it is pending.
  // Callers: ai-model-platform only. Sees: one number, no farm data.
  rpc GetWatermark(GetWatermarkRequest) returns (GetWatermarkResponse);
}

service ExportsService {
  // Starts an export of the selection at a write number: Parquet files by content hash and a manifest, written to
  // the datasets bucket. Export reuse includes (selection, effective data_seq, knowledge_clock,
  // feature_code_version, effective rule_set, anonymization). Different knowledge clocks must never
  // share an export merely because the other inputs match. The manifest records the chosen clock.
  // Callers: ai-model-platform only. Sees: the export it started; the data goes to the datasets bucket as files.
  rpc StartExport(StartExportRequest) returns (StartExportResponse);
  // Where an export stands.
  // Callers: ai-model-platform only. Sees: only the exports it started.
  rpc GetExport(GetExportRequest) returns (GetExportResponse);
}

message GetWatermarkRequest {}

message GetWatermarkResponse {
  int64 watermark_seq = 1;
  google.protobuf.Timestamp committed_at = 2;
}

message StartExportRequest {
  // The caller's request id, for idempotency.
  string request_id = 1;
  // At or below the watermark. Omitted: resolve the current watermark once when accepting the export.
  // Explicit 0: the initial empty state, matching REST ReadSequence; it must not select current data.
  optional int64 data_seq = 2;
  // The reconciliation rule set in force for the resolved facts; empty: the one in force at data_seq.
  string rule_set_id = 3;
  // Part of the export identity; UNSPECIFIED resolves to PLATFORM, as in the REST build request.
  KnowledgeClock knowledge_clock = 4;
  // MlDatasetSelection as JSON, as defined in the shared types.
  string selection_json = 5;
  string feature_code_version = 6;
  Anonymization anonymization = 7;
  // gs:// prefix the manifest and files go under.
  string destination_prefix = 8;
  // When set, the export is built from the files of this manifest (a variant), never from the database.
  string parent_manifest_path = 9;
}

enum KnowledgeClock {
  KNOWLEDGE_CLOCK_UNSPECIFIED = 0;
  KNOWLEDGE_CLOCK_PLATFORM = 1;
  KNOWLEDGE_CLOCK_SOURCE = 2;
}

enum Anonymization {
  ANONYMIZATION_UNSPECIFIED = 0;
  ANONYMIZATION_IDENTIFIED = 1;
  ANONYMIZATION_PSEUDONYMIZED = 2;
  ANONYMIZATION_ANONYMIZED = 3;
}

message StartExportResponse {
  string export_id = 1;
  // Set when an identical export already exists: the caller reuses it.
  string existing_manifest_path = 2;
}

message GetExportRequest {
  string export_id = 1;
}

message GetExportResponse {
  string export_id = 1;
  ExportStatus status = 2;
  int64 data_seq = 3;
  string manifest_path = 4;
  // SHA-256 of the manifest: the identity of the version.
  string manifest_hash = 5;
  int64 row_count = 6;
  int64 size_bytes = 7;
  repeated ExportedFile files = 8;
  string error = 9;
  google.protobuf.Timestamp finished_at = 10;
}

enum ExportStatus {
  EXPORT_STATUS_UNSPECIFIED = 0;
  EXPORT_STATUS_RUNNING = 1;
  EXPORT_STATUS_READY = 2;
  EXPORT_STATUS_FAILED = 3;
}

message ExportedFile {
  // SHA-256 of the file content; also its object name.
  string hash = 1;
  string path = 2;
  string table = 3;
  // The partition keys as JSON, e.g. {"month": "2025-03", "animal_type": "bovine"}.
  string partition_json = 4;
  int64 row_count = 5;
  int64 size_bytes = 6;
}