Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add fields and the BackupType proto for Hot Backups #1439

Merged
merged 2 commits into from
Sep 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.Bigtable.Admin.V2";
option go_package = "google.golang.org/genproto/googleapis/bigtable/admin/v2;admin";
option go_package = "cloud.google.com/go/bigtable/admin/apiv2/adminpb;adminpb";
option java_multiple_files = true;
option java_outer_classname = "BigtableInstanceAdminProto";
option java_package = "com.google.bigtable.admin.v2";
Expand Down
6 changes: 3 additions & 3 deletions protos/google/bigtable/admin/v2/bigtable_table_admin.proto
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.Bigtable.Admin.V2";
option go_package = "google.golang.org/genproto/googleapis/bigtable/admin/v2;admin";
option go_package = "cloud.google.com/go/bigtable/admin/apiv2/adminpb;adminpb";
option java_multiple_files = true;
option java_outer_classname = "BigtableTableAdminProto";
option java_package = "com.google.bigtable.admin.v2";
Expand Down Expand Up @@ -360,7 +360,7 @@ service BigtableTableAdmin {
// returned table [long-running operation][google.longrunning.Operation] can
// be used to track the progress of the operation, and to cancel it. The
// [metadata][google.longrunning.Operation.metadata] field type is
// [RestoreTableMetadata][google.bigtable.admin.RestoreTableMetadata]. The
// [RestoreTableMetadata][google.bigtable.admin.v2.RestoreTableMetadata]. The
// [response][google.longrunning.Operation.response] type is
// [Table][google.bigtable.admin.v2.Table], if successful.
rpc RestoreTable(RestoreTableRequest) returns (google.longrunning.Operation) {
Expand Down Expand Up @@ -1219,7 +1219,7 @@ message ListBackupsResponse {
// [CopyBackup][google.bigtable.admin.v2.BigtableTableAdmin.CopyBackup].
message CopyBackupRequest {
// Required. The name of the destination cluster that will contain the backup
// copy. The cluster must already exists. Values are of the form:
// copy. The cluster must already exist. Values are of the form:
// `projects/{project}/instances/{instance}/clusters/{cluster}`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
Expand Down
2 changes: 1 addition & 1 deletion protos/google/bigtable/admin/v2/common.proto
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package google.bigtable.admin.v2;
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.Bigtable.Admin.V2";
option go_package = "google.golang.org/genproto/googleapis/bigtable/admin/v2;admin";
option go_package = "cloud.google.com/go/bigtable/admin/apiv2/adminpb;adminpb";
option java_multiple_files = true;
option java_outer_classname = "CommonProto";
option java_package = "com.google.bigtable.admin.v2";
Expand Down
2 changes: 1 addition & 1 deletion protos/google/bigtable/admin/v2/instance.proto
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import "google/bigtable/admin/v2/common.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.Bigtable.Admin.V2";
option go_package = "google.golang.org/genproto/googleapis/bigtable/admin/v2;admin";
option go_package = "cloud.google.com/go/bigtable/admin/apiv2/adminpb;adminpb";
option java_multiple_files = true;
option java_outer_classname = "InstanceProto";
option java_package = "com.google.bigtable.admin.v2";
Expand Down
50 changes: 42 additions & 8 deletions protos/google/bigtable/admin/v2/table.proto
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";

option csharp_namespace = "Google.Cloud.Bigtable.Admin.V2";
option go_package = "google.golang.org/genproto/googleapis/bigtable/admin/v2;admin";
option go_package = "cloud.google.com/go/bigtable/admin/apiv2/adminpb;adminpb";
option java_multiple_files = true;
option java_outer_classname = "TableProto";
option java_package = "com.google.bigtable.admin.v2";
Expand Down Expand Up @@ -456,6 +456,23 @@ message Backup {
READY = 2;
}

// The type of the backup.
enum BackupType {
// Not specified.
BACKUP_TYPE_UNSPECIFIED = 0;

// The default type for Cloud Bigtable managed backups. Supported for
// backups created in both HDD and SSD instances. Requires optimization when
// restored to a table in an SSD instance.
STANDARD = 1;

// A backup type with faster restore to SSD performance. Only supported for
// backups created in SSD instances. A new SSD table restored from a hot
// backup reaches production performance more quickly than a standard
// backup.
HOT = 2;
}

// A globally unique identifier for the backup which cannot be
// changed. Values are of the form
// `projects/{project}/instances/{instance}/clusters/{cluster}/
Expand All @@ -478,14 +495,17 @@ message Backup {

// Output only. Name of the backup from which this backup was copied. If a
// backup is not created by copying a backup, this field will be empty. Values
// are of the form: projects/<project>/instances/<instance>/backups/<backup>.
// are of the form:
// projects/<project>/instances/<instance>/clusters/<cluster>/backups/<backup>
string source_backup = 10 [(google.api.field_behavior) = OUTPUT_ONLY];

// Required. The expiration time of the backup, with microseconds
// granularity that must be at least 6 hours and at most 90 days
// from the time the request is received. Once the `expire_time`
// has passed, Cloud Bigtable will delete the backup and free the
// resources used by the backup.
// Required. The expiration time of the backup.
// When creating a backup or updating its `expire_time`, the value must be
// greater than the backup creation time by:
// - At least 6 hours
// - At most 90 days
//
// Once the `expire_time` has passed, Cloud Bigtable will delete the backup.
google.protobuf.Timestamp expire_time = 3
[(google.api.field_behavior) = REQUIRED];

Expand All @@ -511,6 +531,19 @@ message Backup {
// Output only. The encryption information for the backup.
EncryptionInfo encryption_info = 9
[(google.api.field_behavior) = OUTPUT_ONLY];

// Indicates the backup type of the backup.
BackupType backup_type = 11;

// The time at which the hot backup will be converted to a standard backup.
// Once the `hot_to_standard_time` has passed, Cloud Bigtable will convert the
// hot backup to a standard backup. This value must be greater than the backup
// creation time by:
// - At least 24 hours
//
// This field only applies for hot backups. When creating or updating a
// standard backup, attempting to set this field will fail the request.
google.protobuf.Timestamp hot_to_standard_time = 12;
}

// Information about a backup.
Expand All @@ -533,7 +566,8 @@ message BackupInfo {

// Output only. Name of the backup from which this backup was copied. If a
// backup is not created by copying a backup, this field will be empty. Values
// are of the form: projects/<project>/instances/<instance>/backups/<backup>.
// are of the form:
// projects/<project>/instances/<instance>/clusters/<cluster>/backups/<backup>
string source_backup = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}

Expand Down
159 changes: 138 additions & 21 deletions protos/google/bigtable/admin/v2/types.proto
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package google.bigtable.admin.v2;
import "google/api/field_behavior.proto";

option csharp_namespace = "Google.Cloud.Bigtable.Admin.V2";
option go_package = "google.golang.org/genproto/googleapis/bigtable/admin/v2;admin";
option go_package = "cloud.google.com/go/bigtable/admin/apiv2/adminpb;adminpb";
option java_multiple_files = true;
option java_outer_classname = "TypesProto";
option java_package = "com.google.bigtable.admin.v2";
Expand All @@ -32,27 +32,22 @@ option ruby_package = "Google::Cloud::Bigtable::Admin::V2";
//
// For compatibility with Bigtable's existing untyped APIs, each `Type` includes
// an `Encoding` which describes how to convert to/from the underlying data.
// This might involve composing a series of steps into an "encoding chain," for
// example to convert from INT64 -> STRING -> raw bytes. In most cases, a "link"
// in the encoding chain will be based an on existing GoogleSQL conversion
// function like `CAST`.
//
// Each link in the encoding chain also defines the following properties:
// * Natural sort: Does the encoded value sort consistently with the original
// typed value? Note that Bigtable will always sort data based on the raw
// encoded value, *not* the decoded type.
// Each encoding also defines the following properties:
//
// * Order-preserving: Does the encoded value sort consistently with the
// original typed value? Note that Bigtable will always sort data based on
// the raw encoded value, *not* the decoded type.
// - Example: BYTES values sort in the same order as their raw encodings.
// - Counterexample: Encoding INT64 to a fixed-width STRING does *not*
// preserve sort order when dealing with negative numbers.
// INT64(1) > INT64(-1), but STRING("-00001") > STRING("00001).
// - The overall encoding chain has this property if *every* link does.
// - Counterexample: Encoding INT64 as a fixed-width decimal string does
// *not* preserve sort order when dealing with negative numbers.
// `INT64(1) > INT64(-1)`, but `STRING("-00001") > STRING("00001)`.
// * Self-delimiting: If we concatenate two encoded values, can we always tell
// where the first one ends and the second one begins?
// - Example: If we encode INT64s to fixed-width STRINGs, the first value
// will always contain exactly N digits, possibly preceded by a sign.
// - Counterexample: If we concatenate two UTF-8 encoded STRINGs, we have
// no way to tell where the first one ends.
// - The overall encoding chain has this property if *any* link does.
// * Compatibility: Which other systems have matching encoding schemes? For
// example, does this encoding have a GoogleSQL equivalent? HBase? Java?
message Type {
Expand All @@ -62,7 +57,7 @@ message Type {
// Rules used to convert to/from lower level types.
message Encoding {
// Leaves the value "as-is"
// * Natural sort? Yes
// * Order-preserving? Yes
// * Self-delimiting? No
// * Compatibility? N/A
message Raw {}
Expand All @@ -83,19 +78,27 @@ message Type {
message String {
// Rules used to convert to/from lower level types.
message Encoding {
// Deprecated: prefer the equivalent `Utf8Bytes`.
message Utf8Raw {
option deprecated = true;
}

// UTF-8 encoding
// * Natural sort? No (ASCII characters only)
// * Order-preserving? Yes (code point order)
// * Self-delimiting? No
// * Compatibility?
// - BigQuery Federation `TEXT` encoding
// - HBase `Bytes.toBytes`
// - Java `String#getBytes(StandardCharsets.UTF_8)`
message Utf8Raw {}
message Utf8Bytes {}

// Which encoding to use.
oneof encoding {
// Use `Utf8Raw` encoding.
Utf8Raw utf8_raw = 1;
// Deprecated: if set, converts to an empty `utf8_bytes`.
Utf8Raw utf8_raw = 1 [deprecated = true];

// Use `Utf8Bytes` encoding.
Utf8Bytes utf8_bytes = 2;
}
}

Expand All @@ -110,14 +113,14 @@ message Type {
message Encoding {
// Encodes the value as an 8-byte big endian twos complement `Bytes`
// value.
// * Natural sort? No (positive values only)
// * Order-preserving? No (positive values only)
// * Self-delimiting? Yes
// * Compatibility?
// - BigQuery Federation `BINARY` encoding
// - HBase `Bytes.toBytes`
// - Java `ByteBuffer.putLong()` with `ByteOrder.BIG_ENDIAN`
message BigEndianBytes {
// The underlying `Bytes` type, which may be able to encode further.
// Deprecated: ignored if set.
Bytes bytes_type = 1;
}

Expand All @@ -132,6 +135,68 @@ message Type {
Encoding encoding = 1;
}

// bool
// Values of type `Bool` are stored in `Value.bool_value`.
message Bool {}

// Float32
// Values of type `Float32` are stored in `Value.float_value`.
message Float32 {}

// Float64
// Values of type `Float64` are stored in `Value.float_value`.
message Float64 {}

// Timestamp
// Values of type `Timestamp` are stored in `Value.timestamp_value`.
message Timestamp {}

// Date
// Values of type `Date` are stored in `Value.date_value`.
message Date {}

// A structured data value, consisting of fields which map to dynamically
// typed values.
// Values of type `Struct` are stored in `Value.array_value` where entries are
// in the same order and number as `field_types`.
message Struct {
// A struct field and its type.
message Field {
// The field name (optional). Fields without a `field_name` are considered
// anonymous and cannot be referenced by name.
string field_name = 1;

// The type of values in this field.
Type type = 2;
}

// The names and types of the fields in this struct.
repeated Field fields = 1;
}

// An ordered list of elements of a given type.
// Values of type `Array` are stored in `Value.array_value`.
message Array {
// The type of the elements in the array. This must not be `Array`.
Type element_type = 1;
}

// A mapping of keys to values of a given type.
// Values of type `Map` are stored in a `Value.array_value` where each entry
// is another `Value.array_value` with two elements (the key and the value,
// in that order).
// Normally encoded Map values won't have repeated keys, however, clients are
// expected to handle the case in which they do. If the same key appears
// multiple times, the _last_ value takes precedence.
message Map {
// The type of a map key.
// Only `Bytes`, `String`, and `Int64` are allowed as key types.
Type key_type = 1;

// The type of the values in a map.
Type value_type = 2;
}

// A value that combines incremental updates into a summarized value.
//
// Data is never directly written or read using type `Aggregate`. Writes will
Expand All @@ -143,6 +208,25 @@ message Type {
// State: same as input
message Sum {}

// Computes the max of the input values.
// Allowed input: `Int64`
// State: same as input
message Max {}

// Computes the min of the input values.
// Allowed input: `Int64`
// State: same as input
message Min {}

// Computes an approximate unique count over the input values. When using
// raw data as input, be careful to use a consistent encoding. Otherwise
// the same value encoded differently could count more than once, or two
// distinct values could count as identical.
// Input: Any, or omit for Raw
// State: TBD
// Special state conversions: `Int64` (the unique count estimate)
message HyperLogLogPlusPlusUniqueCount {}

// Type of the inputs that are accumulated by this `Aggregate`, which must
// specify a full encoding.
// Use `AddInput` mutations to accumulate new inputs.
Expand All @@ -157,6 +241,15 @@ message Type {
oneof aggregator {
// Sum aggregator.
Sum sum = 4;

// HyperLogLogPlusPlusUniqueCount aggregator.
HyperLogLogPlusPlusUniqueCount hllpp_unique_count = 5;

// Max aggregator.
Max max = 6;

// Min aggregator.
Min min = 7;
}
}

Expand All @@ -171,7 +264,31 @@ message Type {
// Int64
Int64 int64_type = 5;

// Float32
Float32 float32_type = 12;

// Float64
Float64 float64_type = 9;

// Bool
Bool bool_type = 8;

// Timestamp
Timestamp timestamp_type = 10;

// Date
Date date_type = 11;

// Aggregate
Aggregate aggregate_type = 6;

// Struct
Struct struct_type = 7;

// Array
Array array_type = 3;

// Map
Map map_type = 4;
}
}
Loading
Loading