Skip to content

Commit

Permalink
feat: A new value CANCEL is added to enum ScalingDecisionType
Browse files Browse the repository at this point in the history
feat: A new value `DO_NOT_CANCEL` is added to enum `ScalingDecisionType`
feat: A new value `SECONDARY_SCALEDOWN_SINGLE_REQUEST_LIMIT_REACHED` is added to enum `ConstrainingFactor`
feat: A new enum `MetricType` is added
feat: A new field `decision_metric` is added to message `.google.cloud.dataproc.logging.AutoscalerRecommendation`
feat: A new message `ReconciliationLog` is added
feat: A new message `ReconciliationClusterHealLog` is added
feat: A new enum `ReconciliationDecisionType` is added
docs: A comment for message `ClusterSize` is changed
docs: A comment for message `AutoscalerRecommendation` is changed

PiperOrigin-RevId: 620350169
  • Loading branch information
Google APIs authored and Copybara-Service committed Mar 29, 2024
1 parent 3ebeb01 commit 8b83319
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 27 deletions.
71 changes: 58 additions & 13 deletions google/cloud/dataproc/logging/BUILD.bazel
@@ -1,5 +1,7 @@
# This file was automatically generated by BuildFileGenerator

# buildifier: disable=load-on-top

# This is an API workspace, having public visibility by default makes perfect sense.
package(default_visibility = ["//visibility:public"])

Expand All @@ -12,6 +14,7 @@ proto_library(
name = "logging_proto",
srcs = [
"autoscaler_log.proto",
"reconciler_log.proto",
],
deps = [
"@com_google_protobuf//:duration_proto",
Expand All @@ -21,48 +24,63 @@ proto_library(
##############################################################################
# Java
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"java_grpc_library",
"java_proto_library",
"java_gapic_assembly_gradle_pkg",
)

java_proto_library(
name = "logging_java_proto",
deps = [":logging_proto"],
)

java_grpc_library(
name = "logging_java_grpc",
srcs = [":logging_proto"],
deps = [":logging_java_proto"],
# Open Source Packages
java_gapic_assembly_gradle_pkg(
name = "google-cloud-dataproc-logging-java",
deps = [
":logging_proto",
":logging_java_proto",
],
)

##############################################################################
# Go
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"go_proto_library",
"go_gapic_assembly_pkg",
)

go_proto_library(
name = "logging_go_proto",
compilers = ["@io_bazel_rules_go//proto:go_grpc"],
importpath = "cloud.google.com/go/dataproc/logging/loggingpb",
protos = [":logging_proto"],
deps = [],
)

go_gapic_assembly_pkg(
name = "google-cloud-dataproc-logging-go",
deps = [
":logging_go_proto",
],
)

##############################################################################
# Python
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"moved_proto_library",
"py_grpc_library",
"py_proto_library",
"py_gapic_library",
"py_gapic_assembly_pkg",
)

moved_proto_library(
Expand All @@ -84,11 +102,28 @@ py_grpc_library(
deps = [":logging_py_proto"],
)

py_gapic_library(
name = "logging_py_gapic",
srcs = [":logging_proto"],
rest_numeric_enums = False,
transport = "grpc+rest",
)

# Open Source Packages
py_gapic_assembly_pkg(
name = "google-cloud-dataproc-logging-py",
deps = [
":logging_py_gapic",
],
)

##############################################################################
# PHP
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"php_gapic_assembly_pkg",
"php_proto_library",
)

Expand All @@ -97,13 +132,17 @@ php_proto_library(
deps = [":logging_proto"],
)

##############################################################################
# Node.js
##############################################################################
php_gapic_assembly_pkg(
name = "google-cloud-dataproc-logging-php",
deps = [
":logging_php_proto",
],
)

##############################################################################
# Ruby
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"ruby_grpc_library",
Expand All @@ -124,26 +163,32 @@ ruby_grpc_library(
##############################################################################
# C#
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"csharp_grpc_library",
"csharp_proto_library",
"csharp_gapic_assembly_pkg",
)

csharp_proto_library(
name = "logging_csharp_proto",
deps = [":logging_proto"],
)

csharp_grpc_library(
name = "logging_csharp_grpc",
srcs = [":logging_proto"],
deps = [":logging_csharp_proto"],
# Open Source Packages
csharp_gapic_assembly_pkg(
name = "google-cloud-dataproc-logging-csharp",
package_name = "Google.Cloud.Dataproc.Logging",
generate_nongapic_package = True,
deps = [
":logging_csharp_proto",
],
)

##############################################################################
# C++
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"cc_grpc_library",
Expand Down
57 changes: 43 additions & 14 deletions google/cloud/dataproc/logging/autoscaler_log.proto
@@ -1,4 +1,4 @@
// Copyright 2020 Google LLC
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -18,11 +18,12 @@ package google.cloud.dataproc.logging;

import "google/protobuf/duration.proto";

option csharp_namespace = "Google.Cloud.Dataproc.Logging";
option go_package = "cloud.google.com/go/dataproc/logging/loggingpb;loggingpb";
option java_multiple_files = true;
option java_package = "com.google.cloud.dataproc.logging";

// The short version of cluster configuration for Cloud logging.
// The short version of cluster configuration for Cloud Logging.
message ClusterSize {
// The number of primary workers in the cluster.
int32 primary_worker_count = 1;
Expand All @@ -31,17 +32,6 @@ message ClusterSize {
int32 secondary_worker_count = 2;
}

// The main proto that will be converted to JSON format and then written to
// Logging.
message AutoscalerLog {
// The current Autoscaler status.
AutoscalerStatus status = 1;

// Optional. The autoscaling recommendation including its inputs, outputs,
// scaling decision, and detailed explanation.
AutoscalerRecommendation recommendation = 2;
}

// The Autoscaler state.
enum AutoscalerState {
AUTOSCALER_STATE_UNSPECIFIED = 0;
Expand Down Expand Up @@ -81,6 +71,12 @@ enum ScalingDecisionType {

// Scale the primary and secondary worker groups in different directions.
MIXED = 4;

// Cancel the ongoing scale down operation.
CANCEL = 5;

// Do not cancel the ongoing scale down operation.
DO_NOT_CANCEL = 6;
}

enum ConstrainingFactor {
Expand All @@ -99,6 +95,36 @@ enum ConstrainingFactor {
// issued if workers were able to be removed from another group that had not
// reached minimum size.
REACHED_MINIMUM_CLUSTER_SIZE = 3;

// The secondary worker group cannot be scaled down by more than 1k nodes in a
// single update request.
SECONDARY_SCALEDOWN_SINGLE_REQUEST_LIMIT_REACHED = 4;
}

// The kind of metric input to the Autoscaling algorithm.
enum MetricType {
// Default.
METRIC_TYPE_UNSPECIFIED = 0;

// The yarn memory metric.
YARN_MEMORY = 1;

// The yarn cores or vCPUs metric.
YARN_CORES = 2;

// The number of executors in Spark serverless.
SPARK_EXECUTORS = 3;
}

// The main proto that will be converted to JSON format and then written to
// Logging.
message AutoscalerLog {
// The current Autoscaler status.
AutoscalerStatus status = 1;

// Optional. The autoscaling recommendation including its inputs, outputs,
// scaling decision, and detailed explanation.
AutoscalerRecommendation recommendation = 2;
}

// The Autoscaler's status, including its state and other details.
Expand All @@ -119,7 +145,7 @@ message AutoscalerStatus {
// The inputs, outputs, and detailed explanation of the Autoscaling
// recommendation.
message AutoscalerRecommendation {
// The input values for the Autoscaling recommendation alogirthm.
// The input values for the Autoscaling recommendation algorithm.
message Inputs {
// The metrics collected by the Dataproc agent running on the cluster.
// For example, {"avg-yarn-pending-memory": "1040 MB"}
Expand Down Expand Up @@ -157,6 +183,9 @@ message AutoscalerRecommendation {
// A unique id for this recommendation that should be included when opening
// a support ticket.
string recommendation_id = 6;

// The metric source deciding the autoscaling recommendation.
MetricType decision_metric = 7;
}

// The autoscaling algorithm inputs.
Expand Down
83 changes: 83 additions & 0 deletions google/cloud/dataproc/logging/reconciler_log.proto
@@ -0,0 +1,83 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.dataproc.logging;

import "google/protobuf/duration.proto";

option csharp_namespace = "Google.Cloud.Dataproc.Logging";
option go_package = "cloud.google.com/go/dataproc/logging/loggingpb;loggingpb";
option java_multiple_files = true;
option java_outer_classname = "ReconcilerLogProto";
option java_package = "com.google.cloud.dataproc.logging";

// Reconciliation log for session ttl event.
message ReconciliationLog {
// The input values for the Reconciler recommendation algorithm.
// We could add more details in future if required.
message Inputs {
// Idle duration
google.protobuf.Duration idle_duration = 1;

// Configured idle TTL
google.protobuf.Duration idle_ttl = 2;

// Total session lifetime
google.protobuf.Duration session_lifetime = 3;

// Configured ttl
google.protobuf.Duration ttl = 4;
}

// Reconciler recommendations.
message Outputs {
// The high-level reconciliation decision.
ReconciliationDecisionType decision = 1;

// Human readable context messages which explain the reconciler decision.
string decision_details = 2;
}

// The reconciliation algorithm inputs.
Inputs inputs = 1;

// The algorithm outputs for the recommended reconciliation operation.
Outputs outputs = 2;
}

// Reconciliation log for cluster heal event.
message ReconciliationClusterHealLog {
// Autohealer decision.
message Outputs {
// The repair operation id triggered by Autohealer if any.
string repair_operation_id = 1;

// Human readable context messages which explain the autohealer decision.
string decision_details = 2;
}

// The algorithm outputs for the recommended reconciliation operation.
Outputs outputs = 1;
}

// Decision type
enum ReconciliationDecisionType {
// Unspecified type
RECONCILIATION_DECISION_TYPE_UNSPECIFIED = 0;

// Terminate session
RECONCILIATION_TERMINATE_SESSION = 1;
}

0 comments on commit 8b83319

Please sign in to comment.