This repository has been archived by the owner on Nov 14, 2023. It is now read-only.
/
classification.proto
175 lines (143 loc) · 7.2 KB
/
classification.proto
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.cloud.automl.v1;
import "google/api/annotations.proto";
option csharp_namespace = "Google.Cloud.AutoML.V1";
option go_package = "google.golang.org/genproto/googleapis/cloud/automl/v1;automl";
option java_multiple_files = true;
option java_outer_classname = "ClassificationProto";
option java_package = "com.google.cloud.automl.v1";
option php_namespace = "Google\\Cloud\\AutoMl\\V1";
option ruby_package = "Google::Cloud::AutoML::V1";
// Type of the classification problem.
enum ClassificationType {
// An un-set value of this enum.
CLASSIFICATION_TYPE_UNSPECIFIED = 0;
// At most one label is allowed per example.
MULTICLASS = 1;
// Multiple labels are allowed for one example.
MULTILABEL = 2;
}
// Contains annotation details specific to classification.
message ClassificationAnnotation {
// Output only. A confidence estimate between 0.0 and 1.0. A higher value
// means greater confidence that the annotation is positive. If a user
// approves an annotation as negative or positive, the score value remains
// unchanged. If a user creates an annotation, the score is 0 for negative or
// 1 for positive.
float score = 1;
}
// Model evaluation metrics for classification problems.
// Note: For Video Classification this metrics only describe quality of the
// Video Classification predictions of "segment_classification" type.
message ClassificationEvaluationMetrics {
// Metrics for a single confidence threshold.
message ConfidenceMetricsEntry {
// Output only. Metrics are computed with an assumption that the model
// never returns predictions with score lower than this value.
float confidence_threshold = 1;
// Output only. Metrics are computed with an assumption that the model
// always returns at most this many predictions (ordered by their score,
// descendingly), but they all still need to meet the confidence_threshold.
int32 position_threshold = 14;
// Output only. Recall (True Positive Rate) for the given confidence
// threshold.
float recall = 2;
// Output only. Precision for the given confidence threshold.
float precision = 3;
// Output only. False Positive Rate for the given confidence threshold.
float false_positive_rate = 8;
// Output only. The harmonic mean of recall and precision.
float f1_score = 4;
// Output only. The Recall (True Positive Rate) when only considering the
// label that has the highest prediction score and not below the confidence
// threshold for each example.
float recall_at1 = 5;
// Output only. The precision when only considering the label that has the
// highest prediction score and not below the confidence threshold for each
// example.
float precision_at1 = 6;
// Output only. The False Positive Rate when only considering the label that
// has the highest prediction score and not below the confidence threshold
// for each example.
float false_positive_rate_at1 = 9;
// Output only. The harmonic mean of [recall_at1][google.cloud.automl.v1.ClassificationEvaluationMetrics.ConfidenceMetricsEntry.recall_at1] and [precision_at1][google.cloud.automl.v1.ClassificationEvaluationMetrics.ConfidenceMetricsEntry.precision_at1].
float f1_score_at1 = 7;
// Output only. The number of model created labels that match a ground truth
// label.
int64 true_positive_count = 10;
// Output only. The number of model created labels that do not match a
// ground truth label.
int64 false_positive_count = 11;
// Output only. The number of ground truth labels that are not matched
// by a model created label.
int64 false_negative_count = 12;
// Output only. The number of labels that were not created by the model,
// but if they would, they would not match a ground truth label.
int64 true_negative_count = 13;
}
// Confusion matrix of the model running the classification.
message ConfusionMatrix {
// Output only. A row in the confusion matrix.
message Row {
// Output only. Value of the specific cell in the confusion matrix.
// The number of values each row has (i.e. the length of the row) is equal
// to the length of the `annotation_spec_id` field or, if that one is not
// populated, length of the [display_name][google.cloud.automl.v1.ClassificationEvaluationMetrics.ConfusionMatrix.display_name] field.
repeated int32 example_count = 1;
}
// Output only. IDs of the annotation specs used in the confusion matrix.
// For Tables CLASSIFICATION
// [prediction_type][google.cloud.automl.v1p1beta.TablesModelMetadata.prediction_type]
// only list of [annotation_spec_display_name-s][] is populated.
repeated string annotation_spec_id = 1;
// Output only. Display name of the annotation specs used in the confusion
// matrix, as they were at the moment of the evaluation. For Tables
// CLASSIFICATION
// [prediction_type-s][google.cloud.automl.v1p1beta.TablesModelMetadata.prediction_type],
// distinct values of the target column at the moment of the model
// evaluation are populated here.
repeated string display_name = 3;
// Output only. Rows in the confusion matrix. The number of rows is equal to
// the size of `annotation_spec_id`.
// `row[i].example_count[j]` is the number of examples that have ground
// truth of the `annotation_spec_id[i]` and are predicted as
// `annotation_spec_id[j]` by the model being evaluated.
repeated Row row = 2;
}
// Output only. The Area Under Precision-Recall Curve metric. Micro-averaged
// for the overall evaluation.
float au_prc = 1;
// Output only. The Area Under Receiver Operating Characteristic curve metric.
// Micro-averaged for the overall evaluation.
float au_roc = 6;
// Output only. The Log Loss metric.
float log_loss = 7;
// Output only. Metrics for each confidence_threshold in
// 0.00,0.05,0.10,...,0.95,0.96,0.97,0.98,0.99 and
// position_threshold = INT32_MAX_VALUE.
// ROC and precision-recall curves, and other aggregated metrics are derived
// from them. The confidence metrics entries may also be supplied for
// additional values of position_threshold, but from these no aggregated
// metrics are computed.
repeated ConfidenceMetricsEntry confidence_metrics_entry = 3;
// Output only. Confusion matrix of the evaluation.
// Only set for MULTICLASS classification problems where number
// of labels is no more than 10.
// Only set for model level evaluation, not for evaluation per label.
ConfusionMatrix confusion_matrix = 4;
// Output only. The annotation spec ids used for this evaluation.
repeated string annotation_spec_id = 5;
}