diff --git a/google/cloud/aiplatform/v1/feature_online_store.proto b/google/cloud/aiplatform/v1/feature_online_store.proto index a6a2040ab0095..53469fc74e01f 100644 --- a/google/cloud/aiplatform/v1/feature_online_store.proto +++ b/google/cloud/aiplatform/v1/feature_online_store.proto @@ -60,6 +60,19 @@ message FeatureOnlineStore { AutoScaling auto_scaling = 1 [(google.api.field_behavior) = REQUIRED]; } + // Optimized storage type + message Optimized {} + + // The dedicated serving endpoint for this FeatureOnlineStore. Only need to + // set when you choose Optimized storage type. Public endpoint is provisioned + // by default. + message DedicatedServingEndpoint { + // Output only. This field will be populated with the domain name to use for + // this FeatureOnlineStore + string public_endpoint_domain_name = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; + } + // Possible states a featureOnlineStore can have. enum State { // Default value. This value is unused. @@ -82,6 +95,13 @@ message FeatureOnlineStore { // to serve featureValues for all FeatureViews under this // FeatureOnlineStore. Bigtable bigtable = 8; + + // Contains settings for the Optimized store that will be created + // to serve featureValues for all FeatureViews under this + // FeatureOnlineStore. When choose Optimized storage type, need to set + // [PrivateServiceConnectConfig.enable_private_service_connect][google.cloud.aiplatform.v1.PrivateServiceConnectConfig.enable_private_service_connect] + // to use private endpoint. Otherwise will use public endpoint by default. + Optimized optimized = 12; } // Identifier. Name of the FeatureOnlineStore. Format: @@ -115,4 +135,9 @@ message FeatureOnlineStore { // Output only. State of the featureOnlineStore. State state = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. The dedicated serving endpoint for this FeatureOnlineStore, which + // is different from common Vertex service endpoint. + DedicatedServingEndpoint dedicated_serving_endpoint = 10 + [(google.api.field_behavior) = OPTIONAL]; } diff --git a/google/cloud/aiplatform/v1/feature_online_store_service.proto b/google/cloud/aiplatform/v1/feature_online_store_service.proto index bb451126e2dee..82667c2434a2e 100644 --- a/google/cloud/aiplatform/v1/feature_online_store_service.proto +++ b/google/cloud/aiplatform/v1/feature_online_store_service.proto @@ -141,6 +141,11 @@ message FetchFeatureValuesResponse { // Feature values in proto Struct format. google.protobuf.Struct proto_struct = 2; } + + // The data key associated with this response. + // Will only be populated for + // [FeatureOnlineStoreService.StreamingFetchFeatureValues][] RPCs. + FeatureViewDataKey data_key = 4; } // A query to find a number of similar entities. diff --git a/google/cloud/aiplatform/v1/feature_view.proto b/google/cloud/aiplatform/v1/feature_view.proto index 07b5b139d3cba..03f080f0db4e9 100644 --- a/google/cloud/aiplatform/v1/feature_view.proto +++ b/google/cloud/aiplatform/v1/feature_view.proto @@ -57,6 +57,85 @@ message FeatureView { string cron = 1; } + // Configuration for vector indexing. + message IndexConfig { + // Configuration options for using brute force search. + message BruteForceConfig {} + + // Configuration options for the tree-AH algorithm. + message TreeAHConfig { + // Optional. Number of embeddings on each leaf node. The default value is + // 1000 if not set. + optional int64 leaf_node_embedding_count = 1 + [(google.api.field_behavior) = OPTIONAL]; + } + + // The distance measure used in nearest neighbor search. + enum DistanceMeasureType { + // Should not be set. + DISTANCE_MEASURE_TYPE_UNSPECIFIED = 0; + + // Euclidean (L_2) Distance. + SQUARED_L2_DISTANCE = 1; + + // Cosine Distance. Defined as 1 - cosine similarity. + // + // We strongly suggest using DOT_PRODUCT_DISTANCE + UNIT_L2_NORM instead + // of COSINE distance. Our algorithms have been more optimized for + // DOT_PRODUCT distance which, when combined with UNIT_L2_NORM, is + // mathematically equivalent to COSINE distance and results in the same + // ranking. + COSINE_DISTANCE = 2; + + // Dot Product Distance. Defined as a negative of the dot product. + DOT_PRODUCT_DISTANCE = 3; + } + + // The configuration with regard to the algorithms used for efficient + // search. + oneof algorithm_config { + // Optional. Configuration options for the tree-AH algorithm (Shallow tree + // + Asymmetric Hashing). Please refer to this paper for more details: + // https://arxiv.org/abs/1908.10396 + TreeAHConfig tree_ah_config = 6 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Configuration options for using brute force search, which + // simply implements the standard linear search in the database for each + // query. It is primarily meant for benchmarking and to generate the + // ground truth for approximate search. + BruteForceConfig brute_force_config = 7 + [(google.api.field_behavior) = OPTIONAL]; + } + + // Optional. Column of embedding. This column contains the source data to + // create index for vector search. embedding_column must be set when using + // vector search. + string embedding_column = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Columns of features that're used to filter vector search + // results. + repeated string filter_columns = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Column of crowding. This column contains crowding attribute + // which is a constraint on a neighbor list produced by + // [FeatureOnlineStoreService.SearchNearestEntities][google.cloud.aiplatform.v1.FeatureOnlineStoreService.SearchNearestEntities] + // to diversify search results. If + // [NearestNeighborQuery.per_crowding_attribute_neighbor_count][google.cloud.aiplatform.v1.NearestNeighborQuery.per_crowding_attribute_neighbor_count] + // is set to K in + // [SearchNearestEntitiesRequest][google.cloud.aiplatform.v1.SearchNearestEntitiesRequest], + // it's guaranteed that no more than K entities of the same crowding + // attribute are returned in the response. + string crowding_column = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The number of dimensions of the input embedding. + optional int32 embedding_dimension = 4 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The distance measure used in nearest neighbor search. + DistanceMeasureType distance_measure_type = 5 + [(google.api.field_behavior) = OPTIONAL]; + } + // A Feature Registry source for features that need to be synced to Online // Store. message FeatureRegistrySource { @@ -123,4 +202,10 @@ message FeatureView { // end of the sync the latest featureValues for each entityId of this // FeatureView are made ready for online serving. SyncConfig sync_config = 7; + + // Optional. Configuration for index preparation for vector search. It + // contains the required configurations to create an index from source data, + // so that approximate nearest neighbor (a.k.a ANN) algorithms search can be + // performed during online serving. + IndexConfig index_config = 15 [(google.api.field_behavior) = OPTIONAL]; }