Skip to content

Commit

Permalink
feat: add remote function options to routine metadata (#2291)
Browse files Browse the repository at this point in the history
  • Loading branch information
Neenu1995 committed Sep 26, 2022
1 parent 28dd4e8 commit d30670e
Show file tree
Hide file tree
Showing 10 changed files with 428 additions and 8 deletions.
12 changes: 6 additions & 6 deletions .kokoro/build.sh
Expand Up @@ -61,7 +61,7 @@ javadoc)
integration)
mvn -B ${INTEGRATION_TEST_ARGS} \
-ntp \
-Dtest=ITBigQueryTest \
-Dtest=ITBigQueryTest,ITRemoteUDFTest \
-DtrimStackTrace=false \
-Dclirr.skip=true \
-Denforcer.skip=true \
Expand All @@ -72,7 +72,7 @@ integration)
nightly-it)
mvn -B ${INTEGRATION_TEST_ARGS} \
-ntp \
-Dtest=ITNightlyBigQueryTest \
-Dtest=ITNightlyBigQueryTest,ITRemoteUDFTest \
-DtrimStackTrace=false \
-Dclirr.skip=true \
-Denforcer.skip=true \
Expand All @@ -82,20 +82,20 @@ nightly-it)
;;
graalvm)
# Run Integration Tests with Native Image. Skip running nightly tests in presubmits.
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITBigQueryTest -Pnative -Penable-integration-tests test
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITBigQueryTest,ITRemoteUDFTest -Pnative -Penable-integration-tests test
RETURN_CODE=$?
;;
graalvm17)
# Run Integration Tests with Native Image. Skip running nightly tests in presubmits.
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITBigQueryTest -Pnative -Penable-integration-tests test
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITBigQueryTest,ITRemoteUDFTest -Pnative -Penable-integration-tests test
RETURN_CODE=$?
;;
nightly-graalvm)
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITNightlyBigQueryTest -Pnative -Penable-integration-tests test
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITNightlyBigQueryTest,ITRemoteUDFTest -Pnative -Penable-integration-tests test
RETURN_CODE=$?
;;
nightly-graalvm17)
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITNightlyBigQueryTest -Pnative -Penable-integration-tests test
mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Dtest=ITNightlyBigQueryTest,ITRemoteUDFTest -Pnative -Penable-integration-tests test
RETURN_CODE=$?
;;
samples)
Expand Down
5 changes: 5 additions & 0 deletions google-cloud-bigquery/clirr-ignored-differences.xml
Expand Up @@ -19,4 +19,9 @@
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*ReferenceFileSchemaUri(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/RoutineInfo*</className>
<method>*RemoteFunctionOptions(*)</method>
</difference>
</differences>
7 changes: 6 additions & 1 deletion google-cloud-bigquery/pom.xml
Expand Up @@ -131,9 +131,14 @@
<artifactId>google-cloud-datacatalog</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigqueryconnection</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-datacatalog-v1</artifactId>
<artifactId>proto-google-cloud-bigqueryconnection-v1</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
@@ -0,0 +1,139 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.cloud.bigquery;

import com.google.auto.value.AutoValue;
import java.io.Serializable;
import java.util.Map;
import javax.annotation.Nullable;

/** Represents Remote Function Options. Options for a remote user-defined function. */
@AutoValue
public abstract class RemoteFunctionOptions implements Serializable {

private static final long serialVersionUID = -7334249450657429792L;

@AutoValue.Builder
public abstract static class Builder {

/**
* Sets Endpoint argument Endpoint of the user-provided remote service, e.g.
* ```https://us-east1-my_gcf_project.cloudfunctions.net/remote_add```
*/
public abstract Builder setEndpoint(String endpoint);

/**
* Fully qualified name of the user-provided connection object which holds the authentication
* information to send requests to the remote service. Format:
* ```\"projects/{projectId}/locations/{locationId}/connections/{connectionId}\"```
*/
public abstract Builder setConnection(String connection);

/**
* User-defined context as a set of key/value pairs, which will be sent as function invocation
* context together with batched arguments in the requests to the remote service. The total
* number of bytes of keys and values must be less than 8KB.
*/
public abstract Builder setUserDefinedContext(Map<String, String> userDefinedContext);

/**
* Max number of rows in each batch sent to the remote service. If absent or if 0, BigQuery
* dynamically decides the number of rows in a batch.
*/
public abstract Builder setMaxBatchingRows(Long maxBatchingRows);

/** Creates a {@code RemoteFunctionOptions} object. */
public abstract RemoteFunctionOptions build();
}

/**
* Returns the endpoint of the user-provided service.
*
* @return String
*/
@Nullable
public abstract String getEndpoint();

/**
* Returns the fully qualified name of the user-provided connection object.
*
* @return String
*/
@Nullable
public abstract String getConnection();

/**
* Returns the user-defined context as a set of key/value pairs.
*
* @return Map<String, String>
*/
@Nullable
public abstract Map<String, String> getUserDefinedContext();

/**
* Returns max number of rows in each batch sent to the remote service.
*
* @return Long
*/
@Nullable
public abstract Long getMaxBatchingRows();

/**
* Returns a builder pre-populated using the current values of this {@code RemoteFunctionOptions}.
*/
public abstract RemoteFunctionOptions.Builder toBuilder();

/** Returns a builder for a {@Code RemoteFunctionOptions} object. */
public static RemoteFunctionOptions.Builder newBuilder() {
return new AutoValue_RemoteFunctionOptions.Builder();
}

public com.google.api.services.bigquery.model.RemoteFunctionOptions toPb() {
com.google.api.services.bigquery.model.RemoteFunctionOptions remoteFunctionOptions =
new com.google.api.services.bigquery.model.RemoteFunctionOptions();
if (getEndpoint() != null) {
remoteFunctionOptions.setEndpoint(getEndpoint());
}
if (getConnection() != null) {
remoteFunctionOptions.setConnection(getConnection());
}
if (getUserDefinedContext() != null) {
remoteFunctionOptions.setUserDefinedContext(getUserDefinedContext());
}
if (getMaxBatchingRows() != null) {
remoteFunctionOptions.setMaxBatchingRows(getMaxBatchingRows());
}
return remoteFunctionOptions;
}

static RemoteFunctionOptions fromPb(
com.google.api.services.bigquery.model.RemoteFunctionOptions remoteFunctionOptionsPb) {
RemoteFunctionOptions.Builder builder = newBuilder();
if (remoteFunctionOptionsPb.getEndpoint() != null) {
builder.setEndpoint(remoteFunctionOptionsPb.getEndpoint());
}
if (remoteFunctionOptionsPb.getConnection() != null) {
builder.setConnection(remoteFunctionOptionsPb.getConnection());
}
if (remoteFunctionOptionsPb.getUserDefinedContext() != null) {
builder.setUserDefinedContext(remoteFunctionOptionsPb.getUserDefinedContext());
}
if (remoteFunctionOptionsPb.getMaxBatchingRows() != null) {
builder.setMaxBatchingRows(remoteFunctionOptionsPb.getMaxBatchingRows());
}
return builder.build();
}
}
Expand Up @@ -129,6 +129,12 @@ public Builder setBody(String body) {
return this;
}

@Override
public Builder setRemoteFunctionOptions(RemoteFunctionOptions remoteFunctionOptions) {
infoBuilder.setRemoteFunctionOptions(remoteFunctionOptions);
return this;
}

@Override
public Routine build() {
return new Routine(bigquery, infoBuilder);
Expand Down
Expand Up @@ -70,6 +70,7 @@ public Routine apply(RoutineInfo routineInfo) {
private final StandardSQLTableType returnTableType;
private final List<String> importedLibrariesList;
private final String body;
private final RemoteFunctionOptions remoteFunctionOptions;

public abstract static class Builder {

Expand Down Expand Up @@ -148,6 +149,14 @@ public abstract static class Builder {
*/
public abstract Builder setBody(String body);

/**
* Optional. Remote function specific options.
*
* @param remoteFunctionOptions
* @return
*/
public abstract Builder setRemoteFunctionOptions(RemoteFunctionOptions remoteFunctionOptions);

/** Creates a {@code RoutineInfo} object. */
public abstract RoutineInfo build();
}
Expand All @@ -166,6 +175,7 @@ static class BuilderImpl extends Builder {
private StandardSQLTableType returnTableType;
private List<String> importedLibrariesList;
private String body;
private RemoteFunctionOptions remoteFunctionOptions;

BuilderImpl() {}

Expand All @@ -183,6 +193,7 @@ static class BuilderImpl extends Builder {
this.returnTableType = routineInfo.returnTableType;
this.importedLibrariesList = routineInfo.importedLibrariesList;
this.body = routineInfo.body;
this.remoteFunctionOptions = routineInfo.remoteFunctionOptions;
}

BuilderImpl(Routine routinePb) {
Expand Down Expand Up @@ -210,6 +221,10 @@ static class BuilderImpl extends Builder {
this.importedLibrariesList = routinePb.getImportedLibraries();
}
this.body = routinePb.getDefinitionBody();
if (routinePb.getRemoteFunctionOptions() != null) {
this.remoteFunctionOptions =
RemoteFunctionOptions.fromPb(routinePb.getRemoteFunctionOptions());
}
}

@Override
Expand Down Expand Up @@ -290,6 +305,12 @@ public Builder setBody(String body) {
return this;
}

@Override
public Builder setRemoteFunctionOptions(RemoteFunctionOptions remoteFunctionOptions) {
this.remoteFunctionOptions = remoteFunctionOptions;
return this;
}

@Override
public RoutineInfo build() {
return new RoutineInfo(this);
Expand All @@ -310,6 +331,7 @@ public RoutineInfo build() {
this.returnTableType = builder.returnTableType;
this.importedLibrariesList = builder.importedLibrariesList;
this.body = builder.body;
this.remoteFunctionOptions = builder.remoteFunctionOptions;
}

/** Returns the RoutineId identified for the routine resource. * */
Expand Down Expand Up @@ -384,6 +406,11 @@ public String getBody() {
return body;
}

/** Returns the Remote function specific options. */
public RemoteFunctionOptions getRemoteFunctionOptions() {
return remoteFunctionOptions;
};

/** Returns a builder pre-populated using the current values of this routine. */
public Builder toBuilder() {
return new BuilderImpl(this);
Expand All @@ -405,6 +432,7 @@ public String toString() {
.add("returnTableType", returnTableType)
.add("importedLibrariesList", importedLibrariesList)
.add("body", body)
.add("remoteFunctionOptions", remoteFunctionOptions)
.toString();
}

Expand All @@ -423,7 +451,8 @@ public int hashCode() {
returnType,
returnTableType,
importedLibrariesList,
body);
body,
remoteFunctionOptions);
}

@Override
Expand Down Expand Up @@ -474,6 +503,9 @@ Routine toPb() {
if (getReturnTableType() != null) {
routinePb.setReturnTableType(getReturnTableType().toPb());
}
if (getRemoteFunctionOptions() != null) {
routinePb.setRemoteFunctionOptions(getRemoteFunctionOptions().toPb());
}
return routinePb;
}

Expand Down

0 comments on commit d30670e

Please sign in to comment.