Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add InputBytes to extract job statistics #2998

Merged
merged 3 commits into from Nov 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Expand Up @@ -60,13 +60,13 @@ implementation 'com.google.cloud:google-cloud-bigquery'
If you are using Gradle without BOM, add this to your dependencies:

```Groovy
implementation 'com.google.cloud:google-cloud-bigquery:2.34.1'
implementation 'com.google.cloud:google-cloud-bigquery:2.34.2'
```

If you are using SBT, add this to your dependencies:

```Scala
libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.34.1"
libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.34.2"
```
<!-- {x-version-update-end} -->

Expand Down Expand Up @@ -351,7 +351,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquery/java11.html
[stability-image]: https://img.shields.io/badge/stability-stable-green
[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquery.svg
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.34.1
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.34.2
[authentication]: https://github.com/googleapis/google-cloud-java#authentication
[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles
Expand Down
Expand Up @@ -101,16 +101,21 @@ public static class ExtractStatistics extends JobStatistics {

private final List<Long> destinationUriFileCounts;

private final Long inputBytes;

static final class Builder extends JobStatistics.Builder<ExtractStatistics, Builder> {

private List<Long> destinationUriFileCounts;

private Long inputBytes;

private Builder() {}

private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) {
super(statisticsPb);
if (statisticsPb.getExtract() != null) {
this.destinationUriFileCounts = statisticsPb.getExtract().getDestinationUriFileCounts();
this.inputBytes = statisticsPb.getExtract().getInputBytes();
}
}

Expand All @@ -119,6 +124,11 @@ Builder setDestinationUriFileCounts(List<Long> destinationUriFileCounts) {
return self();
}

Builder setInputBytes(Long inputBytes) {
this.inputBytes = inputBytes;
return self();
}

@Override
ExtractStatistics build() {
return new ExtractStatistics(this);
Expand All @@ -128,6 +138,7 @@ ExtractStatistics build() {
private ExtractStatistics(Builder builder) {
super(builder);
this.destinationUriFileCounts = builder.destinationUriFileCounts;
this.inputBytes = builder.inputBytes;
}

/**
Expand All @@ -139,6 +150,11 @@ public List<Long> getDestinationUriFileCounts() {
return destinationUriFileCounts;
}

/** Returns number of user bytes extracted into the result. */
public Long getInputBytes() {
return inputBytes;
}

@Override
ToStringHelper toStringHelper() {
return super.toStringHelper().add("destinationUriFileCounts", destinationUriFileCounts);
Expand All @@ -159,9 +175,10 @@ public final int hashCode() {

@Override
com.google.api.services.bigquery.model.JobStatistics toPb() {
com.google.api.services.bigquery.model.JobStatistics statisticsPb = super.toPb();
return statisticsPb.setExtract(
new JobStatistics4().setDestinationUriFileCounts(destinationUriFileCounts));
JobStatistics4 extractStatisticsPb = new JobStatistics4();
extractStatisticsPb.setDestinationUriFileCounts(destinationUriFileCounts);
extractStatisticsPb.setInputBytes(inputBytes);
return super.toPb().setExtract(extractStatisticsPb);
}

static Builder newBuilder() {
Expand Down
Expand Up @@ -97,6 +97,7 @@ public class JobStatisticsTest {
.setEndTime(END_TIME)
.setStartTime(START_TIME)
.setDestinationUriFileCounts(FILE_COUNT)
.setInputBytes(INPUT_BYTES)
.build();
private static final LoadStatistics LOAD_STATISTICS =
LoadStatistics.newBuilder()
Expand Down Expand Up @@ -249,6 +250,7 @@ public void testBuilder() {
assertEquals(START_TIME, EXTRACT_STATISTICS.getStartTime());
assertEquals(END_TIME, EXTRACT_STATISTICS.getEndTime());
assertEquals(FILE_COUNT, EXTRACT_STATISTICS.getDestinationUriFileCounts());
assertEquals(INPUT_BYTES, EXTRACT_STATISTICS.getInputBytes());

assertEquals(CREATION_TIME, LOAD_STATISTICS.getCreationTime());
assertEquals(START_TIME, LOAD_STATISTICS.getStartTime());
Expand Down Expand Up @@ -385,6 +387,7 @@ private void compareExtractStatistics(ExtractStatistics expected, ExtractStatist
assertEquals(expected, value);
compareStatistics(expected, value);
assertEquals(expected.getDestinationUriFileCounts(), value.getDestinationUriFileCounts());
assertEquals(expected.getInputBytes(), value.getInputBytes());
}

private void compareLoadStatistics(LoadStatistics expected, LoadStatistics value) {
Expand Down
Expand Up @@ -87,6 +87,7 @@
import com.google.cloud.bigquery.JobId;
import com.google.cloud.bigquery.JobInfo;
import com.google.cloud.bigquery.JobStatistics;
import com.google.cloud.bigquery.JobStatistics.ExtractStatistics;
import com.google.cloud.bigquery.JobStatistics.LoadStatistics;
import com.google.cloud.bigquery.JobStatistics.QueryStatistics;
import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType;
Expand Down Expand Up @@ -5304,6 +5305,8 @@ public void testExtractJob() throws InterruptedException, TimeoutException {
assertNull(remoteLoadJob.getStatus().getError());
LoadJobConfiguration loadJobConfiguration = remoteLoadJob.getConfiguration();
assertEquals(labels, loadJobConfiguration.getLabels());
LoadStatistics loadStatistics = remoteLoadJob.getStatistics();
assertNotNull(loadStatistics);

ExtractJobConfiguration extractConfiguration =
ExtractJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE)
Expand All @@ -5313,6 +5316,12 @@ public void testExtractJob() throws InterruptedException, TimeoutException {
remoteExtractJob = remoteExtractJob.waitFor();
assertNull(remoteExtractJob.getStatus().getError());

ExtractStatistics extractStatistics = remoteExtractJob.getStatistics();
assertNotNull(extractStatistics);
assertEquals(1L, extractStatistics.getDestinationUriFileCounts().size());
assertEquals(
loadStatistics.getOutputBytes().longValue(), extractStatistics.getInputBytes().longValue());

String extractedCsv =
new String(storage.readAllBytes(BUCKET, EXTRACT_FILE), StandardCharsets.UTF_8);
assertEquals(
Expand Down