Skip to content

Commit

Permalink
feat(bigquery): add external dataset reference (#8545)
Browse files Browse the repository at this point in the history 
Adds preview functionality for manipulating external dataset references.
  • Loading branch information
@shollyman
shollyman committed Sep 11, 2023
1 parent 7f62291 commit 1001acf
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 100 deletions.
44 changes: 44 additions & 0 deletions bigquery/dataset.go
View file
Expand Up @@ -58,6 +58,9 @@ type DatasetMetadata struct {
// More information: https://cloud.google.com/bigquery/docs/reference/standard-sql/collation-concepts
DefaultCollation string

// For externally defined datasets, contains information about the configuration.
ExternalDatasetReference *ExternalDatasetReference

// MaxTimeTravel represents the number of hours for the max time travel for all tables
// in the dataset. Durations are rounded towards zero for the nearest hourly value.
MaxTimeTravel time.Duration
Expand Down Expand Up @@ -135,6 +138,9 @@ type DatasetMetadataToUpdate struct {
// created in the dataset.
DefaultCollation optional.String

// For externally defined datasets, contains information about the configuration.
ExternalDatasetReference *ExternalDatasetReference

// MaxTimeTravel represents the number of hours for the max time travel for all tables
// in the dataset. Durations are rounded towards zero for the nearest hourly value.
MaxTimeTravel optional.Duration
Expand Down Expand Up @@ -239,6 +245,9 @@ func (dm *DatasetMetadata) toBQ() (*bq.Dataset, error) {
if dm.DefaultEncryptionConfig != nil {
ds.DefaultEncryptionConfiguration = dm.DefaultEncryptionConfig.toBQ()
}
if dm.ExternalDatasetReference != nil {
ds.ExternalDatasetReference = dm.ExternalDatasetReference.toBQ()
}
return ds, nil
}

Expand Down Expand Up @@ -304,6 +313,7 @@ func bqToDatasetMetadata(d *bq.Dataset, c *Client) (*DatasetMetadata, error) {
DefaultTableExpiration: time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
DefaultPartitionExpiration: time.Duration(d.DefaultPartitionExpirationMs) * time.Millisecond,
DefaultCollation: d.DefaultCollation,
ExternalDatasetReference: bqToExternalDatasetReference(d.ExternalDatasetReference),
MaxTimeTravel: time.Duration(d.MaxTimeTravelHours) * time.Hour,
StorageBillingModel: d.StorageBillingModel,
DefaultEncryptionConfig: bqToEncryptionConfig(d.DefaultEncryptionConfiguration),
Expand Down Expand Up @@ -395,6 +405,10 @@ func (dm *DatasetMetadataToUpdate) toBQ() (*bq.Dataset, error) {
ds.DefaultCollation = optional.ToString(dm.DefaultCollation)
forceSend("DefaultCollation")
}
if dm.ExternalDatasetReference != nil {
ds.ExternalDatasetReference = dm.ExternalDatasetReference.toBQ()
forceSend("ExternalDatasetReference")
}
if dm.MaxTimeTravel != nil {
dur := optional.ToDuration(dm.MaxTimeTravel)
if dur == 0 {
Expand Down Expand Up @@ -936,3 +950,33 @@ func bqToDatasetAccessEntry(entry *bq.DatasetAccessEntry, c *Client) *DatasetAcc
TargetTypes: entry.TargetTypes,
}
}

// ExternalDatasetReference provides information about external dataset metadata.
type ExternalDatasetReference struct {
//The connection id that is used to access the external_source.
// Format: projects/{project_id}/locations/{location_id}/connections/{connection_id}
Connection string

// External source that backs this dataset.
ExternalSource string
}

func bqToExternalDatasetReference(bq *bq.ExternalDatasetReference) *ExternalDatasetReference {
if bq == nil {
return nil
}
return &ExternalDatasetReference{
Connection: bq.Connection,
ExternalSource: bq.ExternalSource,
}
}

func (edr *ExternalDatasetReference) toBQ() *bq.ExternalDatasetReference {
if edr == nil {
return nil
}
return &bq.ExternalDatasetReference{
Connection: edr.Connection,
ExternalSource: edr.ExternalSource,
}
}
26 changes: 25 additions & 1 deletion bigquery/dataset_test.go
View file
Expand Up @@ -327,6 +327,10 @@ func TestDatasetToBQ(t *testing.T) {
DefaultEncryptionConfig: &EncryptionConfig{
KMSKeyName: "some_key",
},
ExternalDatasetReference: &ExternalDatasetReference{
Connection: "conn",
ExternalSource: "external_src",
},
Location: "EU",
Labels: map[string]string{"x": "y"},
Access: []*AccessEntry{
Expand All @@ -348,6 +352,10 @@ func TestDatasetToBQ(t *testing.T) {
DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
KmsKeyName: "some_key",
},
ExternalDatasetReference: &bq.ExternalDatasetReference{
Connection: "conn",
ExternalSource: "external_src",
},
Location: "EU",
Labels: map[string]string{"x": "y"},
Access: []*bq.DatasetAccess{
Expand Down Expand Up @@ -404,6 +412,10 @@ func TestBQToDatasetMetadata(t *testing.T) {
DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
KmsKeyName: "some_key",
},
ExternalDatasetReference: &bq.ExternalDatasetReference{
Connection: "conn",
ExternalSource: "external_src",
},
Location: "EU",
Labels: map[string]string{"x": "y"},
Access: []*bq.DatasetAccess{
Expand Down Expand Up @@ -436,6 +448,10 @@ func TestBQToDatasetMetadata(t *testing.T) {
DefaultEncryptionConfig: &EncryptionConfig{
KMSKeyName: "some_key",
},
ExternalDatasetReference: &ExternalDatasetReference{
Connection: "conn",
ExternalSource: "external_src",
},
StorageBillingModel: LogicalStorageBillingModel,
Location: "EU",
Labels: map[string]string{"x": "y"},
Expand Down Expand Up @@ -476,6 +492,10 @@ func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
DefaultEncryptionConfig: &EncryptionConfig{
KMSKeyName: "some_key",
},
ExternalDatasetReference: &ExternalDatasetReference{
Connection: "conn",
ExternalSource: "external_src",
},
}
dm.SetLabel("label", "value")
dm.DeleteLabel("del")
Expand All @@ -495,8 +515,12 @@ func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
KmsKeyName: "some_key",
ForceSendFields: []string{"KmsKeyName"},
},
ExternalDatasetReference: &bq.ExternalDatasetReference{
Connection: "conn",
ExternalSource: "external_src",
},
Labels: map[string]string{"label": "value"},
ForceSendFields: []string{"Description", "FriendlyName", "StorageBillingModel"},
ForceSendFields: []string{"Description", "FriendlyName", "ExternalDatasetReference", "StorageBillingModel"},
NullFields: []string{"Labels.del"},
}
if diff := testutil.Diff(got, want); diff != "" {
Expand Down
36 changes: 18 additions & 18 deletions bigquery/go.mod
View file
Expand Up @@ -3,29 +3,29 @@ module cloud.google.com/go/bigquery
go 1.19

require (
cloud.google.com/go v0.110.2
cloud.google.com/go/datacatalog v1.14.0
cloud.google.com/go/iam v1.1.0
cloud.google.com/go v0.110.6
cloud.google.com/go/datacatalog v1.16.0
cloud.google.com/go/iam v1.1.1
cloud.google.com/go/storage v1.30.1
github.com/apache/arrow/go/v12 v12.0.0
github.com/google/go-cmp v0.5.9
github.com/google/uuid v1.3.0
github.com/googleapis/gax-go/v2 v2.12.0
go.opencensus.io v0.24.0
golang.org/x/sync v0.2.0
golang.org/x/sync v0.3.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
google.golang.org/api v0.128.0
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc
google.golang.org/grpc v1.56.1
google.golang.org/api v0.139.0
google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d
google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0
)

require (
cloud.google.com/go/compute v1.19.3 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/longrunning v0.4.2 // indirect
cloud.google.com/go/longrunning v0.5.1 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/apache/thrift v0.16.0 // indirect
github.com/goccy/go-json v0.9.11 // indirect
Expand All @@ -34,8 +34,8 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/google/martian/v3 v3.3.2 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
Expand All @@ -44,12 +44,12 @@ require (
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/stretchr/testify v1.8.2 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/tools v0.9.1 // indirect
google.golang.org/appengine v1.6.7 // indirect
)

0 comments on commit 1001acf

Please sign in to comment.