From af9276d04ec2c94374aac06130f76e08885e9ec3 Mon Sep 17 00:00:00 2001 From: Modular Magician Date: Tue, 6 Dec 2022 18:24:10 +0000 Subject: [PATCH] Added field `network_config` to resource `google_dataproc_metastore_service` (#6893) Signed-off-by: Modular Magician --- .changelog/6893.txt | 3 + google/resource_dataproc_metastore_service.go | 139 ++++++++++++++++++ ...esource_dataproc_metastore_service_test.go | 51 +++++++ .../dataproc_metastore_service.html.markdown | 62 ++++++++ 4 files changed, 255 insertions(+) create mode 100644 .changelog/6893.txt diff --git a/.changelog/6893.txt b/.changelog/6893.txt new file mode 100644 index 00000000000..b0a30592964 --- /dev/null +++ b/.changelog/6893.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +metastore: added general field `network_config` to `google_dataproc_metastore_service` +``` diff --git a/google/resource_dataproc_metastore_service.go b/google/resource_dataproc_metastore_service.go index b45b7ca29f0..79bbfc04cb5 100644 --- a/google/resource_dataproc_metastore_service.go +++ b/google/resource_dataproc_metastore_service.go @@ -184,6 +184,40 @@ Maintenance window is not needed for services with the 'SPANNER' database type.` "projects/{projectNumber}/global/networks/{network_id}".`, }, + "network_config": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: `The configuration specifying the network settings for the Dataproc Metastore service.`, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "consumers": { + Type: schema.TypeList, + Required: true, + ForceNew: true, + Description: `The consumer-side network configuration for the Dataproc Metastore instance.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "subnetwork": { + Type: schema.TypeString, + Required: true, + Description: `The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint. +It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network. +There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form: +'projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id}`, + }, + "endpoint_uri": { + Type: schema.TypeString, + Computed: true, + Description: `The URI of the endpoint used to access the metastore service.`, + }, + }, + }, + }, + }, + }, + }, "port": { Type: schema.TypeInt, Computed: true, @@ -296,6 +330,12 @@ func resourceDataprocMetastoreServiceCreate(d *schema.ResourceData, meta interfa } else if v, ok := d.GetOkExists("hive_metastore_config"); !isEmptyValue(reflect.ValueOf(hiveMetastoreConfigProp)) && (ok || !reflect.DeepEqual(v, hiveMetastoreConfigProp)) { obj["hiveMetastoreConfig"] = hiveMetastoreConfigProp } + networkConfigProp, err := expandDataprocMetastoreServiceNetworkConfig(d.Get("network_config"), d, config) + if err != nil { + return err + } else if v, ok := d.GetOkExists("network_config"); !isEmptyValue(reflect.ValueOf(networkConfigProp)) && (ok || !reflect.DeepEqual(v, networkConfigProp)) { + obj["networkConfig"] = networkConfigProp + } databaseTypeProp, err := expandDataprocMetastoreServiceDatabaseType(d.Get("database_type"), d, config) if err != nil { return err @@ -425,6 +465,9 @@ func resourceDataprocMetastoreServiceRead(d *schema.ResourceData, meta interface if err := d.Set("hive_metastore_config", flattenDataprocMetastoreServiceHiveMetastoreConfig(res["hiveMetastoreConfig"], d, config)); err != nil { return fmt.Errorf("Error reading Service: %s", err) } + if err := d.Set("network_config", flattenDataprocMetastoreServiceNetworkConfig(res["networkConfig"], d, config)); err != nil { + return fmt.Errorf("Error reading Service: %s", err) + } if err := d.Set("database_type", flattenDataprocMetastoreServiceDatabaseType(res["databaseType"], d, config)); err != nil { return fmt.Errorf("Error reading Service: %s", err) } @@ -787,6 +830,46 @@ func flattenDataprocMetastoreServiceHiveMetastoreConfigKerberosConfigKrb5ConfigG return v } +func flattenDataprocMetastoreServiceNetworkConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} { + if v == nil { + return nil + } + original := v.(map[string]interface{}) + if len(original) == 0 { + return nil + } + transformed := make(map[string]interface{}) + transformed["consumers"] = + flattenDataprocMetastoreServiceNetworkConfigConsumers(original["consumers"], d, config) + return []interface{}{transformed} +} +func flattenDataprocMetastoreServiceNetworkConfigConsumers(v interface{}, d *schema.ResourceData, config *Config) interface{} { + if v == nil { + return v + } + l := v.([]interface{}) + transformed := make([]interface{}, 0, len(l)) + for _, raw := range l { + original := raw.(map[string]interface{}) + if len(original) < 1 { + // Do not include empty json objects coming back from the api + continue + } + transformed = append(transformed, map[string]interface{}{ + "endpoint_uri": flattenDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(original["endpointUri"], d, config), + "subnetwork": flattenDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(original["subnetwork"], d, config), + }) + } + return transformed +} +func flattenDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + +func flattenDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + func flattenDataprocMetastoreServiceDatabaseType(v interface{}, d *schema.ResourceData, config *Config) interface{} { return v } @@ -991,6 +1074,62 @@ func expandDataprocMetastoreServiceHiveMetastoreConfigKerberosConfigKrb5ConfigGc return v, nil } +func expandDataprocMetastoreServiceNetworkConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + transformedConsumers, err := expandDataprocMetastoreServiceNetworkConfigConsumers(original["consumers"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedConsumers); val.IsValid() && !isEmptyValue(val) { + transformed["consumers"] = transformedConsumers + } + + return transformed, nil +} + +func expandDataprocMetastoreServiceNetworkConfigConsumers(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + l := v.([]interface{}) + req := make([]interface{}, 0, len(l)) + for _, raw := range l { + if raw == nil { + continue + } + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + transformedEndpointUri, err := expandDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(original["endpoint_uri"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedEndpointUri); val.IsValid() && !isEmptyValue(val) { + transformed["endpointUri"] = transformedEndpointUri + } + + transformedSubnetwork, err := expandDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(original["subnetwork"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedSubnetwork); val.IsValid() && !isEmptyValue(val) { + transformed["subnetwork"] = transformedSubnetwork + } + + req = append(req, transformed) + } + return req, nil +} + +func expandDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func expandDataprocMetastoreServiceDatabaseType(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { return v, nil } diff --git a/google/resource_dataproc_metastore_service_test.go b/google/resource_dataproc_metastore_service_test.go index a073e02263f..bc272d7bbae 100644 --- a/google/resource_dataproc_metastore_service_test.go +++ b/google/resource_dataproc_metastore_service_test.go @@ -51,3 +51,54 @@ resource "google_dataproc_metastore_service" "my_metastore" { } `, name, tier) } + +func TestAccDataprocMetastoreService_PrivateServiceConnect(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": randString(t, 10), + } + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataprocMetastoreServiceDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataprocMetastoreService_PrivateServiceConnect(context), + }, + { + ResourceName: "google_dataproc_metastore_service.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"service_id", "location"}, + }, + }, + }) +} + +func testAccDataprocMetastoreService_PrivateServiceConnect(context map[string]interface{}) string { + return Nprintf(` +// Use data source instead of creating a subnetwork due to a bug on API side. +// With the bug, the new created subnetwork cannot be deleted when deleting the dataproc metastore service. +data "google_compute_subnetwork" "subnet" { + name = "default" + region = "us-central1" +} + +resource "google_dataproc_metastore_service" "default" { + service_id = "tf-test-metastore-srv%{random_suffix}" + location = "us-central1" + + hive_metastore_config { + version = "3.1.2" + } + + network_config { + consumers { + subnetwork = data.google_compute_subnetwork.subnet.id + } + } +} +`, context) +} diff --git a/website/docs/r/dataproc_metastore_service.html.markdown b/website/docs/r/dataproc_metastore_service.html.markdown index 5bef6e1b8d7..e6e687f7130 100644 --- a/website/docs/r/dataproc_metastore_service.html.markdown +++ b/website/docs/r/dataproc_metastore_service.html.markdown @@ -23,6 +23,11 @@ description: |- A managed metastore service that serves metadata queries. +To get more information about Service, see: + +* [API documentation](https://cloud.google.com/dataproc-metastore/docs/reference/rest/v1/projects.locations.services) +* How-to Guides + * [Official Documentation](https://cloud.google.com/dataproc-metastore/docs/overview)
@@ -80,6 +85,38 @@ resource "google_kms_key_ring" "key_ring" { location = "us-central1" } ``` +## Example Usage - Dataproc Metastore Service Private Service Connect + + +```hcl +resource "google_compute_network" "net" { + name = "my-network" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "subnet" { + name = "my-subnetwork" + region = "us-central1" + network = google_compute_network.net.id + ip_cidr_range = "10.0.0.0/22" + private_ip_google_access = true +} + +resource "google_dataproc_metastore_service" "default" { + service_id = "metastore-srv" + location = "us-central1" + + hive_metastore_config { + version = "3.1.2" + } + + network_config { + consumers { + subnetwork = google_compute_subnetwork.subnet.id + } + } +} +``` ## Argument Reference @@ -132,6 +169,11 @@ The following arguments are supported: Configuration information specific to running Hive metastore software as the metastore service. Structure is [documented below](#nested_hive_metastore_config). +* `network_config` - + (Optional) + The configuration specifying the network settings for the Dataproc Metastore service. + Structure is [documented below](#nested_network_config). + * `database_type` - (Optional) The database type that the Metastore service stores its data. @@ -244,6 +286,26 @@ The following arguments are supported: A mapping of Hive metastore configuration key-value pairs to apply to the auxiliary Hive metastore (configured in hive-site.xml) in addition to the primary version's overrides. If keys are present in both the auxiliary version's overrides and the primary version's overrides, the value from the auxiliary version's overrides takes precedence. +The `network_config` block supports: + +* `consumers` - + (Required) + The consumer-side network configuration for the Dataproc Metastore instance. + Structure is [documented below](#nested_consumers). + + +The `consumers` block supports: + +* `endpoint_uri` - + The URI of the endpoint used to access the metastore service. + +* `subnetwork` - + (Required) + The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint. + It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network. + There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form: + `projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id} + The `metadata_integration` block supports: * `data_catalog_config` -