Added CMEK support for cloud data fusion (#6659) (#12737)

Co-authored-by: Niranjan KL <niranjankl@google.com>
Signed-off-by: Modular Magician <magic-modules@google.com>

Signed-off-by: Modular Magician <magic-modules@google.com>
Co-authored-by: Niranjan KL <niranjankl@google.com>

.changelog/6659.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+datafusion: added `crypto_key_config` field to `google_data_fusion_instance` resource
+```

google/resource_data_fusion_instance.go

@@ -63,6 +63,23 @@ available, such as support for streaming pipelines, higher number of concurrent
 with restrictive capabilities. This is to help enterprises design and develop their data ingestion and integration 
 pipelines at low cost. Possible values: ["BASIC", "ENTERPRISE", "DEVELOPER"]`,
 			},
+			"crypto_key_config": {
+				Type:        schema.TypeList,
+				Optional:    true,
+				ForceNew:    true,
+				Description: `The crypto key configuration. This field is used by the Customer-Managed Encryption Keys (CMEK) feature.`,
+				MaxItems:    1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"key_reference": {
+							Type:        schema.TypeString,
+							Required:    true,
+							ForceNew:    true,
+							Description: `The name of the key which is used to encrypt/decrypt customer data. For key in Cloud KMS, the key should be in the format of projects/*/locations/*/keyRings/*/cryptoKeys/*.`,
+						},
+					},
+				},
+			},
 			"dataproc_service_account": {
 				Type:        schema.TypeString,
 				Optional:    true,
@@ -273,6 +290,12 @@ func resourceDataFusionInstanceCreate(d *schema.ResourceData, meta interface{})
 	} else if v, ok := d.GetOkExists("network_config"); !isEmptyValue(reflect.ValueOf(networkConfigProp)) && (ok || !reflect.DeepEqual(v, networkConfigProp)) {
 		obj["networkConfig"] = networkConfigProp
 	}
+	cryptoKeyConfigProp, err := expandDataFusionInstanceCryptoKeyConfig(d.Get("crypto_key_config"), d, config)
+	if err != nil {
+		return err
+	} else if v, ok := d.GetOkExists("crypto_key_config"); !isEmptyValue(reflect.ValueOf(cryptoKeyConfigProp)) && (ok || !reflect.DeepEqual(v, cryptoKeyConfigProp)) {
+		obj["cryptoKeyConfig"] = cryptoKeyConfigProp
+	}
 
 	url, err := replaceVars(d, config, "{{DataFusionBasePath}}projects/{{project}}/locations/{{region}}/instances?instanceId={{name}}")
 	if err != nil {
@@ -430,6 +453,9 @@ func resourceDataFusionInstanceRead(d *schema.ResourceData, meta interface{}) er
 	if err := d.Set("network_config", flattenDataFusionInstanceNetworkConfig(res["networkConfig"], d, config)); err != nil {
 		return fmt.Errorf("Error reading Instance: %s", err)
 	}
+	if err := d.Set("crypto_key_config", flattenDataFusionInstanceCryptoKeyConfig(res["cryptoKeyConfig"], d, config)); err != nil {
+		return fmt.Errorf("Error reading Instance: %s", err)
+	}
 
 	return nil
 }
@@ -660,6 +686,23 @@ func flattenDataFusionInstanceNetworkConfigNetwork(v interface{}, d *schema.Reso
 	return v
 }
 
+func flattenDataFusionInstanceCryptoKeyConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	if v == nil {
+		return nil
+	}
+	original := v.(map[string]interface{})
+	if len(original) == 0 {
+		return nil
+	}
+	transformed := make(map[string]interface{})
+	transformed["key_reference"] =
+		flattenDataFusionInstanceCryptoKeyConfigKeyReference(original["keyReference"], d, config)
+	return []interface{}{transformed}
+}
+func flattenDataFusionInstanceCryptoKeyConfigKeyReference(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	return v
+}
+
 func expandDataFusionInstanceName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
 	return replaceVars(d, config, "projects/{{project}}/locations/{{region}}/instances/{{name}}")
 }
@@ -747,3 +790,26 @@ func expandDataFusionInstanceNetworkConfigIpAllocation(v interface{}, d Terrafor
 func expandDataFusionInstanceNetworkConfigNetwork(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
 	return v, nil
 }
+
+func expandDataFusionInstanceCryptoKeyConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	l := v.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := make(map[string]interface{})
+
+	transformedKeyReference, err := expandDataFusionInstanceCryptoKeyConfigKeyReference(original["key_reference"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedKeyReference); val.IsValid() && !isEmptyValue(val) {
+		transformed["keyReference"] = transformedKeyReference
+	}
+
+	return transformed, nil
+}
+
+func expandDataFusionInstanceCryptoKeyConfigKeyReference(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	return v, nil
+}

google/resource_data_fusion_instance_generated_test.go

@@ -117,6 +117,68 @@ data "google_app_engine_default_service_account" "default" {
 `, context)
 }
 
+func TestAccDataFusionInstance_dataFusionInstanceCmekExample(t *testing.T) {
+	t.Parallel()
+
+	context := map[string]interface{}{
+		"random_suffix": randString(t, 10),
+	}
+
+	vcrTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckDataFusionInstanceDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataFusionInstance_dataFusionInstanceCmekExample(context),
+			},
+			{
+				ResourceName:            "google_data_fusion_instance.basic_cmek",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"region"},
+			},
+		},
+	})
+}
+
+func testAccDataFusionInstance_dataFusionInstanceCmekExample(context map[string]interface{}) string {
+	return Nprintf(`
+resource "google_data_fusion_instance" "basic_cmek" {
+  name   = "tf-test-my-instance%{random_suffix}"
+  region = "us-central1"
+  type   = "BASIC"
+
+  crypto_key_config {
+    key_reference = google_kms_crypto_key.crypto_key.id
+  }
+
+  depends_on = [google_kms_crypto_key_iam_binding.crypto_key_binding]
+}
+
+resource "google_kms_crypto_key" "crypto_key" {
+  name     = "tf-test-my-instance%{random_suffix}"
+  key_ring = google_kms_key_ring.key_ring.id
+}
+
+resource "google_kms_key_ring" "key_ring" {
+  name     = "tf-test-my-instance%{random_suffix}"
+  location = "us-central1"
+}
+
+resource "google_kms_crypto_key_iam_binding" "crypto_key_binding" {
+  crypto_key_id = google_kms_crypto_key.crypto_key.id
+  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+
+  members = [
+    "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datafusion.iam.gserviceaccount.com"
+  ]
+}
+
+data "google_project" "project" {}
+`, context)
+}
+
 func testAccCheckDataFusionInstanceDestroyProducer(t *testing.T) func(s *terraform.State) error {
 	return func(s *terraform.State) error {
 		for name, rs := range s.RootModule().Resources {

website/docs/r/data_fusion_instance.html.markdown

@@ -83,6 +83,48 @@ resource "google_data_fusion_instance" "extended_instance" {
 data "google_app_engine_default_service_account" "default" {
 }
 ```
+<div class = "oics-button" style="float: right; margin: 0 0 -15px">
+  <a href="https://console.cloud.google.com/cloudshell/open?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fterraform-google-modules%2Fdocs-examples.git&cloudshell_working_dir=data_fusion_instance_cmek&cloudshell_image=gcr.io%2Fgraphite-cloud-shell-images%2Fterraform%3Alatest&open_in_editor=main.tf&cloudshell_print=.%2Fmotd&cloudshell_tutorial=.%2Ftutorial.md" target="_blank">
+    <img alt="Open in Cloud Shell" src="//gstatic.com/cloudssh/images/open-btn.svg" style="max-height: 44px; margin: 32px auto; max-width: 100%;">
+  </a>
+</div>
+## Example Usage - Data Fusion Instance Cmek
+
+
+```hcl
+resource "google_data_fusion_instance" "basic_cmek" {
+  name   = "my-instance"
+  region = "us-central1"
+  type   = "BASIC"
+
+  crypto_key_config {
+    key_reference = google_kms_crypto_key.crypto_key.id
+  }
+
+  depends_on = [google_kms_crypto_key_iam_binding.crypto_key_binding]
+}
+
+resource "google_kms_crypto_key" "crypto_key" {
+  name     = "my-instance"
+  key_ring = google_kms_key_ring.key_ring.id
+}
+
+resource "google_kms_key_ring" "key_ring" {
+  name     = "my-instance"
+  location = "us-central1"
+}
+
+resource "google_kms_crypto_key_iam_binding" "crypto_key_binding" {
+  crypto_key_id = google_kms_crypto_key.crypto_key.id
+  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+
+  members = [
+    "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datafusion.iam.gserviceaccount.com"
+  ]
+}
+
+data "google_project" "project" {}
+```
 
 ## Argument Reference
 
@@ -151,6 +193,11 @@ The following arguments are supported:
   Network configuration options. These are required when a private Data Fusion instance is to be created.
   Structure is [documented below](#nested_network_config).
 
+* `crypto_key_config` -
+  (Optional)
+  The crypto key configuration. This field is used by the Customer-Managed Encryption Keys (CMEK) feature.
+  Structure is [documented below](#nested_crypto_key_config).
+
 * `region` -
   (Optional)
   The region of the Data Fusion instance.
@@ -172,6 +219,12 @@ The following arguments are supported:
   will be peered for executing pipelines. In case of shared VPC where the network resides in another host
   project the network should specified in the form of projects/{host-project-id}/global/networks/{network}
 
+<a name="nested_crypto_key_config"></a>The `crypto_key_config` block supports:
+
+* `key_reference` -
+  (Required)
+  The name of the key which is used to encrypt/decrypt customer data. For key in Cloud KMS, the key should be in the format of projects/*/locations/*/keyRings/*/cryptoKeys/*.
+
 ## Attributes Reference
 
 In addition to the arguments listed above, the following computed attributes are exported: