From aebd2c325bb6ca808109c697314dd6e60f588429 Mon Sep 17 00:00:00 2001 From: Modular Magician Date: Tue, 8 Nov 2022 20:14:01 +0000 Subject: [PATCH] Add extra BigQuery options to DLP inspect job trigger (#6749) Signed-off-by: Modular Magician --- .changelog/6749.txt | 3 + ...source_data_loss_prevention_job_trigger.go | 99 +++++++++++++ ...s_prevention_job_trigger_generated_test.go | 140 +++++++++++++++++- ..._loss_prevention_job_trigger.html.markdown | 107 ++++++++++++- 4 files changed, 345 insertions(+), 4 deletions(-) create mode 100644 .changelog/6749.txt diff --git a/.changelog/6749.txt b/.changelog/6749.txt new file mode 100644 index 0000000000..a4f0f9fcae --- /dev/null +++ b/.changelog/6749.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +dlp: added fields `rows_limit`, `rows_limit_percent`, and `sample_method` to `big_query_options` in `google_data_loss_prevention_job_trigger` +``` diff --git a/google/resource_data_loss_prevention_job_trigger.go b/google/resource_data_loss_prevention_job_trigger.go index a74336c61e..bb0324f601 100644 --- a/google/resource_data_loss_prevention_job_trigger.go +++ b/google/resource_data_loss_prevention_job_trigger.go @@ -225,6 +225,28 @@ Only for use with external storage. Possible values: ["BASIC_COLUMNS", "GCS_COLU }, }, }, + "rows_limit": { + Type: schema.TypeInt, + Optional: true, + Description: `Max number of rows to scan. If the table has more rows than this value, the rest of the rows are omitted. +If not set, or if set to 0, all rows will be scanned. Only one of rowsLimit and rowsLimitPercent can be +specified. Cannot be used in conjunction with TimespanConfig.`, + }, + "rows_limit_percent": { + Type: schema.TypeInt, + Optional: true, + Description: `Max percentage of rows to scan. The rest are omitted. The number of rows scanned is rounded down. +Must be between 0 and 100, inclusively. Both 0 and 100 means no limit. Defaults to 0. Only one of +rowsLimit and rowsLimitPercent can be specified. Cannot be used in conjunction with TimespanConfig.`, + }, + "sample_method": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validateEnum([]string{"TOP", "RANDOM_START", ""}), + Description: `How to sample rows if not all rows are scanned. Meaningful only when used in conjunction with either +rowsLimit or rowsLimitPercent. If not specified, rows are scanned in the order BigQuery reads them. Default value: "TOP" Possible values: ["TOP", "RANDOM_START"]`, + Default: "TOP", + }, }, }, }, @@ -1086,6 +1108,12 @@ func flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptions(v transformed := make(map[string]interface{}) transformed["table_reference"] = flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsTableReference(original["tableReference"], d, config) + transformed["rows_limit"] = + flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimit(original["rowsLimit"], d, config) + transformed["rows_limit_percent"] = + flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimitPercent(original["rowsLimitPercent"], d, config) + transformed["sample_method"] = + flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsSampleMethod(original["sampleMethod"], d, config) return []interface{}{transformed} } func flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsTableReference(v interface{}, d *schema.ResourceData, config *Config) interface{} { @@ -1117,6 +1145,44 @@ func flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsTa return v } +func flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimit(v interface{}, d *schema.ResourceData, config *Config) interface{} { + // Handles the string fixed64 format + if strVal, ok := v.(string); ok { + if intVal, err := stringToFixed64(strVal); err == nil { + return intVal + } + } + + // number values are represented as float64 + if floatVal, ok := v.(float64); ok { + intVal := int(floatVal) + return intVal + } + + return v // let terraform core handle it otherwise +} + +func flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimitPercent(v interface{}, d *schema.ResourceData, config *Config) interface{} { + // Handles the string fixed64 format + if strVal, ok := v.(string); ok { + if intVal, err := stringToFixed64(strVal); err == nil { + return intVal + } + } + + // number values are represented as float64 + if floatVal, ok := v.(float64); ok { + intVal := int(floatVal) + return intVal + } + + return v // let terraform core handle it otherwise +} + +func flattenDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsSampleMethod(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + func flattenDataLossPreventionJobTriggerInspectJobActions(v interface{}, d *schema.ResourceData, config *Config) interface{} { if v == nil { return v @@ -1671,6 +1737,27 @@ func expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptions(v transformed["tableReference"] = transformedTableReference } + transformedRowsLimit, err := expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimit(original["rows_limit"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedRowsLimit); val.IsValid() && !isEmptyValue(val) { + transformed["rowsLimit"] = transformedRowsLimit + } + + transformedRowsLimitPercent, err := expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimitPercent(original["rows_limit_percent"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedRowsLimitPercent); val.IsValid() && !isEmptyValue(val) { + transformed["rowsLimitPercent"] = transformedRowsLimitPercent + } + + transformedSampleMethod, err := expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsSampleMethod(original["sample_method"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedSampleMethod); val.IsValid() && !isEmptyValue(val) { + transformed["sampleMethod"] = transformedSampleMethod + } + return transformed, nil } @@ -1719,6 +1806,18 @@ func expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsTab return v, nil } +func expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimit(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsRowsLimitPercent(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandDataLossPreventionJobTriggerInspectJobStorageConfigBigQueryOptionsSampleMethod(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func expandDataLossPreventionJobTriggerInspectJobActions(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { l := v.([]interface{}) req := make([]interface{}, 0, len(l)) diff --git a/google/resource_data_loss_prevention_job_trigger_generated_test.go b/google/resource_data_loss_prevention_job_trigger_generated_test.go index 2a8acbcb0b..3077a07a1f 100644 --- a/google/resource_data_loss_prevention_job_trigger_generated_test.go +++ b/google/resource_data_loss_prevention_job_trigger_generated_test.go @@ -68,8 +68,8 @@ resource "google_data_loss_prevention_job_trigger" "basic" { save_findings { output_config { table { - project_id = "asdf" - dataset_id = "asdf" + project_id = "project" + dataset_id = "dataset" } } } @@ -86,6 +86,142 @@ resource "google_data_loss_prevention_job_trigger" "basic" { `, context) } +func TestAccDataLossPreventionJobTrigger_dlpJobTriggerBigqueryRowLimitExample(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "project": getTestProjectFromEnv(), + "random_suffix": randString(t, 10), + } + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataLossPreventionJobTriggerDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataLossPreventionJobTrigger_dlpJobTriggerBigqueryRowLimitExample(context), + }, + { + ResourceName: "google_data_loss_prevention_job_trigger.bigquery_row_limit", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"parent"}, + }, + }, + }) +} + +func testAccDataLossPreventionJobTrigger_dlpJobTriggerBigqueryRowLimitExample(context map[string]interface{}) string { + return Nprintf(` +resource "google_data_loss_prevention_job_trigger" "bigquery_row_limit" { + parent = "projects/%{project}" + description = "Description" + display_name = "Displayname" + + triggers { + schedule { + recurrence_period_duration = "86400s" + } + } + + inspect_job { + inspect_template_name = "fake" + actions { + save_findings { + output_config { + table { + project_id = "project" + dataset_id = "dataset" + } + } + } + } + storage_config { + big_query_options { + table_reference { + project_id = "project" + dataset_id = "dataset" + table_id = "table_to_scan" + } + + rows_limit = 1000 + sample_method = "RANDOM_START" + } + } + } +} +`, context) +} + +func TestAccDataLossPreventionJobTrigger_dlpJobTriggerBigqueryRowLimitPercentageExample(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "project": getTestProjectFromEnv(), + "random_suffix": randString(t, 10), + } + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataLossPreventionJobTriggerDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataLossPreventionJobTrigger_dlpJobTriggerBigqueryRowLimitPercentageExample(context), + }, + { + ResourceName: "google_data_loss_prevention_job_trigger.bigquery_row_limit_percentage", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"parent"}, + }, + }, + }) +} + +func testAccDataLossPreventionJobTrigger_dlpJobTriggerBigqueryRowLimitPercentageExample(context map[string]interface{}) string { + return Nprintf(` +resource "google_data_loss_prevention_job_trigger" "bigquery_row_limit_percentage" { + parent = "projects/%{project}" + description = "Description" + display_name = "Displayname" + + triggers { + schedule { + recurrence_period_duration = "86400s" + } + } + + inspect_job { + inspect_template_name = "fake" + actions { + save_findings { + output_config { + table { + project_id = "project" + dataset_id = "dataset" + } + } + } + } + storage_config { + big_query_options { + table_reference { + project_id = "project" + dataset_id = "dataset" + table_id = "table_to_scan" + } + + rows_limit_percent = 50 + sample_method = "RANDOM_START" + } + } + } +} +`, context) +} + func testAccCheckDataLossPreventionJobTriggerDestroyProducer(t *testing.T) func(s *terraform.State) error { return func(s *terraform.State) error { for name, rs := range s.RootModule().Resources { diff --git a/website/docs/r/data_loss_prevention_job_trigger.html.markdown b/website/docs/r/data_loss_prevention_job_trigger.html.markdown index 3c963faef8..02322c2b16 100644 --- a/website/docs/r/data_loss_prevention_job_trigger.html.markdown +++ b/website/docs/r/data_loss_prevention_job_trigger.html.markdown @@ -50,8 +50,8 @@ resource "google_data_loss_prevention_job_trigger" "basic" { save_findings { output_config { table { - project_id = "asdf" - dataset_id = "asdf" + project_id = "project" + dataset_id = "dataset" } } } @@ -66,6 +66,90 @@ resource "google_data_loss_prevention_job_trigger" "basic" { } } ``` +## Example Usage - Dlp Job Trigger Bigquery Row Limit + + +```hcl +resource "google_data_loss_prevention_job_trigger" "bigquery_row_limit" { + parent = "projects/my-project-name" + description = "Description" + display_name = "Displayname" + + triggers { + schedule { + recurrence_period_duration = "86400s" + } + } + + inspect_job { + inspect_template_name = "fake" + actions { + save_findings { + output_config { + table { + project_id = "project" + dataset_id = "dataset" + } + } + } + } + storage_config { + big_query_options { + table_reference { + project_id = "project" + dataset_id = "dataset" + table_id = "table_to_scan" + } + + rows_limit = 1000 + sample_method = "RANDOM_START" + } + } + } +} +``` +## Example Usage - Dlp Job Trigger Bigquery Row Limit Percentage + + +```hcl +resource "google_data_loss_prevention_job_trigger" "bigquery_row_limit_percentage" { + parent = "projects/my-project-name" + description = "Description" + display_name = "Displayname" + + triggers { + schedule { + recurrence_period_duration = "86400s" + } + } + + inspect_job { + inspect_template_name = "fake" + actions { + save_findings { + output_config { + table { + project_id = "project" + dataset_id = "dataset" + } + } + } + } + storage_config { + big_query_options { + table_reference { + project_id = "project" + dataset_id = "dataset" + table_id = "table_to_scan" + } + + rows_limit_percent = 50 + sample_method = "RANDOM_START" + } + } + } +} +``` ## Argument Reference @@ -303,6 +387,25 @@ The following arguments are supported: Set of files to scan. Structure is [documented below](#nested_table_reference). +* `rows_limit` - + (Optional) + Max number of rows to scan. If the table has more rows than this value, the rest of the rows are omitted. + If not set, or if set to 0, all rows will be scanned. Only one of rowsLimit and rowsLimitPercent can be + specified. Cannot be used in conjunction with TimespanConfig. + +* `rows_limit_percent` - + (Optional) + Max percentage of rows to scan. The rest are omitted. The number of rows scanned is rounded down. + Must be between 0 and 100, inclusively. Both 0 and 100 means no limit. Defaults to 0. Only one of + rowsLimit and rowsLimitPercent can be specified. Cannot be used in conjunction with TimespanConfig. + +* `sample_method` - + (Optional) + How to sample rows if not all rows are scanned. Meaningful only when used in conjunction with either + rowsLimit or rowsLimitPercent. If not specified, rows are scanned in the order BigQuery reads them. + Default value is `TOP`. + Possible values are `TOP` and `RANDOM_START`. + The `table_reference` block supports: