From da4033b0b2e12d66bbc3869d2be32096f2a685b8 Mon Sep 17 00:00:00 2001 From: Ravi Dalal <12639199+ravi-dalal@users.noreply.github.com> Date: Fri, 7 Oct 2022 13:25:44 -0400 Subject: [PATCH] feat: Added cloud sql restore module that uses database import (#343) adding cloud sql restore module that uses database import --- modules/restore/README.md | 52 ++++++ modules/restore/main.tf | 80 +++++++++ modules/restore/outputs.tf | 29 ++++ modules/restore/templates/import.yaml.tftpl | 181 ++++++++++++++++++++ modules/restore/variables.tf | 56 ++++++ modules/restore/versions.tf | 25 +++ 6 files changed, 423 insertions(+) create mode 100644 modules/restore/README.md create mode 100644 modules/restore/main.tf create mode 100644 modules/restore/outputs.tf create mode 100644 modules/restore/templates/import.yaml.tftpl create mode 100644 modules/restore/variables.tf create mode 100644 modules/restore/versions.tf diff --git a/modules/restore/README.md b/modules/restore/README.md new file mode 100644 index 00000000..0659e9b2 --- /dev/null +++ b/modules/restore/README.md @@ -0,0 +1,52 @@ +# GCP CloudSQL Restore + +## Import from GCS Export Dump + +This module can be used for [importing Cloud SQL Postgres database](https://cloud.google.com/sql/docs/postgres/import-export/import-export-sql) from a SQL export dump stored in GCS bucket. + +This module uses the SQL export dump file timestamp passed as an input parameter to the Workflow to get the exported dumps from GCS. Following are the steps in import workflow: + +1. Fetch list of databases from the source database instance (one that the export was created for) +2. Delete the databases (list from step 1) except system (`postgres` for Postgres and `tempdb` for SQL Server) databases in the database instance that we are going to import databases to +3. Create the databases (list from step 1) except system databases in the import database instance +4. Fetch the SQL export file(s) from GCS and import those into the import database instance +5. The import API call is asynchronous, so the workflow checks the status of the import at regular interval and wait until it finishes + +## How to run + +``` +gcloud workflows run [WORKFLOW_NAME] --data='{"exportTimestamp":"[EXPORT_TIMESTAMP]"}' +``` + +where `WORKFLOW_NAME` is the name of your import workflow and `exportTimestamp` is the timestamp of your export file(s) (you can get it from GCS object key of the export file). For example: + +``` +gcloud workflows run my-import-workflow --data='{"exportTimestamp": "1658779617"}' +``` + +## Required APIs + +- `workflows.googleapis.com` +- `cloudscheduler.googleapis.com` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| import\_databases | The list of databases that should be imported - if is an empty set all databases will be imported | `set(string)` | `[]` | no | +| import\_uri | The bucket and path uri of GCS backup file for importing | `string` | n/a | yes | +| project\_id | The project ID | `string` | n/a | yes | +| region | The region to run the workflow | `string` | `"us-central1"` | no | +| service\_account | The service account to use for running the workflow and triggering the workflow by Cloud Scheduler - If empty or null a service account will be created. If you have provided a service account you need to grant the Cloud SQL Admin and the Workflows Invoker role to that | `string` | `null` | no | +| sql\_instance | The name of the SQL instance to backup | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| import\_workflow\_name | The name for import workflow | +| region | n/a | +| service\_account | The service account email running the scheduler and workflow | + + diff --git a/modules/restore/main.tf b/modules/restore/main.tf new file mode 100644 index 00000000..d73b58e5 --- /dev/null +++ b/modules/restore/main.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +locals { + create_service_account = var.service_account == null || var.service_account == "" ? true : false + service_account = local.create_service_account ? google_service_account.sql_import_serviceaccount[0].email : var.service_account +} + + +################################ +# # +# Service Account and IAM # +# # +################################ +resource "google_service_account" "sql_import_serviceaccount" { + count = local.create_service_account ? 1 : 0 + account_id = trimsuffix(substr("import-${var.sql_instance}", 0, 28), "-") + display_name = "Managed by Terraform - Service account for import of SQL Instance ${var.sql_instance}" + project = var.project_id +} + +resource "google_project_iam_member" "sql_import_serviceaccount_sql_admin" { + count = local.create_service_account ? 1 : 0 + member = "serviceAccount:${google_service_account.sql_import_serviceaccount[0].email}" + role = "roles/cloudsql.admin" + project = var.project_id +} + +resource "google_project_iam_member" "sql_import_serviceaccount_workflow_invoker" { + count = local.create_service_account ? 1 : 0 + member = "serviceAccount:${google_service_account.sql_import_serviceaccount[0].email}" + role = "roles/workflows.invoker" + project = var.project_id +} + +data "google_sql_database_instance" "import_instance" { + name = var.sql_instance + project = var.project_id +} + +################################ +# # +# Import Workflow # +# # +################################ +resource "google_workflows_workflow" "sql_import" { + name = "sql-import-${var.sql_instance}" + region = var.region + description = "Workflow for importing the CloudSQL Instance database using an external import" + project = var.project_id + service_account = local.service_account + source_contents = templatefile("${path.module}/templates/import.yaml.tftpl", { + project = var.project_id + instanceName = var.sql_instance + databases = jsonencode(var.import_databases) + gcsBucket = var.import_uri + exportedInstance = split("/", var.import_uri)[3] + dbType = split("_", data.google_sql_database_instance.import_instance.database_version)[0] + }) +} + +resource "google_storage_bucket_iam_member" "sql_instance_account" { + bucket = split("/", var.import_uri)[2] #Get the name of the bucket out of the URI + member = "serviceAccount:${data.google_sql_database_instance.import_instance.service_account_email_address}" + role = "roles/storage.objectViewer" +} diff --git a/modules/restore/outputs.tf b/modules/restore/outputs.tf new file mode 100644 index 00000000..b9ead5cd --- /dev/null +++ b/modules/restore/outputs.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "import_workflow_name" { + value = google_workflows_workflow.sql_import.name + description = "The name for import workflow" +} + +output "service_account" { + value = local.service_account + description = "The service account email running the scheduler and workflow" +} + +output "region" { + value = var.region +} diff --git a/modules/restore/templates/import.yaml.tftpl b/modules/restore/templates/import.yaml.tftpl new file mode 100644 index 00000000..a2d1ee18 --- /dev/null +++ b/modules/restore/templates/import.yaml.tftpl @@ -0,0 +1,181 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +main: + params: [args] + steps: + - collectInfos: + assign: + - databases: ${databases} +%{ if databases == "[]" } + - list of databases: + call: googleapis.sqladmin.v1.databases.list + args: + project: ${project} + instance: ${exportedInstance} + result: dbListResult + - collect DB list: + for: + value: db + in: $${dbListResult.items} + steps: + - iterate: + assign: + - databases: $${list.concat(databases, db.name)} + +%{ endif } + +%{ if dbType == "POSTGRES" } + - import databases: + for: + value: database + in: $${databases} + steps: + - exclude postgres DB: + switch: + - condition: $${database != "postgres"} + steps: + - delete database: + call: googleapis.sqladmin.v1.databases.delete + args: + database: $${database} + instance: ${instanceName} + project: ${project} + - create database: + call: googleapis.sqladmin.v1.databases.insert + args: + instance: ${instanceName} + project: ${project} + body: + name: $${database} + - import database: + call: http.post + args: + url: $${"https://sqladmin.googleapis.com/v1/projects/" + "${project}" + "/instances/" + "${instanceName}" + "/import"} + auth: + type: OAuth2 + body: + importContext: + uri: $${"${gcsBucket}/${exportedInstance}-" + database + "-" + args.exportTimestamp + ".sql"} + database: $${database} + fileType: SQL + result: importstatus + - checkstatus: + switch: + - condition: $${importstatus.body.status != "DONE"} + next: wait + next: continue + - wait: + call: sys.sleep + args: + seconds: 1 + next: getstatus + - getstatus: + call: http.get + args: + url: $${importstatus.body.selfLink} + auth: + type: OAuth2 + result: importstatus + next: checkstatus +%{ endif } + +%{ if dbType == "SQLSERVER" } + - import databases: + for: + value: database + in: $${databases} + steps: + - exclude System DB: + switch: + - condition: $${database != "tempdb"} + steps: + - delete database: + call: googleapis.sqladmin.v1.databases.delete + args: + database: $${database} + instance: ${instanceName} + project: ${project} + - create database: + call: googleapis.sqladmin.v1.databases.insert + args: + instance: ${instanceName} + project: ${project} + body: + name: $${database} + - import database: + call: http.post + args: + url: $${"https://sqladmin.googleapis.com/v1/projects/" + "${project}" + "/instances/" + "${instanceName}" + "/import"} + auth: + type: OAuth2 + body: + importContext: + uri: $${"${gcsBucket}/${exportedInstance}-" + database + "-" + args.exportTimestamp + ".bak"} + database: $${database} + fileType: BAK + result: importstatus + - checkstatus: + switch: + - condition: $${importstatus.body.status != "DONE"} + next: wait + next: continue + - wait: + call: sys.sleep + args: + seconds: 1 + next: getstatus + - getstatus: + call: http.get + args: + url: $${importstatus.body.selfLink} + auth: + type: OAuth2 + result: importstatus + next: checkstatus +%{ endif } + +%{ if dbType == "MYSQL" } + - import database: + call: http.post + args: + url: $${"https://sqladmin.googleapis.com/v1/projects/" + "${project}" + "/instances/" + "${instanceName}" + "/import"} + auth: + type: OAuth2 + body: + importContext: + uri: $${"${gcsBucket}/${exportedInstance}-" + args.exportTimestamp + ".sql"} + fileType: SQL + result: importstatus + - checkstatus: + switch: + - condition: $${importstatus.body.status != "DONE"} + next: wait + next: completed + - wait: + call: sys.sleep + args: + seconds: 1 + next: getstatus + - getstatus: + call: http.get + args: + url: $${importstatus.body.selfLink} + auth: + type: OAuth2 + result: importstatus + next: checkstatus + - completed: + return: "Done" +%{ endif } diff --git a/modules/restore/variables.tf b/modules/restore/variables.tf new file mode 100644 index 00000000..c294732f --- /dev/null +++ b/modules/restore/variables.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "region" { + description = "The region to run the workflow" + type = string + default = "us-central1" +} + +variable "service_account" { + description = "The service account to use for running the workflow and triggering the workflow by Cloud Scheduler - If empty or null a service account will be created. If you have provided a service account you need to grant the Cloud SQL Admin and the Workflows Invoker role to that" + type = string + default = null +} + +variable "project_id" { + description = "The project ID" + type = string +} + +variable "sql_instance" { + description = "The name of the SQL instance to backup" + type = string +} + +variable "import_databases" { + description = "The list of databases that should be imported - if is an empty set all databases will be imported" + type = set(string) + default = [] + validation { + condition = var.import_databases != null + error_message = "Must not be null." + } +} + +variable "import_uri" { + description = "The bucket and path uri of GCS backup file for importing" + type = string + validation { + condition = can(regex("^gs:\\/\\/", var.import_uri)) + error_message = "Must be a full GCS URI starting with gs://." #TODO: test + } +} diff --git a/modules/restore/versions.tf b/modules/restore/versions.tf new file mode 100644 index 00000000..6ae92224 --- /dev/null +++ b/modules/restore/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.0.0, < 5.0" + } + } +}