feat: Added cloud sql restore module that uses database import (#343)

adding cloud sql restore module that uses database import

modules/restore/README.md

@@ -0,0 +1,52 @@
+# GCP CloudSQL Restore
+
+## Import from GCS Export Dump
+
+This module can be used for [importing Cloud SQL Postgres database](https://cloud.google.com/sql/docs/postgres/import-export/import-export-sql) from a SQL export dump stored in GCS bucket.
+
+This module uses the SQL export dump file timestamp passed as an input parameter to the Workflow to get the exported dumps from GCS. Following are the steps in import workflow:
+
+1. Fetch list of databases from the source database instance (one that the export was created for)
+2. Delete the databases (list from step 1) except system (`postgres` for Postgres and `tempdb` for SQL Server) databases in the database instance that we are going to import databases to
+3. Create the databases (list from step 1) except system databases in the import database instance
+4. Fetch the SQL export file(s) from GCS and import those into the import database instance
+5. The import API call is asynchronous, so the workflow checks the status of the import at regular interval and wait until it finishes
+
+## How to run
+
+```
+gcloud workflows run [WORKFLOW_NAME] --data='{"exportTimestamp":"[EXPORT_TIMESTAMP]"}'
+```
+
+where `WORKFLOW_NAME` is the name of your import workflow and `exportTimestamp` is the timestamp of your export file(s) (you can get it from GCS object key of the export file). For example:
+
+```
+gcloud workflows run my-import-workflow --data='{"exportTimestamp": "1658779617"}'
+```
+
+## Required APIs
+
+- `workflows.googleapis.com`
+- `cloudscheduler.googleapis.com`
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| import\_databases | The list of databases that should be imported - if is an empty set all databases will be imported | `set(string)` | `[]` | no |
+| import\_uri | The bucket and path uri of GCS backup file for importing | `string` | n/a | yes |
+| project\_id | The project ID | `string` | n/a | yes |
+| region | The region to run the workflow | `string` | `"us-central1"` | no |
+| service\_account | The service account to use for running the workflow and triggering the workflow by Cloud Scheduler - If empty or null a service account will be created. If you have provided a service account you need to grant the Cloud SQL Admin and the Workflows Invoker role to that | `string` | `null` | no |
+| sql\_instance | The name of the SQL instance to backup | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| import\_workflow\_name | The name for import workflow |
+| region | n/a |
+| service\_account | The service account email running the scheduler and workflow |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/restore/main.tf

@@ -0,0 +1,80 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+locals {
+  create_service_account = var.service_account == null || var.service_account == "" ? true : false
+  service_account        = local.create_service_account ? google_service_account.sql_import_serviceaccount[0].email : var.service_account
+}
+
+
+################################
+#                              #
+#   Service Account and IAM    #
+#                              #
+################################
+resource "google_service_account" "sql_import_serviceaccount" {
+  count        = local.create_service_account ? 1 : 0
+  account_id   = trimsuffix(substr("import-${var.sql_instance}", 0, 28), "-")
+  display_name = "Managed by Terraform - Service account for import of SQL Instance ${var.sql_instance}"
+  project      = var.project_id
+}
+
+resource "google_project_iam_member" "sql_import_serviceaccount_sql_admin" {
+  count   = local.create_service_account ? 1 : 0
+  member  = "serviceAccount:${google_service_account.sql_import_serviceaccount[0].email}"
+  role    = "roles/cloudsql.admin"
+  project = var.project_id
+}
+
+resource "google_project_iam_member" "sql_import_serviceaccount_workflow_invoker" {
+  count   = local.create_service_account ? 1 : 0
+  member  = "serviceAccount:${google_service_account.sql_import_serviceaccount[0].email}"
+  role    = "roles/workflows.invoker"
+  project = var.project_id
+}
+
+data "google_sql_database_instance" "import_instance" {
+  name    = var.sql_instance
+  project = var.project_id
+}
+
+################################
+#                              #
+#       Import Workflow        #
+#                              #
+################################
+resource "google_workflows_workflow" "sql_import" {
+  name            = "sql-import-${var.sql_instance}"
+  region          = var.region
+  description     = "Workflow for importing the CloudSQL Instance database using an external import"
+  project         = var.project_id
+  service_account = local.service_account
+  source_contents = templatefile("${path.module}/templates/import.yaml.tftpl", {
+    project          = var.project_id
+    instanceName     = var.sql_instance
+    databases        = jsonencode(var.import_databases)
+    gcsBucket        = var.import_uri
+    exportedInstance = split("/", var.import_uri)[3]
+    dbType           = split("_", data.google_sql_database_instance.import_instance.database_version)[0]
+  })
+}
+
+resource "google_storage_bucket_iam_member" "sql_instance_account" {
+  bucket = split("/", var.import_uri)[2] #Get the name of the bucket out of the URI
+  member = "serviceAccount:${data.google_sql_database_instance.import_instance.service_account_email_address}"
+  role   = "roles/storage.objectViewer"
+}

modules/restore/outputs.tf

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "import_workflow_name" {
+  value       = google_workflows_workflow.sql_import.name
+  description = "The name for import workflow"
+}
+
+output "service_account" {
+  value       = local.service_account
+  description = "The service account email running the scheduler and workflow"
+}
+
+output "region" {
+  value = var.region
+}

modules/restore/templates/import.yaml.tftpl

@@ -0,0 +1,181 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+main:
+  params: [args]
+  steps:
+  - collectInfos:
+      assign:
+        - databases: ${databases}
+%{ if databases == "[]" }
+  - list of databases:
+      call: googleapis.sqladmin.v1.databases.list
+      args:
+        project: ${project}
+        instance: ${exportedInstance}
+      result: dbListResult
+  - collect DB list:
+      for:
+        value: db
+        in: $${dbListResult.items}
+        steps:
+          - iterate:
+              assign:
+                - databases: $${list.concat(databases, db.name)}
+
+%{ endif }
+
+%{ if dbType == "POSTGRES" }
+  - import databases:
+      for:
+        value: database
+        in: $${databases}
+        steps:
+          - exclude postgres DB:
+              switch:
+                - condition: $${database != "postgres"}
+                  steps:
+                    - delete database:
+                        call: googleapis.sqladmin.v1.databases.delete
+                        args:
+                          database: $${database}
+                          instance: ${instanceName}
+                          project: ${project}
+                    - create database:
+                        call: googleapis.sqladmin.v1.databases.insert
+                        args:
+                          instance: ${instanceName}
+                          project: ${project}
+                          body:
+                            name: $${database}
+                    - import database:
+                        call: http.post
+                        args:
+                          url: $${"https://sqladmin.googleapis.com/v1/projects/" + "${project}" + "/instances/" + "${instanceName}" + "/import"}
+                          auth:
+                            type: OAuth2
+                          body:
+                            importContext:
+                              uri: $${"${gcsBucket}/${exportedInstance}-" + database + "-" + args.exportTimestamp + ".sql"}
+                              database: $${database}
+                              fileType: SQL
+                        result: importstatus
+                    - checkstatus:
+                        switch:
+                          - condition: $${importstatus.body.status != "DONE"}
+                            next: wait
+                        next: continue
+                    - wait:
+                        call: sys.sleep
+                        args:
+                          seconds: 1
+                        next: getstatus
+                    - getstatus:
+                        call: http.get
+                        args:
+                          url: $${importstatus.body.selfLink}
+                          auth:
+                            type: OAuth2
+                        result: importstatus
+                        next: checkstatus
+%{ endif }
+
+%{ if dbType == "SQLSERVER" }
+  - import databases:
+      for:
+        value: database
+        in: $${databases}
+        steps:
+          - exclude System DB:
+              switch:
+                - condition: $${database != "tempdb"}
+                  steps:
+                    - delete database:
+                        call: googleapis.sqladmin.v1.databases.delete
+                        args:
+                          database: $${database}
+                          instance: ${instanceName}
+                          project: ${project}
+                    - create database:
+                        call: googleapis.sqladmin.v1.databases.insert
+                        args:
+                          instance: ${instanceName}
+                          project: ${project}
+                          body:
+                            name: $${database}
+                    - import database:
+                        call: http.post
+                        args:
+                          url: $${"https://sqladmin.googleapis.com/v1/projects/" + "${project}" + "/instances/" + "${instanceName}" + "/import"}
+                          auth:
+                            type: OAuth2
+                          body:
+                            importContext:
+                              uri: $${"${gcsBucket}/${exportedInstance}-" + database + "-" + args.exportTimestamp + ".bak"}
+                              database: $${database}
+                              fileType: BAK
+                        result: importstatus
+                    - checkstatus:
+                        switch:
+                          - condition: $${importstatus.body.status != "DONE"}
+                            next: wait
+                        next: continue
+                    - wait:
+                        call: sys.sleep
+                        args:
+                          seconds: 1
+                        next: getstatus
+                    - getstatus:
+                        call: http.get
+                        args:
+                          url: $${importstatus.body.selfLink}
+                          auth:
+                            type: OAuth2
+                        result: importstatus
+                        next: checkstatus
+%{ endif }
+
+%{ if dbType == "MYSQL" }
+  - import database:
+      call: http.post
+      args:
+        url: $${"https://sqladmin.googleapis.com/v1/projects/" + "${project}" + "/instances/" + "${instanceName}" + "/import"}
+        auth:
+          type: OAuth2
+        body:
+          importContext:
+            uri: $${"${gcsBucket}/${exportedInstance}-" + args.exportTimestamp + ".sql"}
+            fileType: SQL
+      result: importstatus
+  - checkstatus:
+      switch:
+        - condition: $${importstatus.body.status != "DONE"}
+          next: wait
+      next: completed
+  - wait:
+      call: sys.sleep
+      args:
+        seconds: 1
+      next: getstatus
+  - getstatus:
+      call: http.get
+      args:
+        url: $${importstatus.body.selfLink}
+        auth:
+          type: OAuth2
+      result: importstatus
+      next: checkstatus
+  - completed:
+      return: "Done"
+%{ endif }