Add daisy wf and replacement mechanism to invoke preloading scripts (#…

…2309)

* Update projects and permissions

* Change project to gcp-guest and add comments

* Update install_package_cos.wf.json- update to deb 11 worker (10 is EOL soon)

daisy_workflows/image_build/install_package/cos/install_package_cos.wf.json

@@ -0,0 +1,99 @@
+{
+  "Name": "image-replace-package",
+  "Zone": "us-central1-a",
+  "DefaultTimeout": "30m",
+  "Vars": {
+    "source_image": {
+      "Required": true,
+      "Description": "source image"
+    },
+    "dest_image": {
+      "Required": true,
+      "Description": "dest image"
+    },
+    "package_version": {
+      "Required": true,
+      "Description": "package version"
+    },
+    "cos_branch": {
+      "Required": true,
+      "Description": "cos_branch"
+    },
+    "worker_image": {
+      "Value": "projects/compute-image-tools/global/images/family/debian-11-worker",
+      "Description": "worker image"
+    },
+    "machine_type": {
+      "Value": "e2-medium",
+      "Description": "machine type"
+    }
+  },
+  "Sources": {
+    "work/utils": "../../../linux_common/utils",
+    "work/package_replacement": "package_replacement",
+    "work/replacepackage.sh": "replacepackage.sh",
+    "startup_script": "../../../linux_common/bootstrap.sh"
+  },
+  "Steps": {
+    "create-disks": {
+      "CreateDisks": [
+        {
+          "Name": "disk-worker",
+          "SourceImage": "${worker_image}",
+          "Type": "pd-ssd"
+	}
+      ]
+    },
+    "create-inst": {
+      "CreateInstances": [
+        {
+          "Name": "inst-worker",
+          "Disks": [{"Source": "disk-worker"}],
+          "MachineType": "${machine_type}",
+          "Metadata": {
+	    "files_gcs_dir": "${SOURCESPATH}/work",
+            "prefix": "ReplacePackage",
+            "script": "replacepackage.sh",
+	    "source_image": "${source_image}",
+	    "dest_image": "${dest_image}",
+	    "cos_branch": "${cos_branch}",
+	    "package_version": "${package_version}"
+          },
+          "StartupScript": "startup_script",
+	  "Scopes": [
+	    "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/devstorage.read_write",
+	    "https://www.googleapis.com/auth/compute",
+	    "https://www.googleapis.com/auth/servicecontrol"
+          ]
+        }
+      ]
+    },
+    "wait": {
+      "WaitForInstancesSignal": [
+        {
+          "Name": "inst-worker",
+          "Timeout": "15m",
+          "SerialOutput": {
+            "Port": 1,
+            "FailureMatch": "ReplacePackageFailed:",
+            "SuccessMatch": "ReplacePackageSuccess:",
+            "StatusMatch": "ReplacePackageStatus:"
+          }
+        }
+      ]
+    },
+    "delete-inst": {
+      "DeleteResources": {
+        "Instances": ["inst-worker"],
+        "Disks":["disk-worker"]
+      }
+    }
+
+  },
+  "Dependencies": {
+    "create-inst": ["create-disks"],
+    "wait": ["create-inst"],
+    "delete-inst": ["wait"]
+  }
+}

daisy_workflows/image_build/install_package/cos/package_replacement/dev_cloudbuild.yaml

@@ -23,11 +23,13 @@ steps:
     - '-c'
     - |
       ./compile_debian_package.sh $_OVERLAYS_BRANCH $_GUEST_AGENT_VERSION
+# _DEST_PROJECT in the 'finish-image-build' step should should have access
+# to the gcs-bucket path specified here.
 - name: '${_COS_CUSTOMIZER}'
   args: ['start-image-build',
-         '-image-name=${_BASE_IMAGE}',
+         '-image-family=${_BASE_IMAGE}',
          '-image-project=${_BASE_IMAGE_PROJECT}',
-         '-gcs-bucket=${_CLOUD_BUILD_BUCKET}',
+         '-gcs-bucket=${_DEST_PROJECT}_cloudbuild',
          '-gcs-workdir=customizer-$BUILD_ID']
 - name: '${_COS_CUSTOMIZER}'
   args: ["disable-auto-update"]

daisy_workflows/image_build/install_package/cos/replacepackage.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# This script is responsible for invoking the preloading mechanism that creates
+# a COS image with the new package version that is used for testing.
+#   1) The script sets all the preloading scripts as executable (located in
+#   /cos/package_replacement).
+#   2) Then, collects all passed in data through metadata API (name of source
+#   image, dest image, package version, cos branch).
+#     - cos_branch: the name of the branch associated with a COS milestone (ie
+#     for cos-113, cos_branch=release-R113).
+#   3) Runs a gcloud submit command to invoke the cloudbuild responsible for
+#   preloading.
+
+set -o errexit
+set -o pipefail
+set -x
+
+set_files_executable(){
+  cd /files/package_replacement
+
+  search_dir=.
+  for file in "$search_dir"/*
+  do
+    chmod +x $file
+  done
+}
+
+get_vars(){
+  export SOURCE_IMAGE=$(curl -H "Metadata-Flavor:Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/source_image)
+  export DEST_IMAGE=$(curl -H "Metadata-Flavor:Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/dest_image)
+  export PACKAGE_VERSION=$(curl -H "Metadata-Flavor:Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/package_version)
+  export COS_BRANCH=$(curl -H "Metadata-Flavor:Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/cos_branch)
+}
+
+# The following additional parameters are passed into the cloudbuild script:
+#   _BASE_IMAGE_PROJECT: The project that contains the base/source image.
+#   _BASE_IMAGE: The name of the base/source image.
+#   _DEST_PROJECT: The project that contains the resulting preloaded image.
+#   _NEW_IMAGE: The name of the resulting preloaded image.
+#   _NEW_IMAGE_FAMILY: The new image family for the preloaded image.
+create_preloaded_image(){
+  gcloud builds submit --config=dev_cloudbuild.yaml --disk-size=200 . --substitutions=_NEW_IMAGE_FAMILY="cos-preloaded-images",_BASE_IMAGE_PROJECT="cos-cloud",_BASE_IMAGE="${SOURCE_IMAGE}",_OVERLAYS_BRANCH="${COS_BRANCH}",_GUEST_AGENT_VERSION="${PACKAGE_VERSION}",_NEW_IMAGE="${DEST_IMAGE}",_DEST_PROJECT="gcp-guest"
+}
+
+main (){
+  echo "Creating COS image with the new guest agent version..."
+  set_files_executable
+  get_vars
+  create_preloaded_image
+}
+
+main