From cced94fba75ebb21eb525a80f7439daf3dd54f1a Mon Sep 17 00:00:00 2001
From: Blaine Gardner <blaine.gardner@redhat.com>
Date: Wed, 6 Oct 2021 13:38:02 -0600
Subject: [PATCH] test: run RGW multisite test in nightly job

In the nightly job, run the test with the latest Ceph version so we can
detect if there are RGW changes in Ceph that might break multisite. Use
a reusable GitHub action workflow to duplicate as little code as
possible.

Signed-off-by: Blaine Gardner <blaine.gardner@redhat.com>
---
 .github/actions/rgw-multisite-test/action.yml | 104 +++++++++++++++
 .github/workflows/canary-integration-test.yml | 110 +++-------------
 .github/workflows/z_rgw-multisite.yml         | 120 ++++++++++++++++++
 tests/scripts/github-action-helper.sh         |  10 ++
 4 files changed, 253 insertions(+), 91 deletions(-)
 create mode 100644 .github/actions/rgw-multisite-test/action.yml
 create mode 100644 .github/workflows/z_rgw-multisite.yml

diff --git a/.github/actions/rgw-multisite-test/action.yml b/.github/actions/rgw-multisite-test/action.yml
new file mode 100644
index 0000000000000..caefaf52a9aed
--- /dev/null
+++ b/.github/actions/rgw-multisite-test/action.yml
@@ -0,0 +1,104 @@
+name: RGW Multisite Test
+description: Reusable workflow to test RGW multisite integration
+inputs:
+  github-token:
+    description: GITHUB_TOKEN from the calling workflow
+    required: true
+  ceph-image:
+    description: Ceph image to use for the workflow (e.g., quay.io/ceph/ceph:v16.2.5)
+    required: false
+    type: string
+
+defaults:
+  run:
+    # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+runs:
+  using: "composite"
+  steps:
+    - name: setup golang
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+
+    - name: install deps
+      run: |
+        tests/scripts/github-action-helper.sh install_deps
+        sudo apt-get install -y s3cmd
+
+    - name: setup minikube
+      uses: manusa/actions-setup-minikube@v2.3.1
+      with:
+        minikube version: 'v1.18.1'
+        kubernetes version: 'v1.19.2'
+        start args: --memory 6g --cpus=2
+        github token: ${{ inputs.github-token }}
+
+    - name: use local disk into two partitions
+      run: |
+        BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
+        tests/scripts/github-action-helper.sh use_local_disk
+        tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
+        sudo lsblk
+
+    - name: build rook
+      run: tests/scripts/github-action-helper.sh build_rook
+
+    - name: set Ceph version in CephCluster manifest
+      run: |
+        cd cluster/examples/kubernetes/ceph
+        tests/scripts/github-action-helper.sh replace_ceph_image "cluster-test.yaml" "${{ inputs.ceph-image }}"
+
+    - name: deploy first cluster rook
+      run: |
+        tests/scripts/github-action-helper.sh deploy_first_rook_cluster
+        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml
+        # wait for multisite store to be created
+        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph
+
+    - name: prep second cluster pull realm config
+      run: |
+        cd cluster/examples/kubernetes/ceph/
+        IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
+        yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80
+        BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
+        BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
+        sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml
+        sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml
+
+    - name: deploy second cluster rook
+      run: |
+        tests/scripts/github-action-helper.sh deploy_second_rook_cluster
+        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml
+        # wait for realms to be pulled and zone-b-multisite-store to be created
+        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary
+
+    - name: wait for ceph cluster 1 to be ready
+      run: |
+        mkdir test
+        tests/scripts/validate_cluster.sh osd 1
+        kubectl -n rook-ceph get pods
+
+    - name: write an object to one cluster, read from the other
+      run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2
+
+    # if this test fails, it could mean the RGW `period get` or `period update` output has changed
+    - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile
+      run: |
+        ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended
+        ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended
+        committed_msg="committing changes to RGW configuration period for CephObjectStore"
+        tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}"
+        tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}"
+        tests/scripts/github-action-helper.sh restart_operator
+        not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore"
+        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120
+        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90
+
+    - name: upload test result
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: rgw-multisite-testing
+        path: test
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index 550160788dd0f..2de168b168a53 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -883,97 +883,25 @@ jobs:
       uses: mxschmitt/action-tmate@v3
       timeout-minutes: 60
 
+  # rgw-multisite-testing:
+  #   uses: ./.github/workflows/z_rgw-multisite.yml
+  #   secrets:
+  #     github-token: ${{ secrets.GITHUB_TOKEN }}
+
   rgw-multisite-testing:
-    runs-on: ubuntu-18.04
     if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
+    runs-on: ubuntu-18.04
     steps:
-    - name: checkout
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: setup golang
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.16
-
-    - name: install deps
-      run: |
-        tests/scripts/github-action-helper.sh install_deps
-        sudo apt-get install -y s3cmd
-
-    - name: setup minikube
-      uses: manusa/actions-setup-minikube@v2.3.1
-      with:
-        minikube version: 'v1.18.1'
-        kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
-        github token: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: use local disk into two partitions
-      run: |
-        BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
-        tests/scripts/github-action-helper.sh use_local_disk
-        tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
-        sudo lsblk
-
-    - name: build rook
-      run: tests/scripts/github-action-helper.sh build_rook
-
-    - name: deploy first cluster rook
-      run: |
-        tests/scripts/github-action-helper.sh deploy_first_rook_cluster
-        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml
-        # wait for multisite store to be created
-        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph
-
-    - name: prep second cluster pull realm config
-      run: |
-        cd cluster/examples/kubernetes/ceph/
-        IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
-        yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80
-        BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
-        BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
-        sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml
-        sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml
-
-    - name: deploy second cluster rook
-      run: |
-        tests/scripts/github-action-helper.sh deploy_second_rook_cluster
-        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml
-        # wait for realms to be pulled and zone-b-multisite-store to be created
-        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary
-
-    - name: wait for ceph cluster 1 to be ready
-      run: |
-        mkdir test
-        tests/scripts/validate_cluster.sh osd 1
-        kubectl -n rook-ceph get pods
-
-    - name: write an object to one cluster, read from the other
-      run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2
-
-    # if this test fails, it could mean the RGW `period get` or `period update` output has changed
-    - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile
-      run: |
-        ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended
-        ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended
-        committed_msg="committing changes to RGW configuration period for CephObjectStore"
-        tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}"
-        tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}"
-        tests/scripts/github-action-helper.sh restart_operator
-        not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore"
-        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120
-        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90
-
-    - name: upload test result
-      uses: actions/upload-artifact@v2
-      if: always()
-      with:
-        name: rgw-multisite-testing
-        path: test
-
-    - name: setup tmate session for debugging when event is PR
-      if: failure() && github.event_name == 'pull_request'
-      uses: mxschmitt/action-tmate@v3
-      timeout-minutes: 60
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/rgw-multisite-test
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # ceph-image: # use default
+
+      - name: setup tmate session for debugging when event is PR
+        if: failure() && github.event_name == 'pull_request'
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 60
diff --git a/.github/workflows/z_rgw-multisite.yml b/.github/workflows/z_rgw-multisite.yml
new file mode 100644
index 0000000000000..afae87c9a35ac
--- /dev/null
+++ b/.github/workflows/z_rgw-multisite.yml
@@ -0,0 +1,120 @@
+name: Reusable workflow to test RGW multisite
+
+on:
+  workflow_call:
+    inputs:
+      ceph-image:
+        description: 'Ceph image to use for the workflow (e.g., quay.io/ceph/ceph:v16.2.5)'
+        required: false
+        type: string
+    secrets:
+      github-token:
+        description: 'GITHUB_TOKEN from the calling workflow'
+        required: true
+
+defaults:
+  run:
+    # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+jobs:
+  rgw-multisite-testing:
+    if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
+
+    runs-on: ubuntu-18.04
+    steps:
+    - name: checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: setup golang
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+
+    - name: install deps
+      run: |
+        tests/scripts/github-action-helper.sh install_deps
+        sudo apt-get install -y s3cmd
+
+    - name: setup minikube
+      uses: manusa/actions-setup-minikube@v2.3.1
+      with:
+        minikube version: 'v1.18.1'
+        kubernetes version: 'v1.19.2'
+        start args: --memory 6g --cpus=2
+        github token: ${{ secrets.github-token }}
+
+    - name: use local disk into two partitions
+      run: |
+        BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
+        tests/scripts/github-action-helper.sh use_local_disk
+        tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
+        sudo lsblk
+
+    - name: build rook
+      run: tests/scripts/github-action-helper.sh build_rook
+
+    - name: set Ceph version in CephCluster manifest
+      run: |
+        cd cluster/examples/kubernetes/ceph
+        tests/scripts/github-action-helper.sh replace_ceph_image "cluster-test.yaml" "${{ inputs.ceph-image }}"
+
+    - name: deploy first cluster rook
+      run: |
+        tests/scripts/github-action-helper.sh deploy_first_rook_cluster
+        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml
+        # wait for multisite store to be created
+        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph
+
+    - name: prep second cluster pull realm config
+      run: |
+        cd cluster/examples/kubernetes/ceph/
+        IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
+        yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80
+        BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
+        BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
+        sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml
+        sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml
+
+    - name: deploy second cluster rook
+      run: |
+        tests/scripts/github-action-helper.sh deploy_second_rook_cluster
+        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml
+        # wait for realms to be pulled and zone-b-multisite-store to be created
+        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary
+
+    - name: wait for ceph cluster 1 to be ready
+      run: |
+        mkdir test
+        tests/scripts/validate_cluster.sh osd 1
+        kubectl -n rook-ceph get pods
+
+    - name: write an object to one cluster, read from the other
+      run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2
+
+    # if this test fails, it could mean the RGW `period get` or `period update` output has changed
+    - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile
+      run: |
+        ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended
+        ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended
+        committed_msg="committing changes to RGW configuration period for CephObjectStore"
+        tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}"
+        tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}"
+        tests/scripts/github-action-helper.sh restart_operator
+        not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore"
+        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120
+        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90
+
+    - name: upload test result
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: rgw-multisite-testing
+        path: test
+
+    - name: setup tmate session for debugging when event is PR
+      if: failure() && github.event_name == 'pull_request'
+      uses: mxschmitt/action-tmate@v3
+      timeout-minutes: 60
diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index 961747e54d05c..ba6ed22e1410a 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -154,6 +154,16 @@ function deploy_manifest_with_local_build() {
   kubectl create -f $1
 }
 
+function replace_ceph_image() {
+  local file="$1"  # parameter 1: the file in which to replace the ceph image
+  local ceph_image="${2:-''}"  # parameter 2: the new ceph image to use
+  if [[ -z "${ceph_image}" ]]; then
+    echo "No Ceph image given. Not adjusting manifests."
+    return 0
+  fi
+  sed -i "s|image: .*ceph/ceph:.*|image: ${ceph_image}|g" "${file}"
+}
+
 function deploy_cluster() {
   cd cluster/examples/kubernetes/ceph
   deploy_manifest_with_local_build operator.yaml