Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test: run RGW multisite test in nightly job
In the nightly job, run the test with the latest Ceph version so we can detect if there are RGW changes in Ceph that might break multisite. Use a reusable GitHub action workflow to duplicate as little code as possible. Signed-off-by: Blaine Gardner <blaine.gardner@redhat.com>
- Loading branch information
Showing
4 changed files
with
253 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| name: RGW Multisite Test | ||
| description: Reusable workflow to test RGW multisite integration | ||
| inputs: | ||
| github-token: | ||
| description: GITHUB_TOKEN from the calling workflow | ||
| required: true | ||
| ceph-image: | ||
| description: Ceph image to use for the workflow (e.g., quay.io/ceph/ceph:v16.2.5) | ||
| required: false | ||
| type: string | ||
|
|
||
| defaults: | ||
| run: | ||
| # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell | ||
| shell: bash --noprofile --norc -eo pipefail -x {0} | ||
|
|
||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: setup golang | ||
| uses: actions/setup-go@v2 | ||
| with: | ||
| go-version: 1.16 | ||
|
|
||
| - name: install deps | ||
| run: | | ||
| tests/scripts/github-action-helper.sh install_deps | ||
| sudo apt-get install -y s3cmd | ||
| - name: setup minikube | ||
| uses: manusa/actions-setup-minikube@v2.3.1 | ||
| with: | ||
| minikube version: 'v1.18.1' | ||
| kubernetes version: 'v1.19.2' | ||
| start args: --memory 6g --cpus=2 | ||
| github token: ${{ inputs.github-token }} | ||
|
|
||
| - name: use local disk into two partitions | ||
| run: | | ||
| BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) | ||
| tests/scripts/github-action-helper.sh use_local_disk | ||
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 | ||
| sudo lsblk | ||
| - name: build rook | ||
| run: tests/scripts/github-action-helper.sh build_rook | ||
|
|
||
| - name: set Ceph version in CephCluster manifest | ||
| run: | | ||
| cd cluster/examples/kubernetes/ceph | ||
| tests/scripts/github-action-helper.sh replace_ceph_image "cluster-test.yaml" "${{ inputs.ceph-image }}" | ||
| - name: deploy first cluster rook | ||
| run: | | ||
| tests/scripts/github-action-helper.sh deploy_first_rook_cluster | ||
| kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml | ||
| # wait for multisite store to be created | ||
| tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph | ||
| - name: prep second cluster pull realm config | ||
| run: | | ||
| cd cluster/examples/kubernetes/ceph/ | ||
| IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") | ||
| yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80 | ||
| BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") | ||
| BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") | ||
| sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml | ||
| sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml | ||
| - name: deploy second cluster rook | ||
| run: | | ||
| tests/scripts/github-action-helper.sh deploy_second_rook_cluster | ||
| kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml | ||
| # wait for realms to be pulled and zone-b-multisite-store to be created | ||
| tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary | ||
| - name: wait for ceph cluster 1 to be ready | ||
| run: | | ||
| mkdir test | ||
| tests/scripts/validate_cluster.sh osd 1 | ||
| kubectl -n rook-ceph get pods | ||
| - name: write an object to one cluster, read from the other | ||
| run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 | ||
|
|
||
| # if this test fails, it could mean the RGW `period get` or `period update` output has changed | ||
| - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile | ||
| run: | | ||
| ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended | ||
| ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended | ||
| committed_msg="committing changes to RGW configuration period for CephObjectStore" | ||
| tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}" | ||
| tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}" | ||
| tests/scripts/github-action-helper.sh restart_operator | ||
| not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore" | ||
| tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120 | ||
| tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90 | ||
| - name: upload test result | ||
| uses: actions/upload-artifact@v2 | ||
| if: always() | ||
| with: | ||
| name: rgw-multisite-testing | ||
| path: test |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| name: Reusable workflow to test RGW multisite | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| ceph-image: | ||
| description: 'Ceph image to use for the workflow (e.g., quay.io/ceph/ceph:v16.2.5)' | ||
| required: false | ||
| type: string | ||
| secrets: | ||
| github-token: | ||
| description: 'GITHUB_TOKEN from the calling workflow' | ||
| required: true | ||
|
|
||
| defaults: | ||
| run: | ||
| # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell | ||
| shell: bash --noprofile --norc -eo pipefail -x {0} | ||
|
|
||
| jobs: | ||
| rgw-multisite-testing: | ||
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | ||
|
|
||
| runs-on: ubuntu-18.04 | ||
| steps: | ||
| - name: checkout | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: 0 | ||
|
|
||
| - name: setup golang | ||
| uses: actions/setup-go@v2 | ||
| with: | ||
| go-version: 1.16 | ||
|
|
||
| - name: install deps | ||
| run: | | ||
| tests/scripts/github-action-helper.sh install_deps | ||
| sudo apt-get install -y s3cmd | ||
| - name: setup minikube | ||
| uses: manusa/actions-setup-minikube@v2.3.1 | ||
| with: | ||
| minikube version: 'v1.18.1' | ||
| kubernetes version: 'v1.19.2' | ||
| start args: --memory 6g --cpus=2 | ||
| github token: ${{ secrets.github-token }} | ||
|
|
||
| - name: use local disk into two partitions | ||
| run: | | ||
| BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) | ||
| tests/scripts/github-action-helper.sh use_local_disk | ||
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 | ||
| sudo lsblk | ||
| - name: build rook | ||
| run: tests/scripts/github-action-helper.sh build_rook | ||
|
|
||
| - name: set Ceph version in CephCluster manifest | ||
| run: | | ||
| cd cluster/examples/kubernetes/ceph | ||
| tests/scripts/github-action-helper.sh replace_ceph_image "cluster-test.yaml" "${{ inputs.ceph-image }}" | ||
| - name: deploy first cluster rook | ||
| run: | | ||
| tests/scripts/github-action-helper.sh deploy_first_rook_cluster | ||
| kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml | ||
| # wait for multisite store to be created | ||
| tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph | ||
| - name: prep second cluster pull realm config | ||
| run: | | ||
| cd cluster/examples/kubernetes/ceph/ | ||
| IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") | ||
| yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80 | ||
| BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") | ||
| BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") | ||
| sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml | ||
| sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml | ||
| - name: deploy second cluster rook | ||
| run: | | ||
| tests/scripts/github-action-helper.sh deploy_second_rook_cluster | ||
| kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml | ||
| # wait for realms to be pulled and zone-b-multisite-store to be created | ||
| tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary | ||
| - name: wait for ceph cluster 1 to be ready | ||
| run: | | ||
| mkdir test | ||
| tests/scripts/validate_cluster.sh osd 1 | ||
| kubectl -n rook-ceph get pods | ||
| - name: write an object to one cluster, read from the other | ||
| run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 | ||
|
|
||
| # if this test fails, it could mean the RGW `period get` or `period update` output has changed | ||
| - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile | ||
| run: | | ||
| ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended | ||
| ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended | ||
| committed_msg="committing changes to RGW configuration period for CephObjectStore" | ||
| tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}" | ||
| tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}" | ||
| tests/scripts/github-action-helper.sh restart_operator | ||
| not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore" | ||
| tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120 | ||
| tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90 | ||
| - name: upload test result | ||
| uses: actions/upload-artifact@v2 | ||
| if: always() | ||
| with: | ||
| name: rgw-multisite-testing | ||
| path: test | ||
|
|
||
| - name: setup tmate session for debugging when event is PR | ||
| if: failure() && github.event_name == 'pull_request' | ||
| uses: mxschmitt/action-tmate@v3 | ||
| timeout-minutes: 60 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters