diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index b3a9929fe856f..0cc16bcc5637a 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -762,72 +762,11 @@ jobs: with: fetch-depth: 0 - - name: setup golang - uses: actions/setup-go@v2 + - name: run RGW multisite test + uses: ./.github/actions/rgw-multisite-test with: - go-version: 1.16 - - - name: install deps - run: | - tests/scripts/github-action-helper.sh install_deps - sudo apt-get install -y s3cmd - - name: setup minikube - uses: manusa/actions-setup-minikube@v2.3.1 - with: - minikube version: 'v1.23.2' - kubernetes version: 'v1.22.2' - start args: --memory 6g --cpus=2 - github token: ${{ secrets.GITHUB_TOKEN }} - - - name: use local disk into two partitions - run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 - sudo lsblk - - name: build rook - run: tests/scripts/github-action-helper.sh build_rook - - - name: deploy first cluster rook - run: | - tests/scripts/github-action-helper.sh deploy_first_rook_cluster - kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml - # wait for multisite store to be created - tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph - - name: prep second cluster pull realm config - run: | - cd cluster/examples/kubernetes/ceph/ - IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") - yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80 - BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") - BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") - sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml - sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml - - name: deploy second cluster rook - run: | - tests/scripts/github-action-helper.sh deploy_second_rook_cluster - kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml - # wait for realms to be pulled and zone-b-multisite-store to be created - tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary - - name: write an object to one cluster, read from the other - run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 - - # if this test fails, it could mean the RGW `period get` or `period update` output has changed - - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile - run: | - ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended - ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended - committed_msg="committing changes to RGW configuration period for CephObjectStore" - tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}" - tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}" - tests/scripts/github-action-helper.sh restart_operator - not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore" - tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120 - tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90 - - name: collect common logs - if: always() - run: | - tests/scripts/collect-logs.sh + github-token: ${{ secrets.GITHUB_TOKEN }} + # ceph-image: # use default - name: upload test result uses: actions/upload-artifact@v2 diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index afc437e388004..a89d1c34d084b 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -292,3 +292,27 @@ jobs: with: name: ceph-upgrade-suite-octopus-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + + rgw-multisite-test-with-ceph-devel: + runs-on: ubuntu-18.04 + strategy: + matrix: + ceph-image-tag: ['latest-master-devel', 'latest-octopus-devel', 'latest-pacific-devel'] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: run RGW multisite test + uses: ./.github/workflows/rgw-multisite-test + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + ceph-image: quay.io/ceph/daemon-base:${{ matrix.ceph-image-tag }} + + - name: upload test result + uses: actions/upload-artifact@v2 + if: always() + with: + name: rgw-multisite-testing + path: test diff --git a/.github/workflows/rgw-multisite-test/action.yml b/.github/workflows/rgw-multisite-test/action.yml new file mode 100644 index 0000000000000..33acdc1d864e2 --- /dev/null +++ b/.github/workflows/rgw-multisite-test/action.yml @@ -0,0 +1,88 @@ +name: RGW Multisite Test +description: Reusable workflow to test RGW multisite integration +inputs: + github-token: + description: GITHUB_TOKEN from the calling workflow + required: true + ceph-image: + description: Ceph image to use for the workflow (e.g., quay.io/ceph/ceph:v16.2.5) + required: false + +runs: + using: "composite" + steps: + - name: setup cluster resources + uses: ./.github/workflows/setup-cluster-resources + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: install deps + run: | + tests/scripts/github-action-helper.sh install_deps + sudo apt-get install -y s3cmd + + - name: use local disk into two partitions + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/github-action-helper.sh use_local_disk + tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 + sudo lsblk + + - name: set Ceph version in CephCluster manifest + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh replace_ceph_image \ + "cluster/examples/kubernetes/ceph/cluster-test.yaml" "${{ inputs.ceph-image }}" + + - name: deploy first cluster rook + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh deploy_first_rook_cluster + kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml + # wait for multisite store to be created + tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph + + - name: prep second cluster pull realm config + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + cd cluster/examples/kubernetes/ceph/ + IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") + yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80 + BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") + BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") + sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml + sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml + + - name: deploy second cluster rook + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh deploy_second_rook_cluster + kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml + # wait for realms to be pulled and zone-b-multisite-store to be created + tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary + + - name: wait for ceph cluster 1 to be ready + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + mkdir test + tests/scripts/validate_cluster.sh osd 1 + kubectl -n rook-ceph get pods + + - name: write an object to one cluster, read from the other + shell: bash --noprofile --norc -eo pipefail -x {0} + run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 + + # if this test fails, it could mean the RGW `period get` or `period update` output has changed + - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended + ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended + committed_msg="committing changes to RGW configuration period for CephObjectStore" + tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}" + tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}" + tests/scripts/github-action-helper.sh restart_operator + not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore" + tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120 + tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90 diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 1a4fd88aef4be..15c395ae8e409 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -192,6 +192,16 @@ function deploy_manifest_with_local_build() { kubectl create -f $1 } +function replace_ceph_image() { + local file="$1" # parameter 1: the file in which to replace the ceph image + local ceph_image="${2:-}" # parameter 2: the new ceph image to use + if [[ -z ${ceph_image} ]]; then + echo "No Ceph image given. Not adjusting manifests." + return 0 + fi + sed -i "s|image: .*ceph/ceph:.*|image: ${ceph_image}|g" "${file}" +} + function deploy_cluster() { cd cluster/examples/kubernetes/ceph deploy_manifest_with_local_build operator.yaml @@ -265,7 +275,6 @@ function deploy_first_rook_cluster() { BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) create_cluster_prerequisites cd cluster/examples/kubernetes/ceph/ - deploy_manifest_with_local_build operator.yaml yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false @@ -283,7 +292,7 @@ function deploy_second_rook_cluster() { yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external" kubectl create -f cluster-test.yaml yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary - deploy_manifest_with_local_build toolbox.yaml toolbox.yaml + deploy_manifest_with_local_build toolbox.yaml } function wait_for_rgw() {