daily-nightly-jobs.yml

name: Daily nightly jobs
on:
  schedule:
    - cron: "0 0 * * *" # every day at midnight

defaults:
  run:
    # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
    shell: bash --noprofile --norc -eo pipefail -x {0}

jobs:
  canary-arm64:
    runs-on: [self-hosted, ubuntu-20.04, ARM64]
    if: github.repository == 'rook/rook'
    env:
      BLOCK: /dev/sdb

    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: setup golang
        uses: actions/setup-go@v2
        with:
          go-version: 1.16

      - name: teardown minikube and docker
        run: |
          uptime
          minikube delete
          docker system prune -a

      - name: setup minikube
        run: |
          # sudo apt-get install build-essential -y
          # curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-arm64
          # sudo install minikube-linux-arm64 /usr/local/bin/minikube
          # sudo rm -f minikube-linux-arm64
          # curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"
          # sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
          minikube start --memory 28g --cpus=12 --driver=docker

      - name: print k8s cluster status
        run: tests/scripts/github-action-helper.sh print_k8s_cluster_status

      - name: use local disk and create partitions for osds
        run: |
          tests/scripts/github-action-helper.sh use_local_disk
          tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 1

      - name: validate-yaml
        run: tests/scripts/github-action-helper.sh validate_yaml

      - name: deploy cluster
        run: |
          # Use the official build images for the nightly arm tests instead of rebuilding
          export USE_LOCAL_BUILD=false
          # removing liveness probes since the env is slow and the probe is killing the daemons
          yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mon.disabled" true
          yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mgr.disabled" true
          yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.osd.disabled" true
          tests/scripts/github-action-helper.sh deploy_cluster
          # there are no package for arm64 nfs-ganesha
          kubectl delete -f cluster/examples/kubernetes/ceph/nfs-test.yaml

      - name: wait for prepare pod
        run: timeout 900 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done' || kubectl -n rook-ceph get all && kubectl logs -n rook-ceph deploy/rook-ceph-operator

      - name: wait for ceph to be ready
        run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready "mon,mgr,osd,mds,rgw,rbd_mirror,fs_mirror" 1

      - name: teardown minikube and docker
        run: |
          minikube delete
          docker system prune -a

      - name: collect common logs
        if: always()
        uses: ./.github/workflows/collect-logs
        with:
          name: canary-arm64

      - name: upload canary test result
        uses: actions/upload-artifact@v2
        if: always()
        with:
          name: canary-arm64
          path: test

  smoke-suite-pacific-devel:
    runs-on: ubuntu-18.04
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: setup cluster resources
        uses: ./.github/workflows/integration-test-setup
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          kubernetes-version: "1.22.2"

      - name: TestCephSmokeSuite
        run: |
          export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
          SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="pacific-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration

      - name: collect common logs
        if: always()
        run: |
          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
          export CLUSTER_NAMESPACE="smoke-ns"
          export OPERATOR_NAMESPACE="smoke-ns-system"
          tests/scripts/collect-logs.sh

      - name: Artifact
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: ceph-smoke-suite-pacific-artifact
          path: /home/runner/work/rook/rook/tests/integration/_output/tests/

  smoke-suite-ceph-master:
    runs-on: ubuntu-18.04
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: setup cluster resources
        uses: ./.github/workflows/integration-test-setup
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          kubernetes-version: "1.22.2"

      - name: TestCephSmokeSuite
        run: |
          export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
          SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=master go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration

      - name: collect common logs
        if: always()
        run: |
          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
          export CLUSTER_NAMESPACE="smoke-ns"
          export OPERATOR_NAMESPACE="smoke-ns-system"
          tests/scripts/collect-logs.sh

      - name: Artifact
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: ceph-smoke-suite-master-artifact
          path: /home/runner/work/rook/rook/tests/integration/_output/tests/

  object-suite-pacific-devel:
    runs-on: ubuntu-18.04
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: setup cluster resources
        uses: ./.github/workflows/integration-test-setup
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          kubernetes-version: "1.22.2"

      - name: TestCephObjectSuite
        run: |
          export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
          SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="pacific-devel" go test -v -timeout 1800s -failfast -run TestCephObjectSuite github.com/rook/rook/tests/integration

      - name: collect common logs
        if: always()
        run: |
          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
          export CLUSTER_NAMESPACE="object-ns"
          export OPERATOR_NAMESPACE="object-ns-system"
          tests/scripts/collect-logs.sh

      - name: Artifact
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: ceph-object-suite-pacific-artifact
          path: /home/runner/work/rook/rook/tests/integration/_output/tests/

  object-suite-master:
    runs-on: ubuntu-18.04
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: setup cluster resources
        uses: ./.github/workflows/integration-test-setup
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          kubernetes-version: "1.22.2"

      - name: TestCephObjectSuite
        run: |
          export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
          SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=master go test -v -timeout 1800s -failfast -run TestCephObjectSuite github.com/rook/rook/tests/integration

      - name: collect common logs
        if: always()
        run: |
          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
          export CLUSTER_NAMESPACE="object-ns"
          export OPERATOR_NAMESPACE="object-ns-system"
          tests/scripts/collect-logs.sh

      - name: Artifact
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: ceph-object-suite-master-artifact
          path: /home/runner/work/rook/rook/tests/integration/_output/tests/

  upgrade-from-pacific-stable-to-pacific-devel:
    runs-on: ubuntu-18.04
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: setup cluster resources
        uses: ./.github/workflows/integration-test-setup
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          kubernetes-version: "1.22.2"

      - name: TestCephUpgradeSuite
        run: |
          export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
          go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToPacificDevel github.com/rook/rook/tests/integration

      - name: collect common logs
        if: always()
        run: |
          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
          export CLUSTER_NAMESPACE="upgrade-ns"
          export OPERATOR_NAMESPACE="upgrade-ns-system"
          tests/scripts/collect-logs.sh

      - name: Artifact
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: ceph-upgrade-suite-pacific-artifact
          path: /home/runner/work/rook/rook/tests/integration/_output/tests/

  upgrade-from-octopus-stable-to-octopus-devel:
    runs-on: ubuntu-18.04
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: setup cluster resources
        uses: ./.github/workflows/integration-test-setup
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          kubernetes-version: "1.22.2"

      - name: TestCephUpgradeSuite
        run: |
          export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
          go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToOctopusDevel github.com/rook/rook/tests/integration

      - name: collect common logs
        if: always()
        run: |
          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
          export CLUSTER_NAMESPACE="upgrade-ns"
          export OPERATOR_NAMESPACE="upgrade-ns-system"
          tests/scripts/collect-logs.sh

      - name: Artifact
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: ceph-upgrade-suite-octopus-artifact
          path: /home/runner/work/rook/rook/tests/integration/_output/tests/

  rgw-multisite-test-with-ceph-devel:
    runs-on: ubuntu-18.04
    strategy:
      matrix:
        ceph-image-tag: ['latest-master-devel', 'latest-octopus-devel', 'latest-pacific-devel']

    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: 0

    - name: run RGW multisite test
      uses: ./.github/workflows/rgw-multisite-test
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        ceph-image: quay.io/ceph/daemon-base:${{ matrix.ceph-image-tag }}

    - name: upload test result
      uses: actions/upload-artifact@v2
      if: always()
      with:
        name: rgw-multisite-testing
        path: test