Merge pull request #8743 from leseb/next-pacific

ceph: use next ceph v16.2.6 pacific version

.github/workflows/canary-integration-test.yml

@@ -860,7 +860,7 @@ jobs:
 
     - name: verify fs mirroring is working
       run: |
-        timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -eq 3 ]; do echo "waiting for connection to peer" && sleep 1; done'
+        timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -lt 3 ]; do echo "waiting for connection to peer" && sleep 1; done'
         sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/)
         status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$socket help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done)
         if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi

Documentation/ceph-cluster-crd.md

@@ -32,7 +32,7 @@ metadata:
 spec:
   cephVersion:
     # see the "Cluster Settings" section below for more details on which image of ceph to run
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -60,7 +60,7 @@ metadata:
 spec:
   cephVersion:
     # see the "Cluster Settings" section below for more details on which image of ceph to run
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -129,7 +129,7 @@ spec:
       - name: c
   cephVersion:
     # Stretch cluster is supported in Ceph Pacific or newer.
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     allowUnsupported: true
   # Either storageClassDeviceSets or the storage section can be specified for creating OSDs.
   # This example uses all devices for simplicity.
@@ -167,7 +167,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
 * `external`:
   * `enable`: if `true`, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled **all** the other options will be ignored except `cephVersion.image` and `dataDirHostPath`. See [external cluster configuration](#external-cluster). If `cephVersion.image` is left blank, Rook will refuse the creation of extra CRs like object, file and nfs.
 * `cephVersion`: The version information for launching the ceph daemons.
-  * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v15.2.12` or `v16.2.5`. For more details read the [container images section](#ceph-container-images).
+  * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v15.2.12` or `v16.2.6`. For more details read the [container images section](#ceph-container-images).
   For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/).
   To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version.
   Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v14` will be updated each time a new nautilus build is released.
@@ -685,8 +685,8 @@ kubectl -n rook-ceph get CephCluster -o yaml
       deviceClasses:
       - name: hdd
     version:
-      image: quay.io/ceph/ceph:v16.2.5
-      version: 16.2.5-0
+      image: quay.io/ceph/ceph:v16.2.6
+      version: 16.2.6-0
     conditions:
     - lastHeartbeatTime: "2021-03-02T21:22:11Z"
       lastTransitionTime: "2021-03-02T21:21:09Z"
@@ -747,7 +747,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -779,7 +779,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -819,7 +819,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -866,7 +866,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -972,7 +972,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -1018,7 +1018,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     allowUnsupported: false
   dashboard:
     enabled: true
@@ -1476,7 +1476,7 @@ spec:
     enable: true
   dataDirHostPath: /var/lib/rook
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5 # Should match external cluster version
+    image: quay.io/ceph/ceph:v16.2.6 # Should match external cluster version
 ```
 
 ### Security

Documentation/ceph-upgrade.md

@@ -430,7 +430,7 @@ Prior to August 2021, official images were on docker.io. While those images will
 
 These images are tagged in a few ways:
 
-* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v16.2.5-20210708`).
+* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v16.2.6-20210916`).
   These tags are recommended for production clusters, as there is no possibility for the cluster to
   be heterogeneous with respect to the version of Ceph running in containers.
 * Ceph major version tags (e.g., `v16`) are useful for development and test clusters so that the
@@ -446,7 +446,7 @@ The majority of the upgrade will be handled by the Rook operator. Begin the upgr
 Ceph image field in the cluster CRD (`spec.cephVersion.image`).
 
 ```sh
-NEW_CEPH_IMAGE='quay.io/ceph/ceph:v16.2.5-20210708'
+NEW_CEPH_IMAGE='quay.io/ceph/ceph:v16.2.6-20210916'
 CLUSTER_NAME="$ROOK_CLUSTER_NAMESPACE"  # change if your cluster name is not the Rook namespace
 kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $CLUSTER_NAME --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}"
 ```
@@ -466,9 +466,9 @@ Determining when the Ceph has fully updated is rather simple.
 kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"ceph-version="}{.metadata.labels.ceph-version}{"\n"}{end}' | sort | uniq
 This cluster is not yet finished:
     ceph-version=15.2.13-0
-    ceph-version=16.2.5-0
+    ceph-version=16.2.6-0
 This cluster is finished:
-    ceph-version=16.2.5-0
+    ceph-version=16.2.6-0
 ```
 
 #### **3. Verify the updated cluster**

cluster/charts/rook-ceph-cluster/values.yaml

@@ -42,7 +42,7 @@ cephClusterSpec:
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
     # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v15.2.11-20200419
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     # Whether to allow unsupported versions of Ceph. Currently `nautilus` and `octopus` are supported.
     # Future versions such as `pacific` would require this to be set to `true`.
     # Do not set to true in production.

cluster/examples/kubernetes/ceph/cluster-external-management.yaml

@@ -19,4 +19,4 @@ spec:
   dataDirHostPath: /var/lib/rook
   # providing an image is required, if you want to create other CRs (rgw, mds, nfs)
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5 # Should match external cluster version
+    image: quay.io/ceph/ceph:v16.2.6 # Should match external cluster version

cluster/examples/kubernetes/ceph/cluster-on-local-pvc.yaml

@@ -171,7 +171,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false

cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml

@@ -33,7 +33,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false

cluster/examples/kubernetes/ceph/cluster-stretched.yaml

@@ -39,7 +39,7 @@ spec:
     count: 2
   cephVersion:
     # Stretch cluster support upstream is only available starting in Ceph Pacific
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     allowUnsupported: true
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false

cluster/examples/kubernetes/ceph/cluster-test.yaml

@@ -28,7 +28,7 @@ metadata:
 spec:
   dataDirHostPath: /var/lib/rook
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     allowUnsupported: true
   mon:
     count: 1

cluster/examples/kubernetes/ceph/cluster.yaml

@@ -19,9 +19,9 @@ spec:
     # v14 is nautilus, v15 is octopus, and v16 is pacific.
     # RECOMMENDATION: In production, use a specific version tag instead of the general v16 flag, which pulls the latest release and could result in different
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
-    # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v16.2.5-20210708
+    # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v16.2.6-20210916
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
     # Whether to allow unsupported versions of Ceph. Currently `nautilus`, `octopus`, and `pacific` are supported.
     # Future versions such as `pacific` would require this to be set to `true`.
     # Do not set to true in production.

cluster/examples/kubernetes/ceph/images.txt

@@ -1,5 +1,5 @@
  rook/ceph:master
- quay.io/ceph/ceph:v16.2.5
+ quay.io/ceph/ceph:v16.2.6
  quay.io/cephcsi/cephcsi:v3.4.0
  k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0
  k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2

cluster/olm/ceph/assemble/metadata-common.yaml

@@ -230,7 +230,7 @@ metadata:
           },
           "spec": {
             "cephVersion": {
-              "image": "quay.io/ceph/ceph:v16.2.5"
+              "image": "quay.io/ceph/ceph:v16.2.6"
             },
             "dataDirHostPath": "/var/lib/rook",
             "mon": {

design/ceph/ceph-cluster-cleanup.md

@@ -34,7 +34,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v16.2.5
+    image: quay.io/ceph/ceph:v16.2.6
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3

images/ceph/Makefile

@@ -18,9 +18,9 @@ include ../image.mk
 # Image Build Options
 
 ifeq ($(GOARCH),amd64)
-CEPH_VERSION = v16.2.5-20210708
+CEPH_VERSION = v16.2.6-20210916
 else
-CEPH_VERSION = v16.2.5-20210708
+CEPH_VERSION = v16.2.6-20210916
 endif
 REGISTRY_NAME = quay.io
 BASEIMAGE = $(REGISTRY_NAME)/ceph/ceph-$(GOARCH):$(CEPH_VERSION)

pkg/operator/ceph/cluster/cluster.go

@@ -224,8 +224,9 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
 	cluster.isUpgrade = isUpgrade
 
 	if cluster.Spec.IsStretchCluster() {
-		if !cephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Build: 5}) {
-			return errors.Errorf("stretch clusters minimum ceph version is v16.2.5, but is running %s", cephVersion.String())
+		stretchVersion := cephver.CephVersion{Major: 16, Minor: 2, Build: 5}
+		if !cephVersion.IsAtLeast(stretchVersion) {
+			return errors.Errorf("stretch clusters minimum ceph version is %q, but is running %s", stretchVersion.String(), cephVersion.String())
 		}
 	}
 

pkg/operator/ceph/file/filesystem.go

@@ -18,10 +18,8 @@ package file
 
 import (
 	"fmt"
-	"syscall"
 
 	"github.com/rook/rook/pkg/operator/k8sutil"
-	"github.com/rook/rook/pkg/util/exec"
 
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -51,37 +49,31 @@ func createFilesystem(
 	ownerInfo *k8sutil.OwnerInfo,
 	dataDirHostPath string,
 ) error {
+	logger.Infof("start running mdses for filesystem %q", fs.Name)
+	c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, ownerInfo, dataDirHostPath)
+	if err := c.Start(); err != nil {
+		return err
+	}
+
 	if len(fs.Spec.DataPools) != 0 {
 		f := newFS(fs.Name, fs.Namespace)
 		if err := f.doFilesystemCreate(context, clusterInfo, clusterSpec, fs.Spec); err != nil {
 			return errors.Wrapf(err, "failed to create filesystem %q", fs.Name)
 		}
 	}
-
-	filesystem, err := cephclient.GetFilesystem(context, clusterInfo, fs.Name)
-	if err != nil {
-		return errors.Wrapf(err, "failed to get filesystem %q", fs.Name)
-	}
-
 	if fs.Spec.MetadataServer.ActiveStandby {
-		if err = cephclient.AllowStandbyReplay(context, clusterInfo, fs.Name, fs.Spec.MetadataServer.ActiveStandby); err != nil {
+		if err := cephclient.AllowStandbyReplay(context, clusterInfo, fs.Name, fs.Spec.MetadataServer.ActiveStandby); err != nil {
 			return errors.Wrapf(err, "failed to set allow_standby_replay to filesystem %q", fs.Name)
 		}
 	}
 
 	// set the number of active mds instances
 	if fs.Spec.MetadataServer.ActiveCount > 1 {
-		if err = cephclient.SetNumMDSRanks(context, clusterInfo, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil {
+		if err := cephclient.SetNumMDSRanks(context, clusterInfo, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil {
 			logger.Warningf("failed setting active mds count to %d. %v", fs.Spec.MetadataServer.ActiveCount, err)
 		}
 	}
 
-	logger.Infof("start running mdses for filesystem %q", fs.Name)
-	c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, filesystem, ownerInfo, dataDirHostPath)
-	if err := c.Start(); err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -94,23 +86,15 @@ func deleteFilesystem(
 	ownerInfo *k8sutil.OwnerInfo,
 	dataDirHostPath string,
 ) error {
-	filesystem, err := cephclient.GetFilesystem(context, clusterInfo, fs.Name)
-	if err != nil {
-		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
-			// If we're deleting the filesystem anyway, ignore the error that the filesystem doesn't exist
-			return nil
-		}
-		return errors.Wrapf(err, "failed to get filesystem %q", fs.Name)
-	}
-	c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, filesystem, ownerInfo, dataDirHostPath)
+	c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, ownerInfo, dataDirHostPath)
 
 	// Delete mds CephX keys and configuration in centralized mon database
 	replicas := fs.Spec.MetadataServer.ActiveCount * 2
 	for i := 0; i < int(replicas); i++ {
 		daemonLetterID := k8sutil.IndexToName(i)
 		daemonName := fmt.Sprintf("%s-%s", fs.Name, daemonLetterID)
 
-		err = c.DeleteMdsCephObjects(daemonName)
+		err := c.DeleteMdsCephObjects(daemonName)
 		if err != nil {
 			return errors.Wrapf(err, "failed to delete mds ceph objects for filesystem %q", fs.Name)
 		}