core: allow downgrade of all daemons consistently

In the event a ceph image is specified that is lower than the current running
version of the daemons, the downgrade is allowed, even if not technically
supported. All of the core daemons (mon,mgr,osd) were being downgraded,
but the daemons for other controllers (rgw,mds,rbdmirror) were not being
downgraded, resulting in an inconsistent cluster. Now we log that the downgrade
is not supported and all all of the daemons to be downgraded.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>

pkg/operator/ceph/cluster/version.go

@@ -109,7 +109,8 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
 			}
 
 			if cephver.IsInferior(imageSpecVersion, clusterRunningVersion) {
-				return true, errors.Errorf("image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String())
+				logger.Warningf("image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String())
+				return true, nil
 			}
 		}
 	}

pkg/operator/ceph/cluster/version_test.go

@@ -76,8 +76,9 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
 	err = json.Unmarshal([]byte(fakeRunningVersions), &dummyRunningVersions3)
 	assert.NoError(t, err)
 
+	// Allow the downgrade
 	m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions3)
-	assert.Error(t, err)
+	assert.NoError(t, err)
 	assert.True(t, m)
 
 	// 4 test - spec version is higher than running cluster --> we upgrade