helm: allow further configurability of ingress version

The ingress api version changed when it went to v1, and this has caused some upheaval throughout the kubernetes ecosystem. This commit uses a common method of deciding which ingress api to use, and allows the optional override of the kubernetes version presented to helm using the helm build-in capabilities. also add an ingress into the helm integration tests so any regressions to how ingresses are handled in the future are caught easier. Closes #9174 Signed-off-by: Tom Hellier <me@tomhellier.com> (cherry picked from commit ba44602)
rook · Nov 24, 2021 · 2322a17 · 2322a17
1 parent 10e6878
commit 2322a17
Show file tree

Hide file tree

Showing 15 changed files with 97 additions and 60 deletions.
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
@@ -34,9 +34,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status
@@ -121,9 +121,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: use local disk and create partitions for osds
@@ -198,9 +198,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: use local disk
@@ -272,9 +272,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: use local disk
@@ -343,9 +343,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status
@@ -423,9 +423,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: use local disk
@@ -498,9 +498,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: use local disk
@@ -570,9 +570,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status
@@ -666,9 +666,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: use local disk
@@ -736,9 +736,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.19.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: use local disk into two partitions

diff --git a/.github/workflows/integration-test-flex-suite.yaml b/.github/workflows/integration-test-flex-suite.yaml
@@ -28,9 +28,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.15.12'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status

diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
@@ -30,11 +30,11 @@ jobs:
         go-version: 1.16
 
     - name: setup minikube
-      uses: manusa/actions-setup-minikube@v2.3.1
+      uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.18.1'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: remove read permission from kube config file

diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
@@ -29,9 +29,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status

diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -32,9 +32,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status

diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
@@ -32,9 +32,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status

diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -32,9 +32,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status

diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml
@@ -33,9 +33,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status
@@ -91,11 +91,11 @@ jobs:
         go-version: 1.16
 
     - name: setup minikube
-      uses: manusa/actions-setup-minikube@v2.3.1
+      uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.18.1'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: remove read permission from kube config file
@@ -158,9 +158,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status
@@ -219,9 +219,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status
@@ -279,9 +279,9 @@ jobs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.21.0'
+        minikube version: 'v1.24.0'
         kubernetes version: ${{ matrix.kubernetes-versions }}
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: print k8s cluster status

diff --git a/Documentation/helm-ceph-cluster.md b/Documentation/helm-ceph-cluster.md
@@ -57,6 +57,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | Parameter              | Description                                                          | Default     |
 | ---------------------- | -------------------------------------------------------------------- | ----------- |
 | `operatorNamespace`    | Namespace of the Rook Operator                                       | `rook-ceph` |
+| `kubeVersion`          | Optional override of the target kubernetes version                   | ``          |
 | `configOverride`       | Cluster ceph.conf override                                           | <empty>     |
 | `toolbox.enabled`      | Enable Ceph debugging pod deployment. See [toolbox](ceph-toolbox.md) | `false`     |
 | `toolbox.tolerations`  | Toolbox tolerations                                                  | `[]`        |

diff --git a/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl b/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl
@@ -31,3 +31,23 @@ Define the clusterName as defaulting to the release namespace
 {{- define "clusterName" -}}
 {{ .Values.clusterName | default .Release.Namespace }}
 {{- end -}}
+
+{{/*
+Return the target Kubernetes version.
+*/}}
+{{- define "capabilities.kubeVersion" -}}
+{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}}
+{{- end -}}
+
+{{/*
+Return the appropriate apiVersion for ingress.
+*/}}
+{{- define "capabilities.ingress.apiVersion" -}}
+{{- if semverCompare "<1.14-0" (include "capabilities.kubeVersion" .) -}}
+{{- print "extensions/v1beta1" -}}
+{{- else if semverCompare "<1.19-0" (include "capabilities.kubeVersion" .) -}}
+{{- print "networking.k8s.io/v1beta1" -}}
+{{- else -}}
+{{- print "networking.k8s.io/v1" -}}
+{{- end -}}
+{{- end -}}
diff --git a/cluster/charts/rook-ceph-cluster/templates/ingress.yaml b/cluster/charts/rook-ceph-cluster/templates/ingress.yaml
@@ -1,12 +1,6 @@
 {{- if .Values.ingress.dashboard.host }}
 ---
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
-apiVersion: networking.k8s.io/v1
-{{ else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
-apiVersion: networking.k8s.io/v1beta1
-{{ else }}
-apiVersion: extensions/v1beta1
-{{ end -}}
+apiVersion: {{ include "capabilities.ingress.apiVersion" . }}
 kind: Ingress
 metadata:
   name: {{ template "clusterName" . }}-dashboard
@@ -20,7 +14,14 @@ spec:
         paths:
           - path: {{ .Values.ingress.dashboard.host.path | default "/" }}
             backend:
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+{{- if (semverCompare "<1.18-0" (include "capabilities.kubeVersion" .)) }}
+              serviceName: rook-ceph-mgr-dashboard
+              {{- if .Values.cephClusterSpec.dashboard.ssl }}
+              servicePort: https-dashboard
+              {{- else }}
+              servicePort: http-dashboard
+              {{- end }}
+{{- else }}
               service:
                 name: rook-ceph-mgr-dashboard
                 port:
@@ -30,13 +31,6 @@ spec:
                   name: http-dashboard
                   {{- end }}
             pathType: Prefix
-{{- else }}
-              serviceName: rook-ceph-mgr-dashboard
-              {{- if .Values.cephClusterSpec.dashboard.ssl }}
-              servicePort: https-dashboard
-              {{- else }}
-              servicePort: http-dashboard
-              {{- end }}
 {{- end }}
   {{- if .Values.ingress.dashboard.tls }}
   tls: {{- toYaml .Values.ingress.dashboard.tls | nindent 4 }}

diff --git a/cluster/charts/rook-ceph-cluster/values.yaml b/cluster/charts/rook-ceph-cluster/values.yaml
@@ -8,6 +8,9 @@ operatorNamespace: rook-ceph
 # The metadata.name of the CephCluster CR. The default name is the same as the namespace.
 # clusterName: rook-ceph
 
+# Ability to override the kubernetes version used in rendering the helm chart
+# kubeVersion: 1.21
+
 # Ability to override ceph.conf
 # configOverride: |
 #   [global]

diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go
@@ -69,14 +69,26 @@ func (h *CephInstaller) CreateRookCephClusterViaHelm(values map[string]interface
 	if err := yaml.Unmarshal([]byte(h.Manifests.GetCephCluster()), &clusterCRD); err != nil {
 		return err
 	}
+	values["cephClusterSpec"] = clusterCRD["spec"]
 
 	values["operatorNamespace"] = h.settings.OperatorNamespace
 	values["configOverride"] = clusterCustomSettings
 	values["toolbox"] = map[string]interface{}{
 		"enabled": true,
 		"image":   "rook/ceph:" + LocalBuildTag,
 	}
-	values["cephClusterSpec"] = clusterCRD["spec"]
+	values["ingress"] = map[string]interface{}{
+		"dashboard": map[string]interface{}{
+			"annotations": map[string]interface{}{
+				"kubernetes.io/ingress-class":                "nginx",
+				"nginx.ingress.kubernetes.io/rewrite-target": "/ceph-dashboard/$2",
+			},
+			"host": map[string]interface{}{
+				"name": "localhost",
+				"path": "/ceph-dashboard(/|$)(.*)",
+			},
+		},
+	}
 
 	if err := h.CreateBlockPoolConfiguration(values, blockPoolName, blockPoolSCName); err != nil {
 		return err

diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go
@@ -86,6 +86,12 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
 	require.Nil(s.T(), err)
 }
 
+func checkIfRookClusterHasHealthyIngress(s suite.Suite, k8sh *utils.K8sHelper, clusterNamespace string) {
+	logger.Infof("Testing ingress %s health", clusterNamespace)
+	_, err := k8sh.GetResourceStatus("Ingress", clusterNamespace+"-dashboard", clusterNamespace)
+	assert.NoError(s.T(), err)
+}
+
 func HandlePanics(r interface{}, uninstaller func(), t func() *testing.T) {
 	if r != nil {
 		logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)

diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go
@@ -92,6 +92,7 @@ func (h *HelmSuite) AfterTest(suiteName, testName string) {
 // Test to make sure all rook components are installed and Running
 func (h *HelmSuite) TestARookInstallViaHelm() {
 	checkIfRookClusterIsInstalled(h.Suite, h.k8shelper, h.settings.Namespace, h.settings.Namespace, 1)
+	checkIfRookClusterHasHealthyIngress(h.Suite, h.k8shelper, h.settings.Namespace)
 }
 
 // Test BlockCreation on Rook that was installed via Helm