Merge pull request #9176 from TomHellier/9174-ingress-support-more-k8…

…s-versions helm: Allow further configurability of the ingress version
rook · Nov 23, 2021 · ba54567 · ba54567
2 parents d340bcc + ba44602
commit ba54567
Show file tree

Hide file tree

Showing 9 changed files with 57 additions and 20 deletions.
diff --git a/.github/workflows/integration-test-setup/action.yaml b/.github/workflows/integration-test-setup/action.yaml
@@ -19,9 +19,9 @@ runs:
       - name: setup minikube
         uses: manusa/actions-setup-minikube@v2.4.2
         with:
-          minikube version: "v1.23.2"
+          minikube version: "v1.24.0"
           kubernetes version: ${{ inputs.kubernetes-version }}
-          start args: --memory 6g --cpus=2
+          start args: --memory 6g --cpus=2 --addons ingress
           github token: ${{ inputs.github-token }}
 
       - name: print k8s cluster status

diff --git a/.github/workflows/setup-cluster-resources/action.yaml b/.github/workflows/setup-cluster-resources/action.yaml
@@ -16,9 +16,9 @@ runs:
     - name: setup minikube
       uses: manusa/actions-setup-minikube@v2.4.2
       with:
-        minikube version: 'v1.23.2'
+        minikube version: 'v1.24.0'
         kubernetes version: 'v1.22.2'
-        start args: --memory 6g --cpus=2
+        start args: --memory 6g --cpus=2 --addons ingress
         github token: ${{ inputs.github-token }}
 
     - name: install deps

diff --git a/Documentation/helm-ceph-cluster.md b/Documentation/helm-ceph-cluster.md
@@ -57,6 +57,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | Parameter              | Description                                                          | Default     |
 | ---------------------- | -------------------------------------------------------------------- | ----------- |
 | `operatorNamespace`    | Namespace of the Rook Operator                                       | `rook-ceph` |
+| `kubeVersion`          | Optional override of the target kubernetes version                   | ``          |
 | `configOverride`       | Cluster ceph.conf override                                           | <empty>     |
 | `toolbox.enabled`      | Enable Ceph debugging pod deployment. See [toolbox](ceph-toolbox.md) | `false`     |
 | `toolbox.tolerations`  | Toolbox tolerations                                                  | `[]`        |

diff --git a/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl b/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl
@@ -4,3 +4,23 @@ Define the clusterName as defaulting to the release namespace
 {{- define "clusterName" -}}
 {{ .Values.clusterName | default .Release.Namespace }}
 {{- end -}}
+
+{{/*
+Return the target Kubernetes version.
+*/}}
+{{- define "capabilities.kubeVersion" -}}
+{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}}
+{{- end -}}
+
+{{/*
+Return the appropriate apiVersion for ingress.
+*/}}
+{{- define "capabilities.ingress.apiVersion" -}}
+{{- if semverCompare "<1.14-0" (include "capabilities.kubeVersion" .) -}}
+{{- print "extensions/v1beta1" -}}
+{{- else if semverCompare "<1.19-0" (include "capabilities.kubeVersion" .) -}}
+{{- print "networking.k8s.io/v1beta1" -}}
+{{- else -}}
+{{- print "networking.k8s.io/v1" -}}
+{{- end -}}
+{{- end -}}
diff --git a/cluster/charts/rook-ceph-cluster/templates/ingress.yaml b/cluster/charts/rook-ceph-cluster/templates/ingress.yaml
@@ -1,12 +1,6 @@
 {{- if .Values.ingress.dashboard.host }}
 ---
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
-apiVersion: networking.k8s.io/v1
-{{ else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
-apiVersion: networking.k8s.io/v1beta1
-{{ else }}
-apiVersion: extensions/v1beta1
-{{ end -}}
+apiVersion: {{ include "capabilities.ingress.apiVersion" . }}
 kind: Ingress
 metadata:
   name: {{ template "clusterName" . }}-dashboard
@@ -20,7 +14,14 @@ spec:
         paths:
           - path: {{ .Values.ingress.dashboard.host.path | default "/" }}
             backend:
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+{{- if (semverCompare "<1.18-0" (include "capabilities.kubeVersion" .)) }}
+              serviceName: rook-ceph-mgr-dashboard
+              {{- if .Values.cephClusterSpec.dashboard.ssl }}
+              servicePort: https-dashboard
+              {{- else }}
+              servicePort: http-dashboard
+              {{- end }}
+{{- else }}
               service:
                 name: rook-ceph-mgr-dashboard
                 port:
@@ -30,13 +31,6 @@ spec:
                   name: http-dashboard
                   {{- end }}
             pathType: Prefix
-{{- else }}
-              serviceName: rook-ceph-mgr-dashboard
-              {{- if .Values.cephClusterSpec.dashboard.ssl }}
-              servicePort: https-dashboard
-              {{- else }}
-              servicePort: http-dashboard
-              {{- end }}
 {{- end }}
   {{- if .Values.ingress.dashboard.tls }}
   tls: {{- toYaml .Values.ingress.dashboard.tls | nindent 4 }}

diff --git a/cluster/charts/rook-ceph-cluster/values.yaml b/cluster/charts/rook-ceph-cluster/values.yaml
@@ -8,6 +8,9 @@ operatorNamespace: rook-ceph
 # The metadata.name of the CephCluster CR. The default name is the same as the namespace.
 # clusterName: rook-ceph
 
+# Ability to override the kubernetes version used in rendering the helm chart
+# kubeVersion: 1.21
+
 # Ability to override ceph.conf
 # configOverride: |
 #   [global]

diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go
@@ -72,14 +72,26 @@ func (h *CephInstaller) CreateRookCephClusterViaHelm(values map[string]interface
 	if err := yaml.Unmarshal([]byte(h.Manifests.GetCephCluster()), &clusterCRD); err != nil {
 		return err
 	}
+	values["cephClusterSpec"] = clusterCRD["spec"]
 
 	values["operatorNamespace"] = h.settings.OperatorNamespace
 	values["configOverride"] = clusterCustomSettings
 	values["toolbox"] = map[string]interface{}{
 		"enabled": true,
 		"image":   "rook/ceph:" + LocalBuildTag,
 	}
-	values["cephClusterSpec"] = clusterCRD["spec"]
+	values["ingress"] = map[string]interface{}{
+		"dashboard": map[string]interface{}{
+			"annotations": map[string]interface{}{
+				"kubernetes.io/ingress-class":                "nginx",
+				"nginx.ingress.kubernetes.io/rewrite-target": "/ceph-dashboard/$2",
+			},
+			"host": map[string]interface{}{
+				"name": "localhost",
+				"path": "/ceph-dashboard(/|$)(.*)",
+			},
+		},
+	}
 
 	if err := h.CreateBlockPoolConfiguration(values, blockPoolName, blockPoolSCName); err != nil {
 		return err

diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go
@@ -74,6 +74,12 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
 	require.Nil(s.T(), err)
 }
 
+func checkIfRookClusterHasHealthyIngress(s suite.Suite, k8sh *utils.K8sHelper, clusterNamespace string) {
+	logger.Infof("Testing ingress %s health", clusterNamespace)
+	_, err := k8sh.GetResourceStatus("Ingress", clusterNamespace + "-dashboard", clusterNamespace)
+	assert.NoError(s.T(), err)
+}
+
 func HandlePanics(r interface{}, uninstaller func(), t func() *testing.T) {
 	if r != nil {
 		logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)

diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go
@@ -91,6 +91,7 @@ func (h *HelmSuite) AfterTest(suiteName, testName string) {
 // Test to make sure all rook components are installed and Running
 func (h *HelmSuite) TestARookInstallViaHelm() {
 	checkIfRookClusterIsInstalled(h.Suite, h.k8shelper, h.settings.Namespace, h.settings.Namespace, 1)
+ 	checkIfRookClusterHasHealthyIngress(h.Suite, h.k8shelper, h.settings.Namespace)
 }
 
 // Test BlockCreation on Rook that was installed via Helm