Merge pull request #9080 from cybozu-go/rgw-change-the-way-to-livenes…

…sprobe-and-introduce-readiness-probe

rgw: change the way to livenessProbe and introduce readinessProbe

cluster/charts/rook-ceph/templates/resources.yaml

@@ -7316,6 +7316,102 @@ spec:
                               type: integer
                           type: object
                       type: object
+                    readinessProbe:
+                      description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
+                      properties:
+                        disabled:
+                          description: Disabled determines whether probe is disable or not
+                          type: boolean
+                        probe:
+                          description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.
+                          properties:
+                            exec:
+                              description: One and only one of the following should be specified. Exec specifies the action to take.
+                              properties:
+                                command:
+                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
+                                  items:
+                                    type: string
+                                  type: array
+                              type: object
+                            failureThreshold:
+                              description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.
+                              format: int32
+                              type: integer
+                            httpGet:
+                              description: HTTPGet specifies the http request to perform.
+                              properties:
+                                host:
+                                  description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead.
+                                  type: string
+                                httpHeaders:
+                                  description: Custom headers to set in the request. HTTP allows repeated headers.
+                                  items:
+                                    description: HTTPHeader describes a custom header to be used in HTTP probes
+                                    properties:
+                                      name:
+                                        description: The header field name
+                                        type: string
+                                      value:
+                                        description: The header field value
+                                        type: string
+                                    required:
+                                      - name
+                                      - value
+                                    type: object
+                                  type: array
+                                path:
+                                  description: Path to access on the HTTP server.
+                                  type: string
+                                port:
+                                  anyOf:
+                                    - type: integer
+                                    - type: string
+                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
+                                  x-kubernetes-int-or-string: true
+                                scheme:
+                                  description: Scheme to use for connecting to the host. Defaults to HTTP.
+                                  type: string
+                              required:
+                                - port
+                              type: object
+                            initialDelaySeconds:
+                              description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                              format: int32
+                              type: integer
+                            periodSeconds:
+                              description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.
+                              format: int32
+                              type: integer
+                            successThreshold:
+                              description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
+                              format: int32
+                              type: integer
+                            tcpSocket:
+                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'
+                              properties:
+                                host:
+                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'
+                                  type: string
+                                port:
+                                  anyOf:
+                                    - type: integer
+                                    - type: string
+                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
+                                  x-kubernetes-int-or-string: true
+                              required:
+                                - port
+                              type: object
+                            terminationGracePeriodSeconds:
+                              description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is an alpha field and requires enabling ProbeTerminationGracePeriod feature gate.
+                              format: int64
+                              type: integer
+                            timeoutSeconds:
+                              description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                              format: int32
+                              type: integer
+                          type: object
+                      type: object
                   type: object
                 metadataPool:
                   description: The metadata pool settings

cluster/examples/kubernetes/ceph/crds.yaml

@@ -7311,6 +7311,102 @@ spec:
                               type: integer
                           type: object
                       type: object
+                    readinessProbe:
+                      description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
+                      properties:
+                        disabled:
+                          description: Disabled determines whether probe is disable or not
+                          type: boolean
+                        probe:
+                          description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.
+                          properties:
+                            exec:
+                              description: One and only one of the following should be specified. Exec specifies the action to take.
+                              properties:
+                                command:
+                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
+                                  items:
+                                    type: string
+                                  type: array
+                              type: object
+                            failureThreshold:
+                              description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.
+                              format: int32
+                              type: integer
+                            httpGet:
+                              description: HTTPGet specifies the http request to perform.
+                              properties:
+                                host:
+                                  description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead.
+                                  type: string
+                                httpHeaders:
+                                  description: Custom headers to set in the request. HTTP allows repeated headers.
+                                  items:
+                                    description: HTTPHeader describes a custom header to be used in HTTP probes
+                                    properties:
+                                      name:
+                                        description: The header field name
+                                        type: string
+                                      value:
+                                        description: The header field value
+                                        type: string
+                                    required:
+                                      - name
+                                      - value
+                                    type: object
+                                  type: array
+                                path:
+                                  description: Path to access on the HTTP server.
+                                  type: string
+                                port:
+                                  anyOf:
+                                    - type: integer
+                                    - type: string
+                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
+                                  x-kubernetes-int-or-string: true
+                                scheme:
+                                  description: Scheme to use for connecting to the host. Defaults to HTTP.
+                                  type: string
+                              required:
+                                - port
+                              type: object
+                            initialDelaySeconds:
+                              description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                              format: int32
+                              type: integer
+                            periodSeconds:
+                              description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.
+                              format: int32
+                              type: integer
+                            successThreshold:
+                              description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
+                              format: int32
+                              type: integer
+                            tcpSocket:
+                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'
+                              properties:
+                                host:
+                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'
+                                  type: string
+                                port:
+                                  anyOf:
+                                    - type: integer
+                                    - type: string
+                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
+                                  x-kubernetes-int-or-string: true
+                              required:
+                                - port
+                              type: object
+                            terminationGracePeriodSeconds:
+                              description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is an alpha field and requires enabling ProbeTerminationGracePeriod feature gate.
+                              format: int64
+                              type: integer
+                            timeoutSeconds:
+                              description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                              format: int32
+                              type: integer
+                          type: object
+                      type: object
                   type: object
                 metadataPool:
                   description: The metadata pool settings

cluster/examples/kubernetes/ceph/object-ec.yaml

@@ -87,6 +87,8 @@ spec:
     bucket:
       disabled: false
       interval: 60s
-    # Configure the pod liveness probe for the rgw daemon
+    # Configure the pod probes for the rgw daemon
     livenessProbe:
       disabled: false
+    readinessProbe:
+      disabled: false

cluster/examples/kubernetes/ceph/object-openshift.yaml

@@ -101,9 +101,11 @@ spec:
     bucket:
       disabled: false
       interval: 60s
-    # Configure the pod liveness probe for the rgw daemon
+    # Configure the pod probes for the rgw daemon
     livenessProbe:
       disabled: false
+    readinessProbe:
+      disabled: false
   # security oriented settings
   # security:
   # To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file

cluster/examples/kubernetes/ceph/object.yaml

@@ -107,9 +107,11 @@ spec:
     bucket:
       disabled: false
       interval: 60s
-    # Configure the pod liveness probe for the rgw daemon
+    # Configure the pod probes for the rgw daemon
     livenessProbe:
       disabled: false
+    readinessProbe:
+      disabled: false
   # security oriented settings
   # security:
   # To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file

pkg/apis/ceph.rook.io/v1/types.go

@@ -1286,6 +1286,8 @@ type BucketHealthCheckSpec struct {
 	Bucket HealthCheckSpec `json:"bucket,omitempty"`
 	// +optional
 	LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"`
+	// +optional
+	ReadinessProbe *ProbeSpec `json:"readinessProbe,omitempty"`
 }
 
 // HealthCheckSpec represents the health check of an object store bucket

pkg/operator/ceph/config/livenessprobe.go

@@ -44,15 +44,15 @@ func ConfigureLivenessProbe(daemon rook.KeyType, container v1.Container, healthC
 			// If the spec value is not empty, let's apply it along with default when some fields are not specified
 			if probe != nil {
 				// Set the liveness probe on the container to overwrite the default probe created by Rook
-				container.LivenessProbe = GetLivenessProbeWithDefaults(probe, container.LivenessProbe)
+				container.LivenessProbe = GetProbeWithDefaults(probe, container.LivenessProbe)
 			}
 		}
 	}
 
 	return container
 }
 
-func GetLivenessProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe {
+func GetProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe {
 	newProbe := *desiredProbe
 
 	// Do not replace the handler with the previous one!

pkg/operator/ceph/config/livenessprobe_test.go

@@ -75,7 +75,7 @@ func configLivenessProbeHelper(t *testing.T, keyType rook.KeyType) {
 	}
 }
 
-func TestGetLivenessProbeWithDefaults(t *testing.T) {
+func TestGetProbeWithDefaults(t *testing.T) {
 	t.Run("using default probe", func(t *testing.T) {
 		currentProb := &v1.Probe{
 			Handler: v1.Handler{
@@ -94,7 +94,7 @@ func TestGetLivenessProbeWithDefaults(t *testing.T) {
 		}
 		// in case of default probe
 		desiredProbe := &v1.Probe{}
-		desiredProbe = GetLivenessProbeWithDefaults(desiredProbe, currentProb)
+		desiredProbe = GetProbeWithDefaults(desiredProbe, currentProb)
 		assert.Equal(t, desiredProbe, currentProb)
 	})
 
@@ -134,7 +134,7 @@ func TestGetLivenessProbeWithDefaults(t *testing.T) {
 			SuccessThreshold:    4,
 			TimeoutSeconds:      5,
 		}
-		desiredProbe = GetLivenessProbeWithDefaults(desiredProbe, currentProb)
+		desiredProbe = GetProbeWithDefaults(desiredProbe, currentProb)
 		assert.Equal(t, desiredProbe.Exec.Command, []string{"env", "-i", "sh", "-c", "ceph --admin-daemon /run/ceph/ceph-mon.c.asok mon_status"})
 		assert.Equal(t, desiredProbe.InitialDelaySeconds, int32(1))
 		assert.Equal(t, desiredProbe.FailureThreshold, int32(2))