diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 431352eee0e1..0ad531585159 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -617,8 +617,9 @@ Currently three health checks are implemented: * `osd`: health check on the ceph osds * `status`: ceph health status check, periodically check the Ceph health state and reflects it in the CephCluster CR status field. -The liveness probe of each daemon can also be controlled via `livenessProbe`, the setting is valid for `mon`, `mgr` and `osd`. -Here is a complete example for both `daemonHealth` and `livenessProbe`: +The liveness probe and startup probe of each daemon can also be controlled via `livenessProbe` and +`startupProbe` respectively. The settings are valid for `mon`, `mgr` and `osd`. +Here is a complete example for both `daemonHealth`, `livenessProbe`, and `startupProbe`: ```yaml healthCheck: @@ -639,21 +640,34 @@ healthCheck: disabled: false osd: disabled: false + startupProbe: + mon: + disabled: false + mgr: + disabled: false + osd: + disabled: false ``` -The probe itself can also be overridden, refer to the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-command). +The probe's timing values and thresholds (but not the probe itself) can also be overridden. +For more info, refer to the +[Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-command). For example, you could change the `mgr` probe by applying: ```yaml healthCheck: + startupProbe: + mgr: + disabled: false + probe: + initialDelaySeconds: 3 + periodSeconds: 3 + failureThreshold: 30 livenessProbe: mgr: disabled: false probe: - httpGet: - path: / - port: 9283 initialDelaySeconds: 3 periodSeconds: 3 ``` diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index 9495f0f15cf4..575190426877 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -163,6 +163,12 @@ Rook-Ceph will be default monitor the state of the object store endpoints. The following CRD settings are available: * `healthCheck`: main object store health monitoring section + * `bucket`: Rook checks that the object store is usable regularly. This is explained in more + detail below. Use this config to disable or change the interval at which Rook verifies the + object store connectivity. + * `startupProbe`: Disable, or override timing and threshold values of the object gateway startup probe. + * `livenessProbe`: Disable, or override timing and threshold values of the object gateway liveness probe. + * `readinessProbe`: Disable, or override timing and threshold values of the object gateway readiness probe. Here is a complete example: @@ -171,6 +177,16 @@ healthCheck: bucket: disabled: false interval: 60s + startupProbe: + disabled: false + livenessProbe: + disabled: false + periodSeconds: 5 + failureThreshold: 4 + readinessProbe: + disabled: false + periodSeconds: 5 + failureThreshold: 2 ``` The endpoint health check procedure is the following: diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index c3deafc7136e..47abde000e0f 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -1033,7 +1033,106 @@ spec: type: integer type: object type: object - description: LivenessProbe allows to change the livenessprobe configuration for a given daemon + description: LivenessProbe allows changing the livenessProbe configuration for a given daemon + type: object + startupProbe: + additionalProperties: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object + description: StartupProbe allows changing the startupProbe configuration for a given daemon type: object type: object labels: @@ -7858,6 +7957,102 @@ spec: type: integer type: object type: object + startupProbe: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object type: object metadataPool: description: The metadata pool settings diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index a65390cdb1f5..365908df3471 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -265,7 +265,7 @@ spec: status: disabled: false interval: 60s - # Change pod liveness probe, it works for all mon,mgr,osd daemons + # Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons. livenessProbe: mon: disabled: false @@ -273,3 +273,11 @@ spec: disabled: false osd: disabled: false + # Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons. + startupProbe: + mon: + disabled: false + mgr: + disabled: false + osd: + disabled: false diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 485697db8e0e..745411e4cd44 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -1032,7 +1032,106 @@ spec: type: integer type: object type: object - description: LivenessProbe allows to change the livenessprobe configuration for a given daemon + description: LivenessProbe allows changing the livenessProbe configuration for a given daemon + type: object + startupProbe: + additionalProperties: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object + description: StartupProbe allows changing the startupProbe configuration for a given daemon type: object type: object labels: @@ -7851,6 +7950,102 @@ spec: type: integer type: object type: object + startupProbe: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object type: object metadataPool: description: The metadata pool settings diff --git a/deploy/examples/object.yaml b/deploy/examples/object.yaml index e0c4d64d7e1a..9bdf7549c62c 100644 --- a/deploy/examples/object.yaml +++ b/deploy/examples/object.yaml @@ -108,6 +108,8 @@ spec: disabled: false interval: 60s # Configure the pod probes for the rgw daemon + startupProbe: + disabled: false livenessProbe: disabled: false readinessProbe: diff --git a/pkg/apis/ceph.rook.io/v1/livenessprobe.go b/pkg/apis/ceph.rook.io/v1/probe.go similarity index 63% rename from pkg/apis/ceph.rook.io/v1/livenessprobe.go rename to pkg/apis/ceph.rook.io/v1/probe.go index a1839740afe6..c2b20486951e 100644 --- a/pkg/apis/ceph.rook.io/v1/livenessprobe.go +++ b/pkg/apis/ceph.rook.io/v1/probe.go @@ -20,6 +20,10 @@ import ( corev1 "k8s.io/api/core/v1" ) +/* + * Liveness probes + */ + // GetMonLivenessProbe returns the liveness probe for the MON service func GetMonLivenessProbe(l CephClusterHealthCheckSpec) *corev1.Probe { return l.LivenessProbe[ResourcesKeyMon].Probe @@ -39,3 +43,27 @@ func GetOSDLivenessProbe(l CephClusterHealthCheckSpec) *corev1.Probe { func GetMdsLivenessProbe(l CephClusterHealthCheckSpec) *corev1.Probe { return l.LivenessProbe[ResourcesKeyMDS].Probe } + +/* + * Startup probes + */ + +// GetMonStartupProbe returns the startup probe for the MON service +func GetMonStartupProbe(l CephClusterHealthCheckSpec) *corev1.Probe { + return l.StartupProbe[ResourcesKeyMon].Probe +} + +// GetMgrStartupProbe returns the startup probe for the MGR service +func GetMgrStartupProbe(l CephClusterHealthCheckSpec) *corev1.Probe { + return l.StartupProbe[ResourcesKeyMgr].Probe +} + +// GetOSDStartupProbe returns the startup probe for the OSD service +func GetOSDStartupProbe(l CephClusterHealthCheckSpec) *corev1.Probe { + return l.StartupProbe[ResourcesKeyOSD].Probe +} + +// GetMdsStartupProbe returns the startup probe for the MDS service +func GetMdsStartupProbe(l CephClusterHealthCheckSpec) *corev1.Probe { + return l.StartupProbe[ResourcesKeyMDS].Probe +} diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index ffa4016381c1..93c883fb95b8 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -59,9 +59,12 @@ type CephClusterHealthCheckSpec struct { // +optional // +nullable DaemonHealth DaemonHealthSpec `json:"daemonHealth,omitempty"` - // LivenessProbe allows to change the livenessprobe configuration for a given daemon + // LivenessProbe allows changing the livenessProbe configuration for a given daemon // +optional LivenessProbe map[KeyType]*ProbeSpec `json:"livenessProbe,omitempty"` + // StartupProbe allows changing the startupProbe configuration for a given daemon + // +optional + StartupProbe map[KeyType]*ProbeSpec `json:"startupProbe,omitempty"` } // DaemonHealthSpec is a daemon health check @@ -1309,6 +1312,8 @@ type BucketHealthCheckSpec struct { LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` // +optional ReadinessProbe *ProbeSpec `json:"readinessProbe,omitempty"` + // +optional + StartupProbe *ProbeSpec `json:"startupProbe,omitempty"` } // HealthCheckSpec represents the health check of an object store bucket diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 238ed95adf38..c12ac3c17943 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -110,6 +110,11 @@ func (in *BucketHealthCheckSpec) DeepCopyInto(out *BucketHealthCheckSpec) { *out = new(ProbeSpec) (*in).DeepCopyInto(*out) } + if in.StartupProbe != nil { + in, out := &in.StartupProbe, &out.StartupProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } return } @@ -587,6 +592,21 @@ func (in *CephClusterHealthCheckSpec) DeepCopyInto(out *CephClusterHealthCheckSp (*out)[key] = outVal } } + if in.StartupProbe != nil { + in, out := &in.StartupProbe, &out.StartupProbe + *out = make(map[KeyType]*ProbeSpec, len(*in)) + for key, val := range *in { + var outVal *ProbeSpec + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } return } diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index b062a5186a5b..81adaa5b7232 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -184,11 +184,12 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { ), Resources: cephv1.GetMgrResources(c.spec.Resources), SecurityContext: controller.PodSecurityContext(), + StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), WorkingDir: config.VarLogCephDir, } - // If the liveness probe is enabled + container = config.ConfigureStartupProbe(cephv1.KeyMgr, container, c.spec.HealthCheck) container = config.ConfigureLivenessProbe(cephv1.KeyMgr, container, c.spec.HealthCheck) // If host networking is enabled, we don't need a bind addr that is different from the public addr diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 657584197b5b..88a829719a65 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -313,6 +313,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: cephv1.GetMonResources(c.spec.Resources), + StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MonType, monConfig.DaemonName), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MonType, monConfig.DaemonName), WorkingDir: config.VarLogCephDir, } @@ -326,7 +327,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container } } - // If the liveness probe is enabled + container = config.ConfigureStartupProbe(cephv1.KeyMon, container, c.spec.HealthCheck) container = config.ConfigureLivenessProbe(cephv1.KeyMon, container, c.spec.HealthCheck) // If host networking is enabled, we don't need a bind addr that is different from the public addr diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 0e6f14e3328d..61ef68b01402 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -552,6 +552,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC Env: envVars, Resources: osdProps.resources, SecurityContext: securityContext, + StartupProbe: controller.GenerateStartupProbeExecDaemon(opconfig.OsdType, osdID), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(opconfig.OsdType, osdID), WorkingDir: opconfig.VarLogCephDir, }, @@ -571,7 +572,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC podTemplateSpec.Spec.Containers = append(podTemplateSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-osd.%s", osdID), c.clusterInfo.Namespace, c.spec)) } - // If the liveness probe is enabled + podTemplateSpec.Spec.Containers[0] = opconfig.ConfigureStartupProbe(cephv1.KeyOSD, podTemplateSpec.Spec.Containers[0], c.spec.HealthCheck) podTemplateSpec.Spec.Containers[0] = opconfig.ConfigureLivenessProbe(cephv1.KeyOSD, podTemplateSpec.Spec.Containers[0], c.spec.HealthCheck) if c.spec.Network.IsHost() { diff --git a/pkg/operator/ceph/config/livenessprobe.go b/pkg/operator/ceph/config/livenessprobe.go index 558c7d919a08..954b02f29c48 100644 --- a/pkg/operator/ceph/config/livenessprobe.go +++ b/pkg/operator/ceph/config/livenessprobe.go @@ -51,6 +51,32 @@ func ConfigureLivenessProbe(daemon cephv1.KeyType, container v1.Container, healt return container } +// ConfigureStartupProbe returns the desired startup probe for a given daemon +func ConfigureStartupProbe(daemon cephv1.KeyType, container v1.Container, healthCheck cephv1.CephClusterHealthCheckSpec) v1.Container { + // Map of functions + probeFnMap := map[cephv1.KeyType]fn{ + cephv1.KeyMon: cephv1.GetMonStartupProbe, + cephv1.KeyMgr: cephv1.GetMgrStartupProbe, + cephv1.KeyOSD: cephv1.GetOSDStartupProbe, + cephv1.KeyMds: cephv1.GetMdsStartupProbe, + } + + if _, ok := healthCheck.StartupProbe[daemon]; ok { + if healthCheck.StartupProbe[daemon].Disabled { + container.StartupProbe = nil + } else { + probe := probeFnMap[daemon](healthCheck) + // If the spec value is not empty, let's apply it along with default when some fields are not specified + if probe != nil { + // Set the startup probe on the container to overwrite the default probe created by Rook + container.StartupProbe = GetProbeWithDefaults(probe, container.StartupProbe) + } + } + } + + return container +} + func GetProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe { newProbe := *desiredProbe diff --git a/pkg/operator/ceph/config/livenessprobe_test.go b/pkg/operator/ceph/config/livenessprobe_test.go index 90c558ea96b2..1109f781d6f8 100644 --- a/pkg/operator/ceph/config/livenessprobe_test.go +++ b/pkg/operator/ceph/config/livenessprobe_test.go @@ -62,8 +62,8 @@ func configLivenessProbeHelper(t *testing.T, keyType cephv1.KeyType) { args args want v1.Container }{ - {"probe-enabled", args{keyType, container, cephv1.CephClusterHealthCheckSpec{}}, container}, - {"probe-disabled", args{keyType, container, cephv1.CephClusterHealthCheckSpec{LivenessProbe: l}}, v1.Container{}}, + {string(keyType) + "_probe-enabled", args{keyType, container, cephv1.CephClusterHealthCheckSpec{}}, container}, + {string(keyType) + "_probe-disabled", args{keyType, container, cephv1.CephClusterHealthCheckSpec{LivenessProbe: l}}, v1.Container{}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -74,6 +74,94 @@ func configLivenessProbeHelper(t *testing.T, keyType cephv1.KeyType) { } } +func TestConfigureStartupProbe(t *testing.T) { + keyTypes := []cephv1.KeyType{ + cephv1.KeyMds, + cephv1.KeyMon, + cephv1.KeyMgr, + cephv1.KeyOSD, + } + + for _, keyType := range keyTypes { + configStartupProbeHelper(t, keyType) + } + + t.Run("integration check: configured probes should override values", func(t *testing.T) { + defaultProbe := &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/", + Port: intstr.FromInt(8443), + }, + }, + } + userProbe := &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/custom/path", + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 999, + TimeoutSeconds: 888, + PeriodSeconds: 777, + SuccessThreshold: 666, + FailureThreshold: 555, + } + + healthCheckSpec := cephv1.CephClusterHealthCheckSpec{ + StartupProbe: map[cephv1.KeyType]*cephv1.ProbeSpec{ + cephv1.KeyMon: { + Disabled: false, + Probe: userProbe, + }, + }, + } + + container := v1.Container{StartupProbe: defaultProbe} + + got := ConfigureStartupProbe(cephv1.KeyMon, container, healthCheckSpec) + // the resultant container's startup probe should have been overridden, but the handler + // should always be the rook-given default + expectedProbe := *userProbe + expectedProbe.Handler = defaultProbe.Handler + assert.Equal(t, &expectedProbe, got.StartupProbe) + }) +} + +func configStartupProbeHelper(t *testing.T, keyType cephv1.KeyType) { + p := &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/", + Port: intstr.FromInt(8080), + }, + }, + } + container := v1.Container{StartupProbe: p} + l := map[cephv1.KeyType]*cephv1.ProbeSpec{keyType: {Disabled: true}} + type args struct { + daemon cephv1.KeyType + container v1.Container + healthCheck cephv1.CephClusterHealthCheckSpec + } + tests := []struct { + name string + args args + want v1.Container + }{ + {string(keyType) + "_probe-enabled", args{keyType, container, cephv1.CephClusterHealthCheckSpec{}}, container}, + {string(keyType) + "_probe-disabled", args{keyType, container, cephv1.CephClusterHealthCheckSpec{StartupProbe: l}}, v1.Container{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ConfigureStartupProbe(tt.args.daemon, tt.args.container, tt.args.healthCheck); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ConfigureStartupProbe() = %v, want %v", got, tt.want) + } + }) + } +} + func TestGetProbeWithDefaults(t *testing.T) { t.Run("using default probe", func(t *testing.T) { currentProb := &v1.Probe{ diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index aab9710cd300..e7b9b51cb956 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -43,18 +43,19 @@ import ( const ( // ConfigInitContainerName is the name which is given to the config initialization container // in all Ceph pods. - ConfigInitContainerName = "config-init" - logVolumeName = "rook-ceph-log" - volumeMountSubPath = "data" - crashVolumeName = "rook-ceph-crash" - daemonSocketDir = "/run/ceph" - initialDelaySecondsNonOSDDaemon int32 = 10 - initialDelaySecondsOSDDaemon int32 = 45 - logCollector = "log-collector" - DaemonIDLabel = "ceph_daemon_id" - daemonTypeLabel = "ceph_daemon_type" - ExternalMgrAppName = "rook-ceph-mgr-external" - ServiceExternalMetricName = "http-external-metrics" + ConfigInitContainerName = "config-init" + logVolumeName = "rook-ceph-log" + volumeMountSubPath = "data" + crashVolumeName = "rook-ceph-crash" + daemonSocketDir = "/run/ceph" + livenessProbeInitialDelaySeconds int32 = 10 + startupProbeFailuresDaemonDefault int32 = 6 // multiply by 10 = effective startup timeout + startupProbeFailuresDaemonOSD int32 = 9 // multiply by 10 = effective startup timeout + logCollector = "log-collector" + DaemonIDLabel = "ceph_daemon_id" + daemonTypeLabel = "ceph_daemon_type" + ExternalMgrAppName = "rook-ceph-mgr-external" + ServiceExternalMetricName = "http-external-metrics" ) type daemonConfig struct { @@ -554,13 +555,10 @@ func StoredLogAndCrashVolumeMount(varLogCephDir, varLibCephCrashDir string) []v1 } } -// GenerateLivenessProbeExecDaemon makes sure a daemon has a socket and that it can be called and returns 0 +// GenerateLivenessProbeExecDaemon generates a liveness probe that makes sure a daemon has a socket, +// that it can be called, and that it returns 0 func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { confDaemon := getDaemonConfig(daemonType, daemonID) - initialDelaySeconds := initialDelaySecondsNonOSDDaemon - if daemonType == config.OsdType { - initialDelaySeconds = initialDelaySecondsOSDDaemon - } return &v1.Probe{ Handler: v1.Handler{ @@ -579,8 +577,28 @@ func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { }, }, }, - InitialDelaySeconds: initialDelaySeconds, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + } +} + +// GenerateStartupProbeExecDaemon generates a startup probe that makes sure a daemon has a socket, +// that it can be called, and that it returns 0 +func GenerateStartupProbeExecDaemon(daemonType, daemonID string) *v1.Probe { + // startup probe is the same as the liveness probe, but with modified thresholds + probe := GenerateLivenessProbeExecDaemon(daemonType, daemonID) + + // these are hardcoded to 10 so that the failure threshold can be easily multiplied by 10 to + // give the effective startup timeout + probe.InitialDelaySeconds = 10 + probe.PeriodSeconds = 10 + + if daemonType == config.OsdType { + probe.FailureThreshold = startupProbeFailuresDaemonOSD + } else { + probe.FailureThreshold = startupProbeFailuresDaemonDefault } + + return probe } func getDaemonConfig(daemonType, daemonID string) *daemonConfig { diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 6efc0eeeeefa..aba1bb062261 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -158,12 +158,11 @@ func TestGenerateLivenessProbeExecDaemon(t *testing.T) { } assert.Equal(t, expectedCommand, probe.Handler.Exec.Command) - // it's an OSD the delay must be 45 - assert.Equal(t, initialDelaySecondsOSDDaemon, probe.InitialDelaySeconds) + assert.Equal(t, livenessProbeInitialDelaySeconds, probe.InitialDelaySeconds) // test with a mon so the delay should be 10 probe = GenerateLivenessProbeExecDaemon(config.MonType, "a") - assert.Equal(t, initialDelaySecondsNonOSDDaemon, probe.InitialDelaySeconds) + assert.Equal(t, livenessProbeInitialDelaySeconds, probe.InitialDelaySeconds) } func TestDaemonFlags(t *testing.T) { diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 3a8117d590dd..a0cd1f5bb907 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -44,6 +44,7 @@ const ( func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, namespace string) (*apps.Deployment, error) { mdsContainer := c.makeMdsDaemonContainer(mdsConfig) + mdsContainer = config.ConfigureStartupProbe(cephv1.KeyMds, mdsContainer, c.clusterSpec.HealthCheck) mdsContainer = config.ConfigureLivenessProbe(cephv1.KeyMds, mdsContainer, c.clusterSpec.HealthCheck) podSpec := v1.PodTemplateSpec{ @@ -147,6 +148,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { Env: append(controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), k8sutil.PodIPEnvVar(podIPEnvVar)), Resources: c.fs.Spec.MetadataServer.Resources, SecurityContext: controller.PodSecurityContext(), + StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MdsType, mdsConfig.DaemonID), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MdsType, mdsConfig.DaemonID), WorkingDir: config.VarLogCephDir, } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index f54fd30940ed..25656b497a4a 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -281,12 +281,15 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { ), Env: controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), Resources: c.store.Spec.Gateway.Resources, + StartupProbe: c.defaultStartupProbe(), LivenessProbe: c.defaultLivenessProbe(), ReadinessProbe: c.defaultReadinessProbe(), SecurityContext: controller.PodSecurityContext(), WorkingDir: cephconfig.VarLogCephDir, } + // If the startup probe is enabled + configureStartupProbe(&container, c.store.Spec.HealthCheck) // If the liveness probe is enabled configureLivenessProbe(&container, c.store.Spec.HealthCheck) // If the readiness probe is enabled @@ -377,6 +380,21 @@ func configureReadinessProbe(container *v1.Container, healthCheck cephv1.BucketH } } +func configureStartupProbe(container *v1.Container, healthCheck cephv1.BucketHealthCheckSpec) { + if ok := healthCheck.StartupProbe; ok != nil { + if !healthCheck.StartupProbe.Disabled { + probe := healthCheck.StartupProbe.Probe + // If the spec value is empty, let's use a default + if probe != nil { + // Set the startup probe on the container to overwrite the default probe created by Rook + container.StartupProbe = cephconfig.GetProbeWithDefaults(probe, container.StartupProbe) + } + } else { + container.StartupProbe = nil + } + } +} + func (c *clusterConfig) defaultLivenessProbe() *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ @@ -401,6 +419,21 @@ func (c *clusterConfig) defaultReadinessProbe() *v1.Probe { } } +func (c *clusterConfig) defaultStartupProbe() *v1.Probe { + return &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: readinessProbePath, + Port: c.generateProbePort(), + Scheme: c.generateReadinessProbeScheme(), + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 10, + FailureThreshold: 18, + } +} + func (c *clusterConfig) generateReadinessProbeScheme() v1.URIScheme { // Default to HTTP uriScheme := v1.URISchemeHTTP