Skip to content

Commit

Permalink
Adds Daprd option --dapr-block-shutdown-duration (#7268)
Browse files Browse the repository at this point in the history 
* Adds Daprd option `--block-shutdown-seconds`

Closes #4313
Docs: dapr/docs#3893

PR adds the `--block-shutdown-seconds` CLI flag and corresponding
`dapr.io/block-shutdown-seconds` Kubernetes annotation which configures
Daprd to block the graceful shutdown procedure until _either_, the
block shutdown seconds has elapsed _or_ the application has become
unhealthy, as according to the normal app health status.

By default, this option is unset, and therefore there is no effect to
the current behaviour of graceful shutdown. When set, Daprd will
block the interrupt signal cascading into runtime until the above
requirements have been met.

The framework process `Cleanup` order has been reversed to mimic
`t.Cleanup` and allow the `logline` process to function correctly.

Signed-off-by: joshvanl <me@joshvanl.dev>

* Revert if check on killing process exec proc cleanup

Signed-off-by: joshvanl <me@joshvanl.dev>

* Revert error ignore of processes already killed in unix

Signed-off-by: joshvanl <me@joshvanl.dev>

* Skip shutdown/graceful/block/healthy on windows.

* Skip shutdown/block/unhealthy test on windows.

* Linting

Signed-off-by: joshvanl <me@joshvanl.dev>

* Updates `dapr-block-shutdown-seconds` to `dapr-block-shutdown-duration`

Signed-off-by: joshvanl <me@joshvanl.dev>

---------

Signed-off-by: joshvanl <me@joshvanl.dev>
Co-authored-by: Loong Dai <long.dai@intel.com>
  • Loading branch information
@JoshVanL @daixiang0
JoshVanL and daixiang0 committed Dec 6, 2023
1 parent 9758d99 commit ed34172
Show file tree
Hide file tree
Showing 26 changed files with 1,029 additions and 84 deletions.
2 changes: 2 additions & 0 deletions cmd/daprd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ func main() {
UnixDomainSocket: opts.UnixDomainSocket,
DaprHTTPReadBufferSize: opts.DaprHTTPReadBufferSize,
DaprGracefulShutdownSeconds: opts.DaprGracefulShutdownSeconds,
DaprBlockShutdownDuration: opts.DaprBlockShutdownDuration,
DisableBuiltinK8sSecretStore: opts.DisableBuiltinK8sSecretStore,
EnableAppHealthCheck: opts.EnableAppHealthCheck,
AppHealthCheckPath: opts.AppHealthCheckPath,
Expand All @@ -175,4 +176,5 @@ func main() {
if err != nil {
log.Fatalf("Fatal error from runtime: %s", err)
}
log.Info("Daprd shutdown gracefully")
}
8 changes: 8 additions & 0 deletions cmd/daprd/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type Options struct {
DaprPublicPort string
AppPort string
DaprGracefulShutdownSeconds int
DaprBlockShutdownDuration *time.Duration
PlacementServiceHostAddr string
DaprAPIListenAddresses string
AppHealthProbeInterval int
Expand All @@ -79,6 +80,8 @@ func New(args []string) *Options {
EnableAPILogging: new(bool),
}

var blockShutdownDuration time.Duration

flag.StringVar(&opts.Mode, "mode", string(modes.StandaloneMode), "Runtime mode for Dapr")
flag.StringVar(&opts.DaprHTTPPort, "dapr-http-port", strconv.Itoa(runtime.DefaultDaprHTTPPort), "HTTP port for Dapr API to listen on")
flag.StringVar(&opts.DaprAPIListenAddresses, "dapr-listen-addresses", runtime.DefaultAPIListenAddress, "One or more addresses for the Dapr API to listen on, CSV limited")
Expand Down Expand Up @@ -109,6 +112,7 @@ func New(args []string) *Options {
flag.StringVar(&opts.UnixDomainSocket, "unix-domain-socket", "", "Path to a unix domain socket dir mount. If specified, Dapr API servers will use Unix Domain Sockets")
flag.IntVar(&opts.DaprHTTPReadBufferSize, "dapr-http-read-buffer-size", runtime.DefaultReadBufferSize, "Increasing max size of read buffer in KB to handle sending multi-KB headers")
flag.IntVar(&opts.DaprGracefulShutdownSeconds, "dapr-graceful-shutdown-seconds", int(runtime.DefaultGracefulShutdownDuration/time.Second), "Graceful shutdown time in seconds")
flag.DurationVar(&blockShutdownDuration, "dapr-block-shutdown-duration", 0, "If enabled, will block graceful shutdown after terminate signal is received until either the given duration has elapsed or the app reports unhealthy. Disabled by default")
flag.BoolVar(opts.EnableAPILogging, "enable-api-logging", false, "Enable API logging for API calls")
flag.BoolVar(&opts.DisableBuiltinK8sSecretStore, "disable-builtin-k8s-secret-store", false, "Disable the built-in Kubernetes Secret Store")
flag.BoolVar(&opts.EnableAppHealthCheck, "enable-app-health-check", false, "Enable health checks for the application using the protocol defined with app-protocol")
Expand Down Expand Up @@ -150,6 +154,10 @@ func New(args []string) *Options {
}
}

if isFlagPassed("dapr-block-shutdown-duration") {
opts.DaprBlockShutdownDuration = &blockShutdownDuration
}

return &opts
}

Expand Down
1 change: 1 addition & 0 deletions pkg/injector/annotations/annotations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ const (
KeyHTTPMaxRequestSize = "dapr.io/http-max-request-size"
KeyHTTPReadBufferSize = "dapr.io/http-read-buffer-size"
KeyGracefulShutdownSeconds = "dapr.io/graceful-shutdown-seconds"
KeyBlockShutdownDuration = "dapr.io/block-shutdown-duration"
KeyEnableAPILogging = "dapr.io/enable-api-logging"
KeyUnixDomainSocketPath = "dapr.io/unix-domain-socket-path"
KeyVolumeMountsReadOnly = "dapr.io/volume-mounts"
Expand Down
103 changes: 52 additions & 51 deletions pkg/injector/patcher/sidecar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,57 +56,58 @@ type SidecarConfig struct {
SidecarInternalGRPCPort int32 `default:"50002"`
SidecarPublicPort int32 `default:"3501"`

Enabled bool `annotation:"dapr.io/enabled"`
AppPort int32 `annotation:"dapr.io/app-port"`
Config string `annotation:"dapr.io/config"`
AppProtocol string `annotation:"dapr.io/app-protocol" default:"http"`
AppSSL bool `annotation:"dapr.io/app-ssl"` // TODO: Deprecated in Dapr 1.11; remove in a future Dapr version
AppID string `annotation:"dapr.io/app-id"`
EnableProfiling bool `annotation:"dapr.io/enable-profiling"`
LogLevel string `annotation:"dapr.io/log-level" default:"info"`
APITokenSecret string `annotation:"dapr.io/api-token-secret"`
AppTokenSecret string `annotation:"dapr.io/app-token-secret"`
LogAsJSON bool `annotation:"dapr.io/log-as-json"`
AppMaxConcurrency *int `annotation:"dapr.io/app-max-concurrency"`
EnableMetrics bool `annotation:"dapr.io/enable-metrics" default:"true"`
SidecarMetricsPort int32 `annotation:"dapr.io/metrics-port" default:"9090"`
EnableDebug bool `annotation:"dapr.io/enable-debug" default:"false"`
SidecarDebugPort int32 `annotation:"dapr.io/debug-port" default:"40000"`
Env string `annotation:"dapr.io/env"`
SidecarCPURequest string `annotation:"dapr.io/sidecar-cpu-request"`
SidecarCPULimit string `annotation:"dapr.io/sidecar-cpu-limit"`
SidecarMemoryRequest string `annotation:"dapr.io/sidecar-memory-request"`
SidecarMemoryLimit string `annotation:"dapr.io/sidecar-memory-limit"`
SidecarListenAddresses string `annotation:"dapr.io/sidecar-listen-addresses" default:"[::1],127.0.0.1"`
SidecarLivenessProbeDelaySeconds int32 `annotation:"dapr.io/sidecar-liveness-probe-delay-seconds" default:"3"`
SidecarLivenessProbeTimeoutSeconds int32 `annotation:"dapr.io/sidecar-liveness-probe-timeout-seconds" default:"3"`
SidecarLivenessProbePeriodSeconds int32 `annotation:"dapr.io/sidecar-liveness-probe-period-seconds" default:"6"`
SidecarLivenessProbeThreshold int32 `annotation:"dapr.io/sidecar-liveness-probe-threshold" default:"3"`
SidecarReadinessProbeDelaySeconds int32 `annotation:"dapr.io/sidecar-readiness-probe-delay-seconds" default:"3"`
SidecarReadinessProbeTimeoutSeconds int32 `annotation:"dapr.io/sidecar-readiness-probe-timeout-seconds" default:"3"`
SidecarReadinessProbePeriodSeconds int32 `annotation:"dapr.io/sidecar-readiness-probe-period-seconds" default:"6"`
SidecarReadinessProbeThreshold int32 `annotation:"dapr.io/sidecar-readiness-probe-threshold" default:"3"`
SidecarImage string `annotation:"dapr.io/sidecar-image"`
SidecarSeccompProfileType string `annotation:"dapr.io/sidecar-seccomp-profile-type"`
HTTPMaxRequestSize *int `annotation:"dapr.io/http-max-request-size"`
HTTPReadBufferSize *int `annotation:"dapr.io/http-read-buffer-size"`
GracefulShutdownSeconds int `annotation:"dapr.io/graceful-shutdown-seconds" default:"-1"`
EnableAPILogging *bool `annotation:"dapr.io/enable-api-logging"`
UnixDomainSocketPath string `annotation:"dapr.io/unix-domain-socket-path"`
VolumeMounts string `annotation:"dapr.io/volume-mounts"`
VolumeMountsRW string `annotation:"dapr.io/volume-mounts-rw"`
DisableBuiltinK8sSecretStore bool `annotation:"dapr.io/disable-builtin-k8s-secret-store"`
EnableAppHealthCheck bool `annotation:"dapr.io/enable-app-health-check"`
AppHealthCheckPath string `annotation:"dapr.io/app-health-check-path" default:"/healthz"`
AppHealthProbeInterval int32 `annotation:"dapr.io/app-health-probe-interval" default:"5"` // In seconds
AppHealthProbeTimeout int32 `annotation:"dapr.io/app-health-probe-timeout" default:"500"` // In milliseconds
AppHealthThreshold int32 `annotation:"dapr.io/app-health-threshold" default:"3"`
PlacementAddress string `annotation:"dapr.io/placement-host-address"`
PluggableComponents string `annotation:"dapr.io/pluggable-components"`
PluggableComponentsSocketsFolder string `annotation:"dapr.io/pluggable-components-sockets-folder"`
ComponentContainer string `annotation:"dapr.io/component-container"`
InjectPluggableComponents bool `annotation:"dapr.io/inject-pluggable-components"`
AppChannelAddress string `annotation:"dapr.io/app-channel-address"`
Enabled bool `annotation:"dapr.io/enabled"`
AppPort int32 `annotation:"dapr.io/app-port"`
Config string `annotation:"dapr.io/config"`
AppProtocol string `annotation:"dapr.io/app-protocol" default:"http"`
AppSSL bool `annotation:"dapr.io/app-ssl"` // TODO: Deprecated in Dapr 1.11; remove in a future Dapr version
AppID string `annotation:"dapr.io/app-id"`
EnableProfiling bool `annotation:"dapr.io/enable-profiling"`
LogLevel string `annotation:"dapr.io/log-level" default:"info"`
APITokenSecret string `annotation:"dapr.io/api-token-secret"`
AppTokenSecret string `annotation:"dapr.io/app-token-secret"`
LogAsJSON bool `annotation:"dapr.io/log-as-json"`
AppMaxConcurrency *int `annotation:"dapr.io/app-max-concurrency"`
EnableMetrics bool `annotation:"dapr.io/enable-metrics" default:"true"`
SidecarMetricsPort int32 `annotation:"dapr.io/metrics-port" default:"9090"`
EnableDebug bool `annotation:"dapr.io/enable-debug" default:"false"`
SidecarDebugPort int32 `annotation:"dapr.io/debug-port" default:"40000"`
Env string `annotation:"dapr.io/env"`
SidecarCPURequest string `annotation:"dapr.io/sidecar-cpu-request"`
SidecarCPULimit string `annotation:"dapr.io/sidecar-cpu-limit"`
SidecarMemoryRequest string `annotation:"dapr.io/sidecar-memory-request"`
SidecarMemoryLimit string `annotation:"dapr.io/sidecar-memory-limit"`
SidecarListenAddresses string `annotation:"dapr.io/sidecar-listen-addresses" default:"[::1],127.0.0.1"`
SidecarLivenessProbeDelaySeconds int32 `annotation:"dapr.io/sidecar-liveness-probe-delay-seconds" default:"3"`
SidecarLivenessProbeTimeoutSeconds int32 `annotation:"dapr.io/sidecar-liveness-probe-timeout-seconds" default:"3"`
SidecarLivenessProbePeriodSeconds int32 `annotation:"dapr.io/sidecar-liveness-probe-period-seconds" default:"6"`
SidecarLivenessProbeThreshold int32 `annotation:"dapr.io/sidecar-liveness-probe-threshold" default:"3"`
SidecarReadinessProbeDelaySeconds int32 `annotation:"dapr.io/sidecar-readiness-probe-delay-seconds" default:"3"`
SidecarReadinessProbeTimeoutSeconds int32 `annotation:"dapr.io/sidecar-readiness-probe-timeout-seconds" default:"3"`
SidecarReadinessProbePeriodSeconds int32 `annotation:"dapr.io/sidecar-readiness-probe-period-seconds" default:"6"`
SidecarReadinessProbeThreshold int32 `annotation:"dapr.io/sidecar-readiness-probe-threshold" default:"3"`
SidecarImage string `annotation:"dapr.io/sidecar-image"`
SidecarSeccompProfileType string `annotation:"dapr.io/sidecar-seccomp-profile-type"`
HTTPMaxRequestSize *int `annotation:"dapr.io/http-max-request-size"`
HTTPReadBufferSize *int `annotation:"dapr.io/http-read-buffer-size"`
GracefulShutdownSeconds int `annotation:"dapr.io/graceful-shutdown-seconds" default:"-1"`
BlockShutdownDuration *string `annotation:"dapr.io/block-shutdown-duration"`
EnableAPILogging *bool `annotation:"dapr.io/enable-api-logging"`
UnixDomainSocketPath string `annotation:"dapr.io/unix-domain-socket-path"`
VolumeMounts string `annotation:"dapr.io/volume-mounts"`
VolumeMountsRW string `annotation:"dapr.io/volume-mounts-rw"`
DisableBuiltinK8sSecretStore bool `annotation:"dapr.io/disable-builtin-k8s-secret-store"`
EnableAppHealthCheck bool `annotation:"dapr.io/enable-app-health-check"`
AppHealthCheckPath string `annotation:"dapr.io/app-health-check-path" default:"/healthz"`
AppHealthProbeInterval int32 `annotation:"dapr.io/app-health-probe-interval" default:"5"` // In seconds
AppHealthProbeTimeout int32 `annotation:"dapr.io/app-health-probe-timeout" default:"500"` // In milliseconds
AppHealthThreshold int32 `annotation:"dapr.io/app-health-threshold" default:"3"`
PlacementAddress string `annotation:"dapr.io/placement-host-address"`
PluggableComponents string `annotation:"dapr.io/pluggable-components"`
PluggableComponentsSocketsFolder string `annotation:"dapr.io/pluggable-components-sockets-folder"`
ComponentContainer string `annotation:"dapr.io/component-container"`
InjectPluggableComponents bool `annotation:"dapr.io/inject-pluggable-components"`
AppChannelAddress string `annotation:"dapr.io/app-channel-address"`

pod *corev1.Pod
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/injector/patcher/sidecar_container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,10 @@ func (c *SidecarConfig) getSidecarContainer(opts getSidecarContainerOpts) (*core
args = append(args, "--unix-domain-socket", injectorConsts.UnixDomainSocketDaprdPath)
}

if c.BlockShutdownDuration != nil {
args = append(args, "--dapr-block-shutdown-duration", *c.BlockShutdownDuration)
}

// When debugging is enabled, we need to override the command and the flags
if c.EnableDebug {
ports = append(ports, corev1.ContainerPort{
Expand Down
21 changes: 21 additions & 0 deletions pkg/injector/patcher/sidecar_container_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,27 @@ func TestGetSidecarContainer(t *testing.T) {
},
}))

t.Run("block shutdown duration", testSuiteGenerator([]testCase{
{
name: "default to empty",
annotations: map[string]string{},
assertFn: func(t *testing.T, container *corev1.Container) {
args := strings.Join(container.Args, " ")
assert.NotContains(t, args, "--dapr-block-shutdown-duration")
},
},
{
name: "add a block shutdown duration",
annotations: map[string]string{
"dapr.io/block-shutdown-duration": "3s",
},
assertFn: func(t *testing.T, container *corev1.Container) {
args := strings.Join(container.Args, " ")
assert.Contains(t, args, "--dapr-block-shutdown-duration 3s")
},
},
}))

t.Run("sidecar image", testSuiteGenerator([]testCase{
{
name: "no annotation",
Expand Down
7 changes: 5 additions & 2 deletions pkg/runtime/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ type Config struct {
DaprPublicPort string
ApplicationPort string
DaprGracefulShutdownSeconds int
DaprBlockShutdownDuration *time.Duration
PlacementServiceHostAddr string
DaprAPIListenAddresses string
AppHealthProbeInterval int
Expand Down Expand Up @@ -129,6 +130,7 @@ type internalConfig struct {
unixDomainSocket string
readBufferSize int
gracefulShutdownDuration time.Duration
blockShutdownDuration *time.Duration
enableAPILogging *bool
disableBuiltinK8sSecretStore bool
config []string
Expand Down Expand Up @@ -284,8 +286,9 @@ func (c *Config) toInternal() (*internalConfig, error) {
HealthCheckHTTPPath: c.AppHealthCheckPath,
MaxConcurrency: c.AppMaxConcurrency,
},
registry: registry.New(c.Registry),
metricsExporter: metrics.NewExporterWithOptions(log, metrics.DefaultMetricNamespace, c.Metrics),
registry: registry.New(c.Registry),
metricsExporter: metrics.NewExporterWithOptions(log, metrics.DefaultMetricNamespace, c.Metrics),
blockShutdownDuration: c.DaprBlockShutdownDuration,
}

if len(intc.standalone.ResourcesPath) == 0 && c.ComponentsPath != "" {
Expand Down
3 changes: 3 additions & 0 deletions pkg/runtime/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ func TestParsePlacementAddr(t *testing.T) {
func Test_toInternal(t *testing.T) {
cfg := defaultTestConfig()

var nilDuration *time.Duration

intc, err := cfg.toInternal()
require.NoError(t, err)

Expand All @@ -81,6 +83,7 @@ func Test_toInternal(t *testing.T) {
assert.Equal(t, "", intc.unixDomainSocket)
assert.Equal(t, 4, intc.readBufferSize)
assert.Equal(t, time.Second, intc.gracefulShutdownDuration)
assert.Equal(t, nilDuration, intc.blockShutdownDuration)
assert.Equal(t, ptr.Of(true), intc.enableAPILogging)
assert.True(t, intc.disableBuiltinK8sSecretStore)
assert.Equal(t, "1.1.1.1", intc.appConnectionConfig.ChannelAddress)
Expand Down

0 comments on commit ed34172

Please sign in to comment.