Configure XFS max-retry=1 max-timeout=5 for EIO/ENOSPC errors (#753)

Signed-off-by: Bala.FA <bala@minio.io>
minio · Apr 13, 2023 · 8e8a598 · 8e8a598
1 parent de23fc4
commit 8e8a598
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 4 deletions.
diff --git a/apparmor.profile b/apparmor.profile
@@ -34,6 +34,7 @@ profile directpv flags=(attach_disconnected,mediate_deleted) {
   /var/lib/directpv/** w,
   /var/lib/kubelet/** w,
   /csi/** w,
+  /sys/fs/xfs/**/error/metadata/{EIO,ENOSPC}/retry_timeout_seconds rw,
 
   # only a limited set of binaries can be executed
   /usr/sbin/mkfs ix,

diff --git a/pkg/installer/daemonset.go b/pkg/installer/daemonset.go
@@ -111,7 +111,7 @@ func getVolumesAndMounts(pluginSocketDir string) (volumes []corev1.Volume, volum
 		newVolumeMount(volumeNameMountpointDir, kubeletDirPath+"/pods", corev1.MountPropagationBidirectional, false),
 		newVolumeMount(volumeNamePluginDir, kubeletDirPath+"/plugins", corev1.MountPropagationBidirectional, false),
 		newVolumeMount(volumeNameAppRootDir, appRootDir, corev1.MountPropagationBidirectional, false),
-		newVolumeMount(volumeNameSysDir, volumePathSysDir, corev1.MountPropagationBidirectional, true),
+		newVolumeMount(volumeNameSysDir, volumePathSysDir, corev1.MountPropagationBidirectional, false),
 		newVolumeMount(volumeNameDevDir, volumePathDevDir, corev1.MountPropagationHostToContainer, true),
 		newVolumeMount(volumeNameRunUdevData, volumePathRunUdevData, corev1.MountPropagationBidirectional, true),
 		newVolumeMount(volumeNameLegacyAppRootDir, legacyAppRootDir, corev1.MountPropagationBidirectional, false),

diff --git a/pkg/installer/psp.go b/pkg/installer/psp.go
@@ -155,7 +155,7 @@ func createPodSecurityPolicy(ctx context.Context, args *Args) (err error) {
 			Volumes:             []policy.FSType{policy.HostPath},
 			AllowedHostPaths: []policy.AllowedHostPath{
 				{PathPrefix: "/proc", ReadOnly: true},
-				{PathPrefix: volumePathSysDir, ReadOnly: true},
+				{PathPrefix: volumePathSysDir},
 				{PathPrefix: consts.UdevDataDir, ReadOnly: true},
 				{PathPrefix: consts.AppRootDir},
 				{PathPrefix: socketDir},

diff --git a/pkg/xfs/mount_linux.go b/pkg/xfs/mount_linux.go
@@ -21,16 +21,44 @@ package xfs
 import (
 	"errors"
 	"os"
+	"path"
 
 	"github.com/minio/directpv/pkg/sys"
+	"k8s.io/klog/v2"
 )
 
 func mount(device, target string) error {
-	if err := os.Mkdir(target, 0o777); err != nil && !errors.Is(err, os.ErrExist) {
+	if err := sys.Mkdir(target, 0o777); err != nil && !errors.Is(err, os.ErrExist) {
 		return err
 	}
 
-	return sys.Mount(device, target, "xfs", []string{"noatime"}, "prjquota")
+	if err := sys.Mount(device, target, "xfs", []string{"noatime"}, "prjquota"); err != nil {
+		return err
+	}
+
+	name := path.Base(device)
+	if name == "/" || name == "." {
+		klog.Errorf("unable to get device name from device %v", device)
+		return nil
+	}
+
+	if err := os.WriteFile("/sys/fs/xfs/"+name+"/error/metadata/EIO/max_retries", []byte("1"), 0o644); err != nil {
+		klog.ErrorS(err, "unable to set EIO max_retires device", "name", name)
+	}
+
+	if err := os.WriteFile("/sys/fs/xfs/"+name+"/error/metadata/EIO/retry_timeout_seconds", []byte("5"), 0o644); err != nil {
+		klog.ErrorS(err, "unable to set EIO retry_timeout_seconds for device", "name", name)
+	}
+
+	if err := os.WriteFile("/sys/fs/xfs/"+name+"/error/metadata/ENOSPC/max_retries", []byte("1"), 0o644); err != nil {
+		klog.ErrorS(err, "unable to set ENOSPC max_retires device", "name", name)
+	}
+
+	if err := os.WriteFile("/sys/fs/xfs/"+name+"/error/metadata/ENOSPC/retry_timeout_seconds", []byte("5"), 0o644); err != nil {
+		klog.ErrorS(err, "unable to set ENOSPC retry_timeout_seconds for device", "name", name)
+	}
+
+	return nil
 }
 
 func bindMount(source, target string, readOnly bool) error {