diff --git a/pkg/internal/testing/addr/manager.go b/pkg/internal/testing/addr/manager.go
index caaafa2627..a584d2d011 100644
--- a/pkg/internal/testing/addr/manager.go
+++ b/pkg/internal/testing/addr/manager.go
@@ -43,11 +43,17 @@ var (
 
 func init() {
 	baseDir, err := os.UserCacheDir()
+	if err == nil {
+		cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
+		err = os.MkdirAll(cacheDir, 0o750)
+	}
 	if err != nil {
+		// Either we didn't get a cache directory, or we can't use it
 		baseDir = os.TempDir()
+		cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
+		err = os.MkdirAll(cacheDir, 0o750)
 	}
-	cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
-	if err := os.MkdirAll(cacheDir, 0750); err != nil {
+	if err != nil {
 		panic(err)
 	}
 }