diff --git a/pkg/internal/testing/addr/manager.go b/pkg/internal/testing/addr/manager.go
index caaafa2627..2ce884dd7c 100644
--- a/pkg/internal/testing/addr/manager.go
+++ b/pkg/internal/testing/addr/manager.go
@@ -42,16 +42,30 @@ var (
 )
 
 func init() {
-	baseDir, err := os.UserCacheDir()
+	var err error
+	cacheDir, err = tryUserCacheDir()
 	if err != nil {
-		baseDir = os.TempDir()
+		// Either we didn't get a cache directory, or we can't use it
+		cacheDir, err = tryCacheInDir(os.TempDir())
 	}
-	cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
-	if err := os.MkdirAll(cacheDir, 0750); err != nil {
+	if err != nil {
 		panic(err)
 	}
 }
 
+func tryUserCacheDir() (string, error) {
+	baseDir, err := os.UserCacheDir()
+	if err != nil {
+		return "", err
+	}
+	return tryCacheInDir(baseDir)
+}
+
+func tryCacheInDir(dir string) (string, error) {
+	dir = filepath.Join(dir, "kubebuilder-envtest")
+	return dir, os.MkdirAll(dir, 0o750)
+}
+
 type portCache struct{}
 
 func (c *portCache) add(port int) (bool, error) {