test: Use daemonset count to calculate maxPods and podsPerCore (#2497)

Author: jonathan-innis

Makefile

@@ -49,7 +49,7 @@ battletest: ## Run randomized, racing, code coveraged, tests
 
 e2etests: ## Run the e2e suite against your local cluster
 	go clean -testcache
-	go test -p 1 -timeout 180m -v ./test/suites/... -run=${TEST_FILTER}
+	CLUSTER_NAME=${CLUSTER_NAME} go test -p 1 -timeout 180m -v ./test/suites/... -run=${TEST_FILTER}
 
 benchmark:
 	go test -tags=test_performance -run=NoTests -bench=. ./...

test/suites/integration/kubelet_config_test.go

@@ -16,6 +16,8 @@ package integration_test
 
 import (
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"knative.dev/pkg/ptr"
 
@@ -27,15 +29,22 @@ import (
 
 var _ = Describe("KubeletConfiguration Overrides", func() {
 	It("should schedule pods onto separate nodes when maxPods is set", func() {
+		// Get the total number of daemonsets so that we see how many pods will be taken up
+		// by daemonset overhead
+		dsList := &appsv1.DaemonSetList{}
+		Expect(env.Client.List(env.Context, dsList)).To(Succeed())
+		dsCount := len(dsList.Items)
+
 		provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{
 			SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
 			SubnetSelector:        map[string]string{"karpenter.sh/discovery": env.ClusterName},
 		}})
+
 		// MaxPods needs to account for the daemonsets that will run on the nodes
 		provisioner := test.Provisioner(test.ProvisionerOptions{
 			ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name},
 			Kubelet: &v1alpha5.KubeletConfiguration{
-				MaxPods: ptr.Int32(3),
+				MaxPods: ptr.Int32(1 + int32(dsCount)),
 			},
 		})
 
@@ -48,6 +57,12 @@ var _ = Describe("KubeletConfiguration Overrides", func() {
 		env.ExpectCreatedNodeCount("==", 3)
 	})
 	It("should schedule pods onto separate nodes when podsPerCore is set", func() {
+		// Get the total number of daemonsets so that we see how many pods will be taken up
+		// by daemonset overhead
+		dsList := &appsv1.DaemonSetList{}
+		Expect(env.Client.List(env.Context, dsList)).To(Succeed())
+		dsCount := len(dsList.Items)
+
 		provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{
 			SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
 			SubnetSelector:        map[string]string{"karpenter.sh/discovery": env.ClusterName},
@@ -57,13 +72,13 @@ var _ = Describe("KubeletConfiguration Overrides", func() {
 		provisioner := test.Provisioner(test.ProvisionerOptions{
 			ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name},
 			Kubelet: &v1alpha5.KubeletConfiguration{
-				PodsPerCore: ptr.Int32(2),
+				PodsPerCore: ptr.Int32(2 + int32(dsCount)),
 			},
 			Requirements: []v1.NodeSelectorRequirement{
 				{
 					Key:      awsv1alpha1.LabelInstanceCPU,
 					Operator: v1.NodeSelectorOpIn,
-					Values:   []string{"2"},
+					Values:   []string{"1"},
 				},
 			},
 		})
@@ -87,13 +102,13 @@ var _ = Describe("KubeletConfiguration Overrides", func() {
 		provisioner := test.Provisioner(test.ProvisionerOptions{
 			ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name},
 			Kubelet: &v1alpha5.KubeletConfiguration{
-				PodsPerCore: ptr.Int32(2),
+				PodsPerCore: ptr.Int32(1),
 			},
 			Requirements: []v1.NodeSelectorRequirement{
 				{
 					Key:      awsv1alpha1.LabelInstanceCPU,
 					Operator: v1.NodeSelectorOpIn,
-					Values:   []string{"2"},
+					Values:   []string{"1"},
 				},
 			},
 		})
@@ -104,6 +119,6 @@ var _ = Describe("KubeletConfiguration Overrides", func() {
 			env.ExpectCreated(pod)
 		}
 		env.EventuallyExpectHealthy(pods...)
-		env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching
+		env.ExpectCreatedNodeCount("==", 1)
 	})
 })