Skip to content

Commit eb6b5ed

Browse files
jonathan-innisjonathan-innis
authoredSep 12, 2022
feat: Added evictionHard, kubeReserved and podsPerCore to Provisioner (#2444)
·
v1.3.3v0.16.2
1 parent 1671850 commit eb6b5ed

20 files changed

+762
-53
lines changed
 

‎charts/karpenter/crds/karpenter.sh_provisioners.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,35 @@ spec:
6262
description: ContainerRuntime is the container runtime to be used
6363
with your worker nodes.
6464
type: string
65+
evictionHard:
66+
additionalProperties:
67+
type: string
68+
description: EvictionHard is the map of signal names to quantities
69+
that define hard eviction thresholds
70+
type: object
71+
kubeReserved:
72+
additionalProperties:
73+
anyOf:
74+
- type: integer
75+
- type: string
76+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
77+
x-kubernetes-int-or-string: true
78+
description: KubeReserved contains resources reserved for Kubernetes
79+
system components.
80+
type: object
6581
maxPods:
6682
description: MaxPods is an override for the maximum number of
6783
pods that can run on a worker node instance.
6884
format: int32
85+
minimum: 0
86+
type: integer
87+
podsPerCore:
88+
description: PodsPerCore is an override for the number of pods
89+
that can run on a worker node instance based on the number of
90+
cpu cores. This value cannot exceed MaxPods, so, if MaxPods
91+
is a lower value, that value will be used.
92+
format: int32
93+
minimum: 0
6994
type: integer
7095
systemReserved:
7196
additionalProperties:

‎pkg/apis/provisioning/v1alpha5/provisioner.go

+14
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,24 @@ type KubeletConfiguration struct {
120120
ContainerRuntime *string `json:"containerRuntime,omitempty"`
121121
// MaxPods is an override for the maximum number of pods that can run on
122122
// a worker node instance.
123+
// +kubebuilder:validation:Minimum:=0
123124
// +optional
124125
MaxPods *int32 `json:"maxPods,omitempty"`
126+
// PodsPerCore is an override for the number of pods that can run on a worker node
127+
// instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if
128+
// MaxPods is a lower value, that value will be used.
129+
// +kubebuilder:validation:Minimum:=0
130+
// +optional
131+
PodsPerCore *int32 `json:"podsPerCore,omitempty"`
125132
// SystemReserved contains resources reserved for OS system daemons and kernel memory.
133+
// +optional
126134
SystemReserved v1.ResourceList `json:"systemReserved,omitempty"`
135+
// KubeReserved contains resources reserved for Kubernetes system components.
136+
// +optional
137+
KubeReserved v1.ResourceList `json:"kubeReserved,omitempty"`
138+
// EvictionHard is the map of signal names to quantities that define hard eviction thresholds
139+
// +optional
140+
EvictionHard map[string]string `json:"evictionHard,omitempty"`
127141
}
128142

129143
// Provisioner is the Schema for the Provisioners API

‎pkg/apis/provisioning/v1alpha5/provisioner_validation.go

+89
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@ import (
1818
"context"
1919
"fmt"
2020
"strconv"
21+
"strings"
2122

2223
"go.uber.org/multierr"
2324
v1 "k8s.io/api/core/v1"
25+
"k8s.io/apimachinery/pkg/api/resource"
2426
"k8s.io/apimachinery/pkg/util/sets"
2527
"k8s.io/apimachinery/pkg/util/validation"
2628
"knative.dev/pkg/apis"
@@ -36,6 +38,22 @@ var (
3638
string(v1.NodeSelectorOpExists),
3739
string(v1.NodeSelectorOpDoesNotExist),
3840
)
41+
42+
SupportedReservedResources = sets.NewString(
43+
v1.ResourceCPU.String(),
44+
v1.ResourceMemory.String(),
45+
v1.ResourceEphemeralStorage.String(),
46+
"pid",
47+
)
48+
49+
SupportedEvictionSignals = sets.NewString(
50+
"memory.available",
51+
"nodefs.available",
52+
"nodefs.inodesFree",
53+
"imagefs.available",
54+
"imagefs.inodesFree",
55+
"pid.available",
56+
)
3957
)
4058

4159
const (
@@ -84,6 +102,7 @@ func (s *ProvisionerSpec) Validate(ctx context.Context) (errs *apis.FieldError)
84102
s.validateLabels(),
85103
s.validateTaints(),
86104
s.validateRequirements(),
105+
s.validateKubeletConfiguration(),
87106
)
88107
}
89108

@@ -173,6 +192,76 @@ func (s *ProvisionerSpec) validateProvider() *apis.FieldError {
173192
return nil
174193
}
175194

195+
func (s *ProvisionerSpec) validateKubeletConfiguration() (errs *apis.FieldError) {
196+
if s.KubeletConfiguration == nil {
197+
return
198+
}
199+
return errs.Also(
200+
s.KubeletConfiguration.validateEvictionHard(),
201+
s.KubeletConfiguration.validateKubeReserved(),
202+
s.KubeletConfiguration.validateSystemReserved(),
203+
)
204+
}
205+
206+
func (kc *KubeletConfiguration) validateKubeReserved() (errs *apis.FieldError) {
207+
if kc.KubeReserved == nil {
208+
return
209+
}
210+
for k, v := range kc.KubeReserved {
211+
if !SupportedReservedResources.Has(k.String()) {
212+
errs = errs.Also(apis.ErrInvalidKeyName(k.String(), "kubeReserved"))
213+
}
214+
if v.Value() < 0 {
215+
errs = errs.Also(apis.ErrInvalidValue(v.String(), fmt.Sprintf(`kubeReserved["%s"]`, k), "Value cannot be a negative resource quantity"))
216+
}
217+
}
218+
return errs
219+
}
220+
221+
func (kc *KubeletConfiguration) validateSystemReserved() (errs *apis.FieldError) {
222+
if kc.SystemReserved == nil {
223+
return
224+
}
225+
for k, v := range kc.SystemReserved {
226+
if !SupportedReservedResources.Has(k.String()) {
227+
errs = errs.Also(apis.ErrInvalidKeyName(k.String(), "systemReserved"))
228+
}
229+
if v.Value() < 0 {
230+
errs = errs.Also(apis.ErrInvalidValue(v.String(), fmt.Sprintf(`systemReserved["%s"]`, k), "Value cannot be a negative resource quantity"))
231+
}
232+
}
233+
return errs
234+
}
235+
236+
func (kc *KubeletConfiguration) validateEvictionHard() (errs *apis.FieldError) {
237+
if kc.EvictionHard == nil {
238+
return
239+
}
240+
for k, v := range kc.EvictionHard {
241+
if !SupportedEvictionSignals.Has(k) {
242+
errs = errs.Also(apis.ErrInvalidKeyName(k, "evictionHard"))
243+
}
244+
if strings.HasSuffix(v, "%") {
245+
p, err := strconv.ParseFloat(strings.Trim(v, "%"), 64)
246+
if err != nil {
247+
errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`evictionHard["%s"]`, k), fmt.Sprintf("Value could not be parsed as a percentage value, %v", err.Error())))
248+
}
249+
if p < 0 {
250+
errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`evictionHard["%s"]`, k), "Percentage values cannot be negative"))
251+
}
252+
if p > 100 {
253+
errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`evictionHard["%s"]`, k), "Percentage values cannot be greater than 100"))
254+
}
255+
} else {
256+
_, err := resource.ParseQuantity(v)
257+
if err != nil {
258+
errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf("evictionHard[%s]", k), fmt.Sprintf("Value could not be parsed as a resource quantity, %v", err.Error())))
259+
}
260+
}
261+
}
262+
return errs
263+
}
264+
176265
func ValidateRequirement(requirement v1.NodeSelectorRequirement) error { //nolint:gocyclo
177266
var errs error
178267
if normalized, ok := NormalizedLabels[requirement.Key]; ok {

‎pkg/apis/provisioning/v1alpha5/suite_test.go

+64
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"testing"
2121

2222
"github.com/aws/aws-sdk-go/aws"
23+
"k8s.io/apimachinery/pkg/api/resource"
2324

2425
"github.com/Pallinder/go-randomdata"
2526
. "github.com/onsi/ginkgo/v2"
@@ -267,4 +268,67 @@ var _ = Describe("Validation", func() {
267268
}
268269
})
269270
})
271+
Context("KubeletConfiguration", func() {
272+
It("should fail on kubeReserved with invalid keys", func() {
273+
provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{
274+
KubeReserved: v1.ResourceList{
275+
v1.ResourcePods: resource.MustParse("2"),
276+
},
277+
}
278+
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
279+
})
280+
It("should fail on systemReserved with invalid keys", func() {
281+
provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{
282+
SystemReserved: v1.ResourceList{
283+
v1.ResourcePods: resource.MustParse("2"),
284+
},
285+
}
286+
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
287+
})
288+
It("should succeed on evictionHard with valid keys", func() {
289+
provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{
290+
EvictionHard: map[string]string{
291+
"memory.available": "5%",
292+
"nodefs.available": "10%",
293+
"nodefs.inodesFree": "15%",
294+
"imagefs.available": "5%",
295+
"imagefs.inodesFree": "5%",
296+
"pid.available": "5%",
297+
},
298+
}
299+
Expect(provisioner.Validate(ctx)).To(Succeed())
300+
})
301+
It("should fail on evictionHard with invalid keys", func() {
302+
provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{
303+
EvictionHard: map[string]string{
304+
"memory": "5%",
305+
},
306+
}
307+
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
308+
})
309+
It("should fail on invalid formatted percentage value in evictionHard", func() {
310+
provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{
311+
EvictionHard: map[string]string{
312+
"memory.available": "5%3",
313+
},
314+
}
315+
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
316+
})
317+
It("should fail on invalid percentage value (too large) in evictionHard", func() {
318+
provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{
319+
EvictionHard: map[string]string{
320+
"memory.available": "110%",
321+
},
322+
}
323+
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
324+
})
325+
It("should fail on invalid quantity value in evictionHard", func() {
326+
provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{
327+
EvictionHard: map[string]string{
328+
"memory.available": "110GB",
329+
},
330+
}
331+
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
332+
})
333+
})
270334
})

‎pkg/apis/provisioning/v1alpha5/zz_generated.deepcopy.go

+19
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/cloudprovider/aws/amifamily/al2.go

+4
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,7 @@ func (a AL2) EphemeralBlockDevice() *string {
9292
func (a AL2) ENILimitedMemoryOverhead() bool {
9393
return true
9494
}
95+
96+
func (a AL2) PodsPerCoreEnabled() bool {
97+
return true
98+
}

‎pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go

+2
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ func (b Bottlerocket) Script() (string, error) {
5353
}
5454
if b.KubeletConfig != nil {
5555
s.Settings.Kubernetes.SystemReserved = resources.StringMap(b.KubeletConfig.SystemReserved)
56+
s.Settings.Kubernetes.KubeReserved = resources.StringMap(b.KubeletConfig.KubeReserved)
57+
s.Settings.Kubernetes.EvictionHard = b.KubeletConfig.EvictionHard
5658
}
5759

5860
s.Settings.Kubernetes.NodeTaints = map[string][]string{}

‎pkg/cloudprovider/aws/amifamily/bootstrap/eksbootstrap.go

+38-3
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,8 @@ import (
2828
"strings"
2929
"sync"
3030

31-
"knative.dev/pkg/ptr"
32-
3331
"github.com/samber/lo"
32+
"knative.dev/pkg/ptr"
3433

3534
"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
3635
)
@@ -46,6 +45,7 @@ const (
4645
MIMEContentTypeHeaderTemplate = "Content-Type: multipart/mixed; boundary=\"%s\""
4746
)
4847

48+
//nolint:gocyclo
4949
func (e EKS) Script() (string, error) {
5050
var caBundleArg string
5151
if e.CABundle != nil {
@@ -59,16 +59,21 @@ func (e EKS) Script() (string, error) {
5959

6060
kubeletExtraArgs := strings.Join([]string{e.nodeLabelArg(), e.nodeTaintArg()}, " ")
6161

62-
// Backwards compatibility for AWSENILimitedPodDensity flag
6362
if e.KubeletConfig != nil && e.KubeletConfig.MaxPods != nil {
6463
userData.WriteString(" \\\n--use-max-pods false")
6564
kubeletExtraArgs += fmt.Sprintf(" --max-pods=%d", ptr.Int32Value(e.KubeletConfig.MaxPods))
6665
} else if !e.AWSENILimitedPodDensity {
6766
userData.WriteString(" \\\n--use-max-pods false")
6867
kubeletExtraArgs += " --max-pods=110"
6968
}
69+
if e.KubeletConfig != nil && e.KubeletConfig.PodsPerCore != nil {
70+
kubeletExtraArgs += fmt.Sprintf(" --pods-per-core=%d", ptr.Int32Value(e.KubeletConfig.PodsPerCore))
71+
}
72+
7073
if e.KubeletConfig != nil {
7174
kubeletExtraArgs += e.systemReservedArg()
75+
kubeletExtraArgs += e.kubeReservedArg()
76+
kubeletExtraArgs += e.evictionThresholdArg()
7277
}
7378
if e.ContainerRuntime != "" {
7479
userData.WriteString(fmt.Sprintf(" \\\n--container-runtime %s", e.ContainerRuntime))
@@ -131,6 +136,36 @@ func (e EKS) systemReservedArg() string {
131136
return ""
132137
}
133138

139+
// kubeReservedArg gets the kubelet-defined arguments for any valid resource
140+
// values that are specified within the kube reserved resource list
141+
func (e EKS) kubeReservedArg() string {
142+
var args []string
143+
if e.KubeletConfig.KubeReserved != nil {
144+
for k, v := range e.KubeletConfig.KubeReserved {
145+
args = append(args, fmt.Sprintf("%v=%v", k.String(), v.String()))
146+
}
147+
}
148+
if len(args) > 0 {
149+
return " --kube-reserved=" + strings.Join(args, ",")
150+
}
151+
return ""
152+
}
153+
154+
// evictionThresholdArg gets the kubelet-defined arguments for eviction
155+
// threshold values that are specified within the eviction threshold list
156+
func (e EKS) evictionThresholdArg() string {
157+
var args []string
158+
if e.KubeletConfig.EvictionHard != nil {
159+
for k, v := range e.KubeletConfig.EvictionHard {
160+
args = append(args, fmt.Sprintf("%v<%v", k, v))
161+
}
162+
}
163+
if len(args) > 0 {
164+
return " --eviction-hard=" + strings.Join(args, ",")
165+
}
166+
return ""
167+
}
168+
134169
func (e EKS) mergeCustomUserData(userData *bytes.Buffer) (*bytes.Buffer, error) {
135170
var outputBuffer bytes.Buffer
136171
writer := multipart.NewWriter(&outputBuffer)

‎pkg/cloudprovider/aws/amifamily/bottlerocket.go

+9
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,12 @@ func (b Bottlerocket) EphemeralBlockDevice() *string {
8585
func (b Bottlerocket) ENILimitedMemoryOverhead() bool {
8686
return false
8787
}
88+
89+
// PodsPerCoreEnabled is currently disabled for Bottlerocket AMIFamily because it does
90+
// not currently support the podsPerCore parameter passed through the kubernetes settings TOML userData
91+
// If a Provisioner sets the podsPerCore value when using the Bottlerocket AMIFamily in the provider,
92+
// podsPerCore will be ignored
93+
// https://github.com/bottlerocket-os/bottlerocket/issues/1721
94+
func (b Bottlerocket) PodsPerCoreEnabled() bool {
95+
return false
96+
}

‎pkg/cloudprovider/aws/amifamily/custom.go

+4
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,7 @@ func (c Custom) EphemeralBlockDevice() *string {
5555
func (c Custom) ENILimitedMemoryOverhead() bool {
5656
return true
5757
}
58+
59+
func (c Custom) PodsPerCoreEnabled() bool {
60+
return true
61+
}

‎pkg/cloudprovider/aws/amifamily/resolver.go

+1
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ type AMIFamily interface {
7777
DefaultMetadataOptions() *v1alpha1.MetadataOptions
7878
EphemeralBlockDevice() *string
7979
ENILimitedMemoryOverhead() bool
80+
PodsPerCoreEnabled() bool
8081
}
8182

8283
// New constructs a new launch template Resolver

‎pkg/cloudprovider/aws/amifamily/ubuntu.go

+4
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,7 @@ func (u Ubuntu) EphemeralBlockDevice() *string {
6666
func (u Ubuntu) ENILimitedMemoryOverhead() bool {
6767
return true
6868
}
69+
70+
func (u Ubuntu) PodsPerCoreEnabled() bool {
71+
return true
72+
}

‎pkg/cloudprovider/aws/fake/ec2api.go

+9
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
363363
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
364364
},
365365
VCpuInfo: &ec2.VCpuInfo{
366+
DefaultCores: aws.Int64(1),
366367
DefaultVCpus: aws.Int64(2),
367368
},
368369
MemoryInfo: &ec2.MemoryInfo{
@@ -384,6 +385,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
384385
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
385386
},
386387
VCpuInfo: &ec2.VCpuInfo{
388+
DefaultCores: aws.Int64(1),
387389
DefaultVCpus: aws.Int64(2),
388390
},
389391
MemoryInfo: &ec2.MemoryInfo{
@@ -405,6 +407,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
405407
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
406408
},
407409
VCpuInfo: &ec2.VCpuInfo{
410+
DefaultCores: aws.Int64(2),
408411
DefaultVCpus: aws.Int64(4),
409412
},
410413
MemoryInfo: &ec2.MemoryInfo{
@@ -426,6 +429,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
426429
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
427430
},
428431
VCpuInfo: &ec2.VCpuInfo{
432+
DefaultCores: aws.Int64(16),
429433
DefaultVCpus: aws.Int64(32),
430434
},
431435
MemoryInfo: &ec2.MemoryInfo{
@@ -457,6 +461,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
457461
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
458462
},
459463
VCpuInfo: &ec2.VCpuInfo{
464+
DefaultCores: aws.Int64(16),
460465
DefaultVCpus: aws.Int64(32),
461466
},
462467
MemoryInfo: &ec2.MemoryInfo{
@@ -492,6 +497,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
492497
SupportedArchitectures: aws.StringSlice([]string{v1alpha5.ArchitectureArm64}),
493498
},
494499
VCpuInfo: &ec2.VCpuInfo{
500+
DefaultCores: aws.Int64(2),
495501
DefaultVCpus: aws.Int64(2),
496502
},
497503
MemoryInfo: &ec2.MemoryInfo{
@@ -513,6 +519,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
513519
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
514520
},
515521
VCpuInfo: &ec2.VCpuInfo{
522+
DefaultCores: aws.Int64(4),
516523
DefaultVCpus: aws.Int64(8),
517524
},
518525
MemoryInfo: &ec2.MemoryInfo{
@@ -539,6 +546,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
539546
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
540547
},
541548
VCpuInfo: &ec2.VCpuInfo{
549+
DefaultCores: aws.Int64(12),
542550
DefaultVCpus: aws.Int64(24),
543551
},
544552
MemoryInfo: &ec2.MemoryInfo{
@@ -565,6 +573,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
565573
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
566574
},
567575
VCpuInfo: &ec2.VCpuInfo{
576+
DefaultCores: aws.Int64(48),
568577
DefaultVCpus: aws.Int64(96),
569578
},
570579
MemoryInfo: &ec2.MemoryInfo{

‎pkg/cloudprovider/aws/instancetype.go

+77-36
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"fmt"
2020
"math"
2121
"regexp"
22+
"strconv"
2223
"strings"
2324

2425
"github.com/aws/aws-sdk-go/aws"
@@ -37,6 +38,10 @@ import (
3738
"github.com/aws/karpenter/pkg/utils/resources"
3839
)
3940

41+
const (
42+
memoryAvailable = "memory.available"
43+
)
44+
4045
var (
4146
_ cloudprovider.InstanceType = (*InstanceType)(nil)
4247
instanceTypeScheme = regexp.MustCompile(`(^[a-z]+)(\-[0-9]+tb)?([0-9]+).*\.`)
@@ -49,7 +54,7 @@ type InstanceType struct {
4954
requirements scheduling.Requirements
5055
resources v1.ResourceList
5156
provider *v1alpha1.AWS
52-
maxPods *int32
57+
maxPods *int64
5358
region string
5459
}
5560

@@ -60,14 +65,7 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alph
6065
offerings: offerings,
6166
region: region,
6267
}
63-
64-
// set max pods before computing resources
65-
// backwards compatibility for AWSENILimitedPodDensity flag
66-
if kc != nil && kc.MaxPods != nil {
67-
instanceType.maxPods = kc.MaxPods
68-
} else if !injection.GetOptions(ctx).AWSENILimitedPodDensity {
69-
instanceType.maxPods = ptr.Int32(110)
70-
}
68+
instanceType.maxPods = instanceType.computeMaxPods(ctx, kc)
7169

7270
// Precompute to minimize memory/compute overhead
7371
instanceType.resources = instanceType.computeResources(injection.GetOptions(ctx).AWSEnablePodENI)
@@ -201,10 +199,7 @@ func (i *InstanceType) ephemeralStorage() *resource.Quantity {
201199
}
202200

203201
func (i *InstanceType) pods() *resource.Quantity {
204-
if i.maxPods != nil {
205-
return resources.Quantity(fmt.Sprint(ptr.Int32Value(i.maxPods)))
206-
}
207-
return resources.Quantity(fmt.Sprint(i.eniLimitedPods()))
202+
return resources.Quantity(fmt.Sprint(ptr.Int64Value(i.maxPods)))
208203
}
209204

210205
func (i *InstanceType) awsPodENI(enablePodENI bool) *resource.Quantity {
@@ -251,17 +246,11 @@ func (i *InstanceType) awsNeurons() *resource.Quantity {
251246
}
252247

253248
func (i *InstanceType) computeOverhead(vmMemOverhead float64, kc *v1alpha5.KubeletConfiguration) v1.ResourceList {
254-
pods := i.pods()
255-
amiFamily := amifamily.GetAMIFamily(i.provider.AMIFamily, &amifamily.Options{})
256-
podsQuantity := pods.Value()
257-
if amiFamily.ENILimitedMemoryOverhead() {
258-
podsQuantity = i.eniLimitedPods()
259-
}
260-
261249
srr := i.systemReservedResources(kc)
262-
krr := i.kubeReservedResources(podsQuantity)
250+
krr := i.kubeReservedResources(kc)
263251
misc := i.miscResources(vmMemOverhead)
264-
overhead := resources.Merge(srr, krr, misc)
252+
et := i.evictionThreshold(kc, misc[v1.ResourceMemory])
253+
overhead := resources.Merge(srr, krr, et, misc)
265254

266255
return overhead
267256
}
@@ -280,18 +269,22 @@ func (i *InstanceType) systemReservedResources(kc *v1alpha5.KubeletConfiguration
280269
v1.ResourceMemory: resource.MustParse("100Mi"),
281270
v1.ResourceEphemeralStorage: resource.MustParse("1Gi"),
282271
}
283-
284272
if kc != nil && kc.SystemReserved != nil {
285-
for _, name := range []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory, v1.ResourceEphemeralStorage} {
286-
if v, ok := kc.SystemReserved[name]; ok {
287-
resources[name] = v
288-
}
289-
}
273+
return lo.Assign(resources, kc.SystemReserved)
290274
}
291275
return resources
292276
}
293277

294-
func (i *InstanceType) kubeReservedResources(pods int64) v1.ResourceList {
278+
func (i *InstanceType) kubeReservedResources(kc *v1alpha5.KubeletConfiguration) v1.ResourceList {
279+
// We reserve memory based off of --max-pods unless we are using the EKS-optimized AMI
280+
// which relies on ENI-limited pod density regardless of the --max-pods value
281+
// https://github.com/awslabs/amazon-eks-ami/issues/782
282+
amiFamily := amifamily.GetAMIFamily(i.provider.AMIFamily, &amifamily.Options{})
283+
pods := i.pods().Value()
284+
if amiFamily.ENILimitedMemoryOverhead() {
285+
pods = i.eniLimitedPods()
286+
}
287+
295288
resources := v1.ResourceList{
296289
v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dMi", (11*pods)+255)),
297290
v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), // default kube-reserved ephemeral-storage
@@ -319,19 +312,67 @@ func (i *InstanceType) kubeReservedResources(pods int64) v1.ResourceList {
319312
resources[v1.ResourceCPU] = *cpuOverhead
320313
}
321314
}
315+
if kc != nil && kc.KubeReserved != nil {
316+
return lo.Assign(resources, kc.KubeReserved)
317+
}
322318
return resources
323319
}
324320

325-
func (i *InstanceType) miscResources(vmMemOverhead float64) v1.ResourceList {
321+
func (i *InstanceType) evictionThreshold(kc *v1alpha5.KubeletConfiguration, vmMemoryOverhead resource.Quantity) v1.ResourceList {
322+
overhead := v1.ResourceList{
323+
v1.ResourceMemory: resource.MustParse("100Mi"),
324+
}
325+
if kc == nil || kc.EvictionHard == nil {
326+
return overhead
327+
}
328+
329+
if v, ok := kc.EvictionHard[memoryAvailable]; ok {
330+
if strings.HasSuffix(v, "%") {
331+
p, err := strconv.ParseFloat(strings.Trim(v, "%"), 64)
332+
if err != nil {
333+
panic(fmt.Sprintf("expected percentage value to be a float but got %s, %v", v, err))
334+
}
335+
// Setting percentage value to 100% is considered disabling the threshold according to
336+
// https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/
337+
if p == 100 {
338+
p = 0
339+
}
340+
// Calculation is node.capacity * evictionHard[memory.available] if percentage
341+
// From https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals
342+
totalAllocatable := i.resources.Memory().DeepCopy()
343+
totalAllocatable.Sub(vmMemoryOverhead)
344+
overhead[v1.ResourceMemory] = resource.MustParse(fmt.Sprint(math.Ceil(float64(totalAllocatable.Value()) / 100 * p)))
345+
} else {
346+
overhead[v1.ResourceMemory] = resource.MustParse(v)
347+
}
348+
}
349+
return overhead
350+
}
351+
352+
func (i *InstanceType) miscResources(overheadPercentage float64) v1.ResourceList {
326353
memory := i.memory().Value()
327354
return v1.ResourceList{
328-
v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dMi",
329-
// vm-overhead
330-
(int64(math.Ceil(float64(memory)*vmMemOverhead/1024/1024)))+
331-
// eviction threshold https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/kubelet/apis/config/v1beta1/defaults_linux.go#L23
332-
100,
333-
)),
355+
v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(memory)*overheadPercentage/1024/1024)))),
356+
}
357+
}
358+
359+
func (i *InstanceType) computeMaxPods(ctx context.Context, kc *v1alpha5.KubeletConfiguration) *int64 {
360+
amiFamily := amifamily.GetAMIFamily(i.provider.AMIFamily, &amifamily.Options{})
361+
var mp *int64
362+
363+
switch {
364+
case kc != nil && kc.MaxPods != nil:
365+
mp = ptr.Int64(int64(ptr.Int32Value(kc.MaxPods)))
366+
case !injection.GetOptions(ctx).AWSENILimitedPodDensity:
367+
mp = ptr.Int64(110)
368+
default:
369+
mp = ptr.Int64(i.eniLimitedPods())
370+
}
371+
372+
if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 && amiFamily.PodsPerCoreEnabled() {
373+
mp = ptr.Int64(lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * ptr.Int64Value(i.VCpuInfo.DefaultVCpus), ptr.Int64Value(mp)}))
334374
}
375+
return mp
335376
}
336377

337378
func lowerKabobCase(s string) string {

‎pkg/cloudprovider/aws/instancetypes_test.go

+127-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/aws/aws-sdk-go/service/ec2"
2424
. "github.com/onsi/ginkgo/v2"
2525
. "github.com/onsi/gomega"
26+
"github.com/samber/lo"
2627
v1 "k8s.io/api/core/v1"
2728
"k8s.io/apimachinery/pkg/api/resource"
2829
"k8s.io/apimachinery/pkg/util/sets"
@@ -271,6 +272,89 @@ var _ = Describe("Instance Types", func() {
271272
overhead := it.Overhead()
272273
Expect(overhead.Memory().String()).To(Equal("21473Mi"))
273274
})
275+
It("should override kube reserved when specified", func() {
276+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
277+
Expect(err).To(BeNil())
278+
provisioner = test.Provisioner(test.ProvisionerOptions{
279+
Kubelet: &v1alpha5.KubeletConfiguration{
280+
SystemReserved: v1.ResourceList{
281+
v1.ResourceCPU: resource.MustParse("1"),
282+
v1.ResourceMemory: resource.MustParse("20Gi"),
283+
v1.ResourceEphemeralStorage: resource.MustParse("1Gi"),
284+
},
285+
KubeReserved: v1.ResourceList{
286+
v1.ResourceCPU: resource.MustParse("2"),
287+
v1.ResourceMemory: resource.MustParse("10Gi"),
288+
v1.ResourceEphemeralStorage: resource.MustParse("2Gi"),
289+
},
290+
},
291+
})
292+
it := NewInstanceType(injection.WithOptions(ctx, opts), instanceInfo["m5.xlarge"], provisioner.Spec.KubeletConfiguration, "", provider, nil)
293+
overhead := it.Overhead()
294+
Expect(overhead.Memory().String()).To(Equal("30820Mi"))
295+
Expect(overhead.Cpu().String()).To(Equal("3"))
296+
Expect(overhead.StorageEphemeral().String()).To(Equal("3Gi"))
297+
})
298+
It("should override eviction threshold (hard) when specified as a quantity", func() {
299+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
300+
Expect(err).To(BeNil())
301+
provisioner = test.Provisioner(test.ProvisionerOptions{
302+
Kubelet: &v1alpha5.KubeletConfiguration{
303+
SystemReserved: v1.ResourceList{
304+
v1.ResourceMemory: resource.MustParse("20Gi"),
305+
},
306+
KubeReserved: v1.ResourceList{
307+
v1.ResourceMemory: resource.MustParse("10Gi"),
308+
},
309+
EvictionHard: map[string]string{
310+
memoryAvailable: "500Mi",
311+
},
312+
},
313+
})
314+
it := NewInstanceType(injection.WithOptions(ctx, opts), instanceInfo["m5.xlarge"], provisioner.Spec.KubeletConfiguration, "", provider, nil)
315+
overhead := it.Overhead()
316+
Expect(overhead.Memory().String()).To(Equal("31220Mi"))
317+
})
318+
It("should override eviction threshold (hard) when specified as a percentage value", func() {
319+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
320+
Expect(err).To(BeNil())
321+
provisioner = test.Provisioner(test.ProvisionerOptions{
322+
Kubelet: &v1alpha5.KubeletConfiguration{
323+
SystemReserved: v1.ResourceList{
324+
v1.ResourceMemory: resource.MustParse("20Gi"),
325+
},
326+
KubeReserved: v1.ResourceList{
327+
v1.ResourceMemory: resource.MustParse("10Gi"),
328+
},
329+
EvictionHard: map[string]string{
330+
memoryAvailable: "10%",
331+
},
332+
},
333+
})
334+
it := NewInstanceType(injection.WithOptions(ctx, opts), instanceInfo["m5.xlarge"], provisioner.Spec.KubeletConfiguration, "", provider, nil)
335+
overhead := it.Overhead()
336+
Expect(overhead.Memory().String()).To(Equal("33930241639"))
337+
})
338+
It("should consider the eviction threshold (hard) disabled when specified as 100%", func() {
339+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
340+
Expect(err).To(BeNil())
341+
provisioner = test.Provisioner(test.ProvisionerOptions{
342+
Kubelet: &v1alpha5.KubeletConfiguration{
343+
SystemReserved: v1.ResourceList{
344+
v1.ResourceMemory: resource.MustParse("20Gi"),
345+
},
346+
KubeReserved: v1.ResourceList{
347+
v1.ResourceMemory: resource.MustParse("10Gi"),
348+
},
349+
EvictionHard: map[string]string{
350+
memoryAvailable: "100%",
351+
},
352+
},
353+
})
354+
it := NewInstanceType(injection.WithOptions(ctx, opts), instanceInfo["m5.xlarge"], provisioner.Spec.KubeletConfiguration, "", provider, nil)
355+
overhead := it.Overhead()
356+
Expect(overhead.Memory().String()).To(Equal("30Gi"))
357+
})
274358
It("should set max-pods to user-defined value if specified", func() {
275359
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
276360
Expect(err).To(BeNil())
@@ -281,7 +365,7 @@ var _ = Describe("Instance Types", func() {
281365
Expect(resources.Pods().Value()).To(BeNumerically("==", 10))
282366
}
283367
})
284-
It("should override max-pods value when AWSENILimitedPodDensity is set", func() {
368+
It("should override max-pods value when AWSENILimitedPodDensity is unset", func() {
285369
opts.AWSENILimitedPodDensity = false
286370
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
287371
Expect(err).To(BeNil())
@@ -292,6 +376,48 @@ var _ = Describe("Instance Types", func() {
292376
Expect(resources.Pods().Value()).To(BeNumerically("==", 10))
293377
}
294378
})
379+
It("should override pods-per-core value", func() {
380+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
381+
Expect(err).To(BeNil())
382+
provisioner = test.Provisioner(test.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}})
383+
for _, info := range instanceInfo {
384+
it := NewInstanceType(injection.WithOptions(ctx, opts), info, provisioner.Spec.KubeletConfiguration, "", provider, nil)
385+
resources := it.Resources()
386+
Expect(resources.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus)))
387+
}
388+
})
389+
It("should take the minimum of pods-per-core and max-pods", func() {
390+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
391+
Expect(err).To(BeNil())
392+
provisioner = test.Provisioner(test.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}})
393+
for _, info := range instanceInfo {
394+
it := NewInstanceType(injection.WithOptions(ctx, opts), info, provisioner.Spec.KubeletConfiguration, "", provider, nil)
395+
resources := it.Resources()
396+
Expect(resources.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4})))
397+
}
398+
})
399+
It("should ignore pods-per-core when using Bottlerocket AMI", func() {
400+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
401+
Expect(err).To(BeNil())
402+
provider.AMIFamily = &awsv1alpha1.AMIFamilyBottlerocket
403+
provisioner = test.Provisioner(test.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}, Provider: provider})
404+
for _, info := range instanceInfo {
405+
it := NewInstanceType(injection.WithOptions(ctx, opts), info, provisioner.Spec.KubeletConfiguration, "", provider, nil)
406+
resources := it.Resources()
407+
Expect(resources.Pods().Value()).To(BeNumerically("==", it.eniLimitedPods()))
408+
}
409+
})
410+
It("should take 110 to be the default pods number when pods-per-core is 0 and AWSENILimitedPodDensity is unset", func() {
411+
opts.AWSENILimitedPodDensity = false
412+
instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx)
413+
Expect(err).To(BeNil())
414+
provisioner = test.Provisioner(test.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}})
415+
for _, info := range instanceInfo {
416+
it := NewInstanceType(injection.WithOptions(ctx, opts), info, provisioner.Spec.KubeletConfiguration, "", provider, nil)
417+
resources := it.Resources()
418+
Expect(resources.Pods().Value()).To(BeNumerically("==", 110))
419+
}
420+
})
295421
})
296422
Context("Insufficient Capacity Error Cache", func() {
297423
It("should launch instances of different type on second reconciliation attempt with Insufficient Capacity Error Cache fallback", func() {

‎pkg/cloudprovider/aws/launchtemplate_test.go

+151-5
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ var _ = Describe("LaunchTemplates", func() {
589589
Expect(string(userData)).To(ContainSubstring("--max-pods=10"))
590590
})
591591
It("should specify --system-reserved when overriding system reserved values", func() {
592-
newProvisioner := test.Provisioner(test.ProvisionerOptions{
592+
provisioner = test.Provisioner(test.ProvisionerOptions{
593593
Kubelet: &v1alpha5.KubeletConfiguration{
594594
SystemReserved: v1.ResourceList{
595595
v1.ResourceCPU: resource.MustParse("500m"),
@@ -598,7 +598,7 @@ var _ = Describe("LaunchTemplates", func() {
598598
},
599599
},
600600
})
601-
ExpectApplied(ctx, env.Client, newProvisioner)
601+
ExpectApplied(ctx, env.Client, provisioner)
602602
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
603603
ExpectScheduled(ctx, env.Client, pod)
604604
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
@@ -610,10 +610,92 @@ var _ = Describe("LaunchTemplates", func() {
610610
i := strings.Index(string(userData), arg)
611611
rem := string(userData)[(i + len(arg)):]
612612
i = strings.Index(rem, "'")
613-
for k, v := range newProvisioner.Spec.KubeletConfiguration.SystemReserved {
613+
for k, v := range provisioner.Spec.KubeletConfiguration.SystemReserved {
614+
Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k.String(), v.String())))
615+
}
616+
})
617+
It("should specify --kube-reserved when overriding system reserved values", func() {
618+
provisioner = test.Provisioner(test.ProvisionerOptions{
619+
Kubelet: &v1alpha5.KubeletConfiguration{
620+
KubeReserved: v1.ResourceList{
621+
v1.ResourceCPU: resource.MustParse("500m"),
622+
v1.ResourceMemory: resource.MustParse("1Gi"),
623+
v1.ResourceEphemeralStorage: resource.MustParse("2Gi"),
624+
},
625+
},
626+
})
627+
ExpectApplied(ctx, env.Client, provisioner)
628+
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
629+
ExpectScheduled(ctx, env.Client, pod)
630+
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
631+
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop()
632+
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
633+
634+
// Check whether the arguments are there for --kube-reserved
635+
arg := "--kube-reserved="
636+
i := strings.Index(string(userData), arg)
637+
rem := string(userData)[(i + len(arg)):]
638+
i = strings.Index(rem, "'")
639+
for k, v := range provisioner.Spec.KubeletConfiguration.KubeReserved {
614640
Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k.String(), v.String())))
615641
}
616642
})
643+
It("should pass eviction threshold hard values when specified", func() {
644+
provisioner = test.Provisioner(test.ProvisionerOptions{
645+
Kubelet: &v1alpha5.KubeletConfiguration{
646+
EvictionHard: map[string]string{
647+
"memory.available": "10%",
648+
"nodefs.available": "15%",
649+
"nodefs.inodesFree": "5%",
650+
},
651+
},
652+
})
653+
ExpectApplied(ctx, env.Client, provisioner)
654+
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
655+
ExpectScheduled(ctx, env.Client, pod)
656+
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
657+
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop()
658+
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
659+
660+
// Check whether the arguments are there for --kube-reserved
661+
arg := "--eviction-hard="
662+
i := strings.Index(string(userData), arg)
663+
rem := string(userData)[(i + len(arg)):]
664+
i = strings.Index(rem, "'")
665+
for k, v := range provisioner.Spec.KubeletConfiguration.EvictionHard {
666+
Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v<%v", k, v)))
667+
}
668+
})
669+
It("should specify --pods-per-core", func() {
670+
provisioner = test.Provisioner(test.ProvisionerOptions{
671+
Kubelet: &v1alpha5.KubeletConfiguration{
672+
PodsPerCore: ptr.Int32(2),
673+
},
674+
})
675+
ExpectApplied(ctx, env.Client, provisioner)
676+
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
677+
ExpectScheduled(ctx, env.Client, pod)
678+
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
679+
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop()
680+
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
681+
Expect(string(userData)).To(ContainSubstring(fmt.Sprintf("--pods-per-core=%d", 2)))
682+
})
683+
It("should specify --pods-per-core with --max-pods enabled", func() {
684+
provisioner = test.Provisioner(test.ProvisionerOptions{
685+
Kubelet: &v1alpha5.KubeletConfiguration{
686+
PodsPerCore: ptr.Int32(2),
687+
MaxPods: ptr.Int32(100),
688+
},
689+
})
690+
ExpectApplied(ctx, env.Client, provisioner)
691+
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
692+
ExpectScheduled(ctx, env.Client, pod)
693+
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
694+
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop()
695+
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
696+
Expect(string(userData)).To(ContainSubstring(fmt.Sprintf("--pods-per-core=%d", 2)))
697+
Expect(string(userData)).To(ContainSubstring(fmt.Sprintf("--max-pods=%d", 100)))
698+
})
617699
It("should specify --container-runtime containerd by default", func() {
618700
ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{Provider: provider}))
619701
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
@@ -770,7 +852,7 @@ var _ = Describe("LaunchTemplates", func() {
770852
AWS: *provider,
771853
})
772854
ExpectApplied(ctx, env.Client, nodeTemplate)
773-
newProvisioner := test.Provisioner(test.ProvisionerOptions{
855+
provisioner = test.Provisioner(test.ProvisionerOptions{
774856
ProviderRef: &v1alpha5.ProviderRef{
775857
Name: nodeTemplate.Name,
776858
},
@@ -782,7 +864,7 @@ var _ = Describe("LaunchTemplates", func() {
782864
},
783865
},
784866
})
785-
ExpectApplied(ctx, env.Client, newProvisioner)
867+
ExpectApplied(ctx, env.Client, provisioner)
786868
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
787869
ExpectScheduled(ctx, env.Client, pod)
788870
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
@@ -795,6 +877,70 @@ var _ = Describe("LaunchTemplates", func() {
795877
Expect(config.Settings.Kubernetes.SystemReserved[v1.ResourceMemory.String()]).To(Equal("3Gi"))
796878
Expect(config.Settings.Kubernetes.SystemReserved[v1.ResourceEphemeralStorage.String()]).To(Equal("10Gi"))
797879
})
880+
It("should override kube reserved values in user data", func() {
881+
provider.AMIFamily = &awsv1alpha1.AMIFamilyBottlerocket
882+
nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{
883+
UserData: nil,
884+
AWS: *provider,
885+
})
886+
ExpectApplied(ctx, env.Client, nodeTemplate)
887+
provisioner = test.Provisioner(test.ProvisionerOptions{
888+
ProviderRef: &v1alpha5.ProviderRef{
889+
Name: nodeTemplate.Name,
890+
},
891+
Kubelet: &v1alpha5.KubeletConfiguration{
892+
KubeReserved: v1.ResourceList{
893+
v1.ResourceCPU: resource.MustParse("2"),
894+
v1.ResourceMemory: resource.MustParse("3Gi"),
895+
v1.ResourceEphemeralStorage: resource.MustParse("10Gi"),
896+
},
897+
},
898+
})
899+
ExpectApplied(ctx, env.Client, provisioner)
900+
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
901+
ExpectScheduled(ctx, env.Client, pod)
902+
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
903+
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop()
904+
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
905+
config := &bootstrap.BottlerocketConfig{}
906+
Expect(config.UnmarshalTOML(userData)).To(Succeed())
907+
Expect(len(config.Settings.Kubernetes.KubeReserved)).To(Equal(3))
908+
Expect(config.Settings.Kubernetes.KubeReserved[v1.ResourceCPU.String()]).To(Equal("2"))
909+
Expect(config.Settings.Kubernetes.KubeReserved[v1.ResourceMemory.String()]).To(Equal("3Gi"))
910+
Expect(config.Settings.Kubernetes.KubeReserved[v1.ResourceEphemeralStorage.String()]).To(Equal("10Gi"))
911+
})
912+
It("should override kube reserved values in user data", func() {
913+
provider.AMIFamily = &awsv1alpha1.AMIFamilyBottlerocket
914+
nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{
915+
UserData: nil,
916+
AWS: *provider,
917+
})
918+
ExpectApplied(ctx, env.Client, nodeTemplate)
919+
provisioner = test.Provisioner(test.ProvisionerOptions{
920+
ProviderRef: &v1alpha5.ProviderRef{
921+
Name: nodeTemplate.Name,
922+
},
923+
Kubelet: &v1alpha5.KubeletConfiguration{
924+
EvictionHard: map[string]string{
925+
"memory.available": "10%",
926+
"nodefs.available": "15%",
927+
"nodefs.inodesFree": "5%",
928+
},
929+
},
930+
})
931+
ExpectApplied(ctx, env.Client, provisioner)
932+
pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
933+
ExpectScheduled(ctx, env.Client, pod)
934+
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
935+
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop()
936+
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
937+
config := &bootstrap.BottlerocketConfig{}
938+
Expect(config.UnmarshalTOML(userData)).To(Succeed())
939+
Expect(len(config.Settings.Kubernetes.EvictionHard)).To(Equal(3))
940+
Expect(config.Settings.Kubernetes.EvictionHard["memory.available"]).To(Equal("10%"))
941+
Expect(config.Settings.Kubernetes.EvictionHard["nodefs.available"]).To(Equal("15%"))
942+
Expect(config.Settings.Kubernetes.EvictionHard["nodefs.inodesFree"]).To(Equal("5%"))
943+
})
798944
It("should specify max pods value when passing maxPods in configuration", func() {
799945
bottlerocketProvider := provider.DeepCopy()
800946
bottlerocketProvider.AMIFamily = &awsv1alpha1.AMIFamilyBottlerocket

‎pkg/cloudprovider/aws/suite_test.go

+11-8
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,15 @@ var provisioner *v1alpha5.Provisioner
7575
var provider *awsv1alpha1.AWS
7676
var pricingProvider *PricingProvider
7777

78+
var defaultOpts = options.Options{
79+
ClusterName: "test-cluster",
80+
ClusterEndpoint: "https://test-cluster",
81+
AWSNodeNameConvention: string(options.IPName),
82+
AWSENILimitedPodDensity: true,
83+
AWSEnablePodENI: true,
84+
AWSDefaultInstanceProfile: "test-instance-profile",
85+
}
86+
7887
func TestAWS(t *testing.T) {
7988
ctx = TestContextWithLogger(t)
8089
RegisterFailHandler(Fail)
@@ -83,14 +92,7 @@ func TestAWS(t *testing.T) {
8392

8493
var _ = BeforeSuite(func() {
8594
env = test.NewEnvironment(ctx, func(e *test.Environment) {
86-
opts = options.Options{
87-
ClusterName: "test-cluster",
88-
ClusterEndpoint: "https://test-cluster",
89-
AWSNodeNameConvention: string(options.IPName),
90-
AWSENILimitedPodDensity: true,
91-
AWSEnablePodENI: true,
92-
AWSDefaultInstanceProfile: "test-instance-profile",
93-
}
95+
opts = defaultOpts
9496
Expect(opts.Validate()).To(Succeed(), "Failed to validate options")
9597
ctx = injection.WithOptions(ctx, opts)
9698
ctx, stop = context.WithCancel(ctx)
@@ -161,6 +163,7 @@ var _ = BeforeEach(func() {
161163
SecurityGroupSelector: map[string]string{"*": "*"},
162164
}
163165
provisioner = test.Provisioner(test.ProvisionerOptions{Provider: provider})
166+
opts = defaultOpts
164167
fakeEC2API.Reset()
165168
fakePricingAPI.Reset()
166169
launchTemplateCache.Flush()

‎pkg/controllers/provisioning/suite_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,8 @@ var _ = Describe("Provisioning", func() {
162162
}
163163
})
164164
It("should provision multiple nodes when maxPods is set", func() {
165+
// Kubelet configuration is actually not observed here, the scheduler is relying on the
166+
// pods resource value which is statically set in the fake cloudprovider
165167
ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{
166168
Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(1)},
167169
Requirements: []v1.NodeSelectorRequirement{

‎pkg/utils/resources/resources.go

+3
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,9 @@ func String(list v1.ResourceList) string {
154154

155155
// StringMap returns the string map representation of the resource list
156156
func StringMap(list v1.ResourceList) map[string]string {
157+
if list == nil {
158+
return nil
159+
}
157160
m := make(map[string]string)
158161
for k, v := range list {
159162
m[k.String()] = v.String()

‎test/suites/integration/kubelet_config_test.go

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License.
13+
*/
14+
15+
package integration_test
16+
17+
import (
18+
. "github.com/onsi/ginkgo/v2"
19+
v1 "k8s.io/api/core/v1"
20+
"knative.dev/pkg/ptr"
21+
22+
"github.com/aws/karpenter/pkg/apis/awsnodetemplate/v1alpha1"
23+
"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
24+
awsv1alpha1 "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
25+
"github.com/aws/karpenter/pkg/test"
26+
)
27+
28+
var _ = Describe("KubeletConfiguration Overrides", func() {
29+
It("should schedule pods onto separate nodes when maxPods is set", func() {
30+
provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{
31+
SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
32+
SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
33+
}})
34+
// MaxPods needs to account for the daemonsets that will run on the nodes
35+
provisioner := test.Provisioner(test.ProvisionerOptions{
36+
ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name},
37+
Kubelet: &v1alpha5.KubeletConfiguration{
38+
MaxPods: ptr.Int32(3),
39+
},
40+
})
41+
42+
pods := []*v1.Pod{test.Pod(), test.Pod(), test.Pod()}
43+
env.ExpectCreated(provisioner, provider)
44+
for _, pod := range pods {
45+
env.ExpectCreated(pod)
46+
}
47+
env.EventuallyExpectHealthy(pods...)
48+
env.ExpectCreatedNodeCount("==", 3)
49+
})
50+
It("should schedule pods onto separate nodes when podsPerCore is set", func() {
51+
provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{
52+
SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
53+
SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
54+
}})
55+
// PodsPerCore needs to account for the daemonsets that will run on the nodes
56+
// This will have 4 pods available on each node (2 taken by daemonset pods)
57+
provisioner := test.Provisioner(test.ProvisionerOptions{
58+
ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name},
59+
Kubelet: &v1alpha5.KubeletConfiguration{
60+
PodsPerCore: ptr.Int32(2),
61+
},
62+
Requirements: []v1.NodeSelectorRequirement{
63+
{
64+
Key: awsv1alpha1.LabelInstanceCPU,
65+
Operator: v1.NodeSelectorOpIn,
66+
Values: []string{"2"},
67+
},
68+
},
69+
})
70+
71+
pods := []*v1.Pod{test.Pod(), test.Pod(), test.Pod(), test.Pod()}
72+
env.ExpectCreated(provisioner, provider)
73+
for _, pod := range pods {
74+
env.ExpectCreated(pod)
75+
}
76+
env.EventuallyExpectHealthy(pods...)
77+
env.ExpectCreatedNodeCount("==", 2)
78+
})
79+
It("should ignore podsPerCore value when Bottlerocket is used", func() {
80+
provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{
81+
SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
82+
SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
83+
AMIFamily: &awsv1alpha1.AMIFamilyBottlerocket,
84+
}})
85+
// All pods should schedule to a single node since we are ignoring podsPerCore value
86+
// This would normally schedule to 3 nodes if not using Bottlerocket
87+
provisioner := test.Provisioner(test.ProvisionerOptions{
88+
ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name},
89+
Kubelet: &v1alpha5.KubeletConfiguration{
90+
PodsPerCore: ptr.Int32(2),
91+
},
92+
Requirements: []v1.NodeSelectorRequirement{
93+
{
94+
Key: awsv1alpha1.LabelInstanceCPU,
95+
Operator: v1.NodeSelectorOpIn,
96+
Values: []string{"2"},
97+
},
98+
},
99+
})
100+
101+
pods := []*v1.Pod{test.Pod(), test.Pod(), test.Pod(), test.Pod(), test.Pod(), test.Pod()}
102+
env.ExpectCreated(provisioner, provider)
103+
for _, pod := range pods {
104+
env.ExpectCreated(pod)
105+
}
106+
env.EventuallyExpectHealthy(pods...)
107+
env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching
108+
})
109+
})

0 commit comments

Comments
 (0)
Please sign in to comment.