Skip to content

Commit d23cc4e

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #2086 from MarcusNoble/secondary_cidr
✨ Secondary cidr support for EKS workload clusters
2 parents 7470b49 + 4cc8d96 commit d23cc4e

20 files changed

+754
-8
lines changed

‎api/v1alpha3/conditions_consts.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,14 @@ const (
6161
RouteTableReconciliationFailedReason = "RouteTableReconciliationFailed"
6262
)
6363

64+
const (
65+
// SecondaryCidrsReady condition reports successful reconciliation of secondary CIDR blocks.
66+
// Only applicable to managed clusters.
67+
SecondaryCidrsReadyCondition clusterv1.ConditionType = "SecondaryCidrsReady"
68+
// SecondaryCidrReconciliationFailedReason used when any errors occur during reconciliation of secondary CIDR blocks.
69+
SecondaryCidrReconciliationFailedReason = "SecondaryCidrReconciliationFailed"
70+
)
71+
6472
const (
6573
// ClusterSecurityGroupsReady condition reports successful reconciliation of security groups.
6674
ClusterSecurityGroupsReadyCondition clusterv1.ConditionType = "ClusterSecurityGroupsReady"

‎api/v1alpha3/tags.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ package v1alpha3
1818

1919
import (
2020
"fmt"
21-
"k8s.io/apimachinery/pkg/types"
2221
"reflect"
22+
23+
"k8s.io/apimachinery/pkg/types"
2324
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
2425
)
2526

@@ -106,6 +107,10 @@ const (
106107
// dedicated to this cluster api provider implementation.
107108
NameAWSClusterAPIRole = NameAWSProviderPrefix + "role"
108109

110+
NameAWSSubnetAssociation = NameAWSProviderPrefix + "association"
111+
112+
SecondarySubnetTagValue = "secondary"
113+
109114
// APIServerRoleTagValue describes the value for the apiserver role
110115
APIServerRoleTagValue = "apiserver"
111116

‎controlplane/eks/api/v1alpha3/awsmanagedcontrolplane_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ type AWSManagedControlPlaneSpec struct {
3838
// NetworkSpec encapsulates all things related to AWS network.
3939
NetworkSpec infrav1.NetworkSpec `json:"networkSpec,omitempty"`
4040

41+
// SecondaryCidrBlock is the additional CIDR range to use for pod IPs.
42+
// Must be within the 100.64.0.0/10 or 198.19.0.0/16 range.
43+
// +optional
44+
SecondaryCidrBlock *string `json:"secondaryCidrBlock,omitempty"`
45+
4146
// The AWS Region the cluster lives in.
4247
Region string `json:"region,omitempty"`
4348

‎controlplane/eks/api/v1alpha3/awsmanagedcontrolplane_webhook.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@ package v1alpha3
1818

1919
import (
2020
"fmt"
21+
"net"
2122

23+
"github.com/apparentlymart/go-cidr/cidr"
2224
"github.com/pkg/errors"
2325

2426
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -37,6 +39,11 @@ import (
3739
// log is for logging in this package.
3840
var mcpLog = logf.Log.WithName("awsmanagedcontrolplane-resource")
3941

42+
const (
43+
cidrSizeMax = 65536
44+
cidrSizeMin = 16
45+
)
46+
4047
// SetupWebhookWithManager will setup the webhooks for the AWSManagedControlPlane
4148
func (r *AWSManagedControlPlane) SetupWebhookWithManager(mgr ctrl.Manager) error {
4249
return ctrl.NewWebhookManagedBy(mgr).
@@ -80,6 +87,7 @@ func (r *AWSManagedControlPlane) ValidateCreate() error {
8087
allErrs = append(allErrs, r.validateEKSVersion(nil)...)
8188
allErrs = append(allErrs, r.Spec.Bastion.Validate()...)
8289
allErrs = append(allErrs, r.validateIAMAuthConfig()...)
90+
allErrs = append(allErrs, r.validateSecondaryCIDR()...)
8391

8492
if len(allErrs) == 0 {
8593
return nil
@@ -108,6 +116,7 @@ func (r *AWSManagedControlPlane) ValidateUpdate(old runtime.Object) error {
108116
allErrs = append(allErrs, r.validateEKSVersion(oldAWSManagedControlplane)...)
109117
allErrs = append(allErrs, r.Spec.Bastion.Validate()...)
110118
allErrs = append(allErrs, r.validateIAMAuthConfig()...)
119+
allErrs = append(allErrs, r.validateSecondaryCIDR()...)
111120

112121
if r.Spec.Region != oldAWSManagedControlplane.Spec.Region {
113122
allErrs = append(allErrs,
@@ -207,6 +216,37 @@ func (r *AWSManagedControlPlane) validateIAMAuthConfig() field.ErrorList {
207216
return allErrs
208217
}
209218

219+
func (r *AWSManagedControlPlane) validateSecondaryCIDR() field.ErrorList {
220+
var allErrs field.ErrorList
221+
if r.Spec.SecondaryCidrBlock != nil {
222+
cidrField := field.NewPath("spec", "secondaryCidrBlock")
223+
_, validRange1, _ := net.ParseCIDR("100.64.0.0/10")
224+
_, validRange2, _ := net.ParseCIDR("198.19.0.0/16")
225+
226+
_, ipv4Net, err := net.ParseCIDR(*r.Spec.SecondaryCidrBlock)
227+
if err != nil {
228+
allErrs = append(allErrs, field.Invalid(cidrField, *r.Spec.SecondaryCidrBlock, "must be valid CIDR range"))
229+
return allErrs
230+
}
231+
232+
cidrSize := cidr.AddressCount(ipv4Net)
233+
if cidrSize > cidrSizeMax || cidrSize < cidrSizeMin {
234+
allErrs = append(allErrs, field.Invalid(cidrField, *r.Spec.SecondaryCidrBlock, "CIDR block sizes must be between a /16 netmask and /28 netmask"))
235+
}
236+
237+
start, end := cidr.AddressRange(ipv4Net)
238+
if (!validRange1.Contains(start) || !validRange1.Contains(end)) && (!validRange2.Contains(start) || !validRange2.Contains(end)) {
239+
allErrs = append(allErrs, field.Invalid(cidrField, *r.Spec.SecondaryCidrBlock, "must be within the 100.64.0.0/10 or 198.19.0.0/16 range"))
240+
}
241+
242+
}
243+
244+
if len(allErrs) == 0 {
245+
return nil
246+
}
247+
return allErrs
248+
}
249+
210250
// Default will set default values for the AWSManagedControlPlane
211251
func (r *AWSManagedControlPlane) Default() {
212252
mcpLog.Info("AWSManagedControlPlane setting defaults", "name", r.Name)

‎controlplane/eks/api/v1alpha3/awsmanagedcontrolplane_webhook_test.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,13 @@ func TestDefaultingWebhook(t *testing.T) {
116116
spec: AWSManagedControlPlaneSpec{NetworkSpec: infrav1.NetworkSpec{CNI: &infrav1.CNISpec{}}},
117117
expectSpec: AWSManagedControlPlaneSpec{EKSClusterName: "default_cluster1", Bastion: defaultTestBastion, NetworkSpec: infrav1.NetworkSpec{CNI: &infrav1.CNISpec{}, VPC: defaultVPCSpec}, TokenMethod: &EKSTokenMethodIAMAuthenticator},
118118
},
119+
{
120+
name: "secondary CIDR",
121+
resourceName: "cluster1",
122+
resourceNS: "default",
123+
expectHash: false,
124+
expectSpec: AWSManagedControlPlaneSpec{EKSClusterName: "default_cluster1", Bastion: defaultTestBastion, NetworkSpec: defaultNetworkSpec, SecondaryCidrBlock: nil, TokenMethod: &EKSTokenMethodIAMAuthenticator},
125+
},
119126
}
120127

121128
for _, tc := range tests {
@@ -304,3 +311,140 @@ func TestWebhookUpdate(t *testing.T) {
304311
})
305312
}
306313
}
314+
315+
func TestValidatingWebhookCreate_SecondaryCidr(t *testing.T) {
316+
tests := []struct {
317+
name string
318+
expectError bool
319+
cidrRange string
320+
}{
321+
{
322+
name: "complete range 1",
323+
cidrRange: "100.64.0.0/10",
324+
expectError: true,
325+
},
326+
{
327+
name: "complete range 2",
328+
cidrRange: "198.19.0.0/16",
329+
expectError: false,
330+
},
331+
{
332+
name: "subrange",
333+
cidrRange: "100.67.0.0/16",
334+
expectError: false,
335+
},
336+
{
337+
name: "invalid value",
338+
cidrRange: "not a cidr range",
339+
expectError: true,
340+
},
341+
{
342+
name: "unsupported range",
343+
cidrRange: "10.0.0.1/20",
344+
expectError: true,
345+
},
346+
{
347+
name: "too large",
348+
cidrRange: "100.64.0.0/15",
349+
expectError: true,
350+
},
351+
{
352+
name: "too small",
353+
cidrRange: "100.64.0.0/29",
354+
expectError: true,
355+
},
356+
}
357+
358+
for _, tc := range tests {
359+
t.Run(tc.name, func(t *testing.T) {
360+
g := NewWithT(t)
361+
362+
mcp := &AWSManagedControlPlane{
363+
Spec: AWSManagedControlPlaneSpec{
364+
EKSClusterName: "default_cluster1",
365+
},
366+
}
367+
if tc.cidrRange != "" {
368+
mcp.Spec.SecondaryCidrBlock = &tc.cidrRange
369+
}
370+
err := mcp.ValidateCreate()
371+
372+
if tc.expectError {
373+
g.Expect(err).ToNot(BeNil())
374+
} else {
375+
g.Expect(err).To(BeNil())
376+
}
377+
})
378+
}
379+
}
380+
381+
func TestValidatingWebhookUpdate_SecondaryCidr(t *testing.T) {
382+
tests := []struct {
383+
name string
384+
cidrRange string
385+
expectError bool
386+
}{
387+
{
388+
name: "complete range 1",
389+
cidrRange: "100.64.0.0/10",
390+
expectError: true,
391+
},
392+
{
393+
name: "complete range 2",
394+
cidrRange: "198.19.0.0/16",
395+
expectError: false,
396+
},
397+
{
398+
name: "subrange",
399+
cidrRange: "100.67.0.0/16",
400+
expectError: false,
401+
},
402+
{
403+
name: "invalid value",
404+
cidrRange: "not a cidr range",
405+
expectError: true,
406+
},
407+
{
408+
name: "unsupported range",
409+
cidrRange: "10.0.0.1/20",
410+
expectError: true,
411+
},
412+
{
413+
name: "too large",
414+
cidrRange: "100.64.0.0/15",
415+
expectError: true,
416+
},
417+
{
418+
name: "too small",
419+
cidrRange: "100.64.0.0/29",
420+
expectError: true,
421+
},
422+
}
423+
424+
for _, tc := range tests {
425+
t.Run(tc.name, func(t *testing.T) {
426+
g := NewWithT(t)
427+
428+
newMCP := &AWSManagedControlPlane{
429+
Spec: AWSManagedControlPlaneSpec{
430+
EKSClusterName: "default_cluster1",
431+
SecondaryCidrBlock: &tc.cidrRange,
432+
},
433+
}
434+
oldMCP := &AWSManagedControlPlane{
435+
Spec: AWSManagedControlPlaneSpec{
436+
EKSClusterName: "default_cluster1",
437+
SecondaryCidrBlock: nil,
438+
},
439+
}
440+
441+
err := newMCP.ValidateUpdate(oldMCP)
442+
443+
if tc.expectError {
444+
g.Expect(err).ToNot(BeNil())
445+
} else {
446+
g.Expect(err).To(BeNil())
447+
}
448+
})
449+
}
450+
}

‎controlplane/eks/api/v1alpha3/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎controlplane/eks/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,9 @@ spec:
324324
description: RoleName specifies the name of IAM role that gives EKS permission to make API calls. If the role is pre-existing we will treat it as unmanaged and not delete it on deletion. If the EKSEnableIAM feature flag is true and no name is supplied then a role is created.
325325
minLength: 2
326326
type: string
327+
secondaryCidrBlock:
328+
description: SecondaryCidrBlock is the additional CIDR range to use for pod IPs. Must be within the 100.64.0.0/10 or 198.19.0.0/16 range.
329+
type: string
327330
sshKeyName:
328331
description: SSHKeyName is the name of the ssh key to attach to the bastion host. Valid values are empty string (do not use SSH keys), a valid SSH key name, or omitted (use the default SSH key name)
329332
type: string

‎controlplane/eks/controllers/awsmanagedcontrolplane_controller.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import (
4444
infrav1exp "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha3"
4545
"sigs.k8s.io/cluster-api-provider-aws/feature"
4646
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope"
47+
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/awsnode"
4748
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/ec2"
4849
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks"
4950
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/iamauth"
@@ -214,6 +215,7 @@ func (r *AWSManagedControlPlaneReconciler) reconcileNormal(ctx context.Context,
214215
ekssvc := eks.NewService(managedScope)
215216
sgService := securitygroup.NewServiceWithRoles(managedScope, sgRoles)
216217
authService := iamauth.NewService(managedScope, iamauth.BackendTypeConfigMap, managedScope.Client)
218+
awsnodeService := awsnode.NewService(managedScope)
217219

218220
if err := networkSvc.ReconcileNetwork(); err != nil {
219221
return reconcile.Result{}, fmt.Errorf("failed to reconcile network for AWSManagedControlPlane %s/%s: %w", awsManagedControlPlane.Namespace, awsManagedControlPlane.Name, err)
@@ -233,6 +235,11 @@ func (r *AWSManagedControlPlaneReconciler) reconcileNormal(ctx context.Context,
233235
return reconcile.Result{}, fmt.Errorf("failed to reconcile control plane for AWSManagedControlPlane %s/%s: %w", awsManagedControlPlane.Namespace, awsManagedControlPlane.Name, err)
234236
}
235237

238+
if err := awsnodeService.ReconcileCNI(); err != nil {
239+
conditions.MarkFalse(managedScope.InfraCluster(), infrav1.SecondaryCidrsReadyCondition, infrav1.SecondaryCidrReconciliationFailedReason, clusterv1.ConditionSeverityError, err.Error())
240+
return reconcile.Result{}, fmt.Errorf("failed to reconcile control plane for AWSManagedControlPlane %s/%s: %w", awsManagedControlPlane.Namespace, awsManagedControlPlane.Name, err)
241+
}
242+
236243
if err := authService.ReconcileIAMAuthenticator(ctx); err != nil {
237244
conditions.MarkFalse(awsManagedControlPlane, controlplanev1.IAMAuthenticatorConfiguredCondition, controlplanev1.IAMAuthenticatorConfigurationFailedReason, clusterv1.ConditionSeverityError, err.Error())
238245
return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile aws-iam-authenticator config for AWSManagedControlPlane %s/%s", awsManagedControlPlane.Namespace, awsManagedControlPlane.Name)

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ module sigs.k8s.io/cluster-api-provider-aws
33
go 1.13
44

55
require (
6+
github.com/apparentlymart/go-cidr v1.1.0
7+
github.com/aws/amazon-vpc-cni-k8s v1.7.5
68
github.com/aws/aws-sdk-go v1.36.26
79
github.com/awslabs/goformation/v4 v4.15.0
810
github.com/blang/semver v3.5.1+incompatible

0 commit comments

Comments
 (0)