Skip to content

Commit 54c0d30

Browse files
mcwumblychristianang
mcwumbly
and
christianang
committed
Add ipv6 support to capd
quickstart e2e test can be run with: ``` IP_FAMILY=IPv6 \ DOCKER_SERVICE_CIDRS="fd00:100:64::/108" \ DOCKER_POD_CIDRS="fd00:100:96::/48" \ make test-e2e ``` Note: POD_SUBNET in kindnet.yaml is now set automatically based on DOCKER_POD_CIDRS IPFamily() method on Cluster determines the ip family based on the service CIDRs and pod CIDRs. This is used by the docker provider to configure the docker machines accordingly. Co-authored-by: Christian Ang <[email protected]>
1 parent 4026985 commit 54c0d30

File tree

20 files changed

+606
-58
lines changed

20 files changed

+606
-58
lines changed

‎api/v1alpha4/cluster_types.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"net"
2222
"strings"
2323

24+
"github.com/pkg/errors"
2425
corev1 "k8s.io/api/core/v1"
2526
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2627
"k8s.io/utils/pointer"
@@ -215,6 +216,87 @@ func (c *Cluster) SetConditions(conditions Conditions) {
215216
c.Status.Conditions = conditions
216217
}
217218

219+
func (c *Cluster) GetIPFamily() (ClusterIPFamily, error) {
220+
var podCIDRs, serviceCIDRs []string
221+
if c.Spec.ClusterNetwork != nil {
222+
if c.Spec.ClusterNetwork.Pods != nil {
223+
podCIDRs = c.Spec.ClusterNetwork.Pods.CIDRBlocks
224+
}
225+
if c.Spec.ClusterNetwork.Services != nil {
226+
serviceCIDRs = c.Spec.ClusterNetwork.Services.CIDRBlocks
227+
}
228+
}
229+
if len(podCIDRs) == 0 && len(serviceCIDRs) == 0 {
230+
return IPv4IPFamily, nil
231+
}
232+
233+
podsIPFamily, err := ipFamilyForCIDRStrings(podCIDRs)
234+
if err != nil {
235+
return InvalidIPFamily, fmt.Errorf("pods: %s", err)
236+
}
237+
if len(serviceCIDRs) == 0 {
238+
return podsIPFamily, nil
239+
}
240+
241+
servicesIPFamily, err := ipFamilyForCIDRStrings(serviceCIDRs)
242+
if err != nil {
243+
return InvalidIPFamily, fmt.Errorf("services: %s", err)
244+
}
245+
if len(podCIDRs) == 0 {
246+
return servicesIPFamily, nil
247+
}
248+
249+
if podsIPFamily == DualStackIPFamily {
250+
return DualStackIPFamily, nil
251+
} else if podsIPFamily != servicesIPFamily {
252+
return InvalidIPFamily, errors.New("pods and services IP family mismatch")
253+
}
254+
255+
return podsIPFamily, nil
256+
}
257+
258+
func ipFamilyForCIDRStrings(cidrs []string) (ClusterIPFamily, error) {
259+
if len(cidrs) > 2 {
260+
return InvalidIPFamily, errors.New("too many CIDRs specified")
261+
}
262+
var foundIPv4 bool
263+
var foundIPv6 bool
264+
for _, cidr := range cidrs {
265+
ip, _, err := net.ParseCIDR(cidr)
266+
if err != nil {
267+
return InvalidIPFamily, fmt.Errorf("could not parse CIDR: %s", err)
268+
}
269+
if ip.To4() != nil {
270+
foundIPv4 = true
271+
} else {
272+
foundIPv6 = true
273+
}
274+
}
275+
switch {
276+
case foundIPv4 && foundIPv6:
277+
return DualStackIPFamily, nil
278+
case foundIPv4:
279+
return IPv4IPFamily, nil
280+
case foundIPv6:
281+
return IPv6IPFamily, nil
282+
default:
283+
return InvalidIPFamily, nil
284+
}
285+
}
286+
287+
type ClusterIPFamily int
288+
289+
const (
290+
InvalidIPFamily ClusterIPFamily = iota
291+
IPv4IPFamily
292+
IPv6IPFamily
293+
DualStackIPFamily
294+
)
295+
296+
func (f ClusterIPFamily) String() string {
297+
return [...]string{"InvalidIPFamily", "IPv4IPFamily", "IPv6IPFamily", "DualStackIPFamily"}[f]
298+
}
299+
218300
// +kubebuilder:object:root=true
219301

220302
// ClusterList contains a list of Cluster.

‎api/v1alpha4/cluster_types_test.go‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package v1alpha4
18+
19+
import (
20+
"testing"
21+
22+
. "github.com/onsi/gomega"
23+
)
24+
25+
func TestClusterIPFamily(t *testing.T) {
26+
clusterWithNetwork := func(podCIDRs, serviceCIDRs []string) *Cluster {
27+
return &Cluster{
28+
Spec: ClusterSpec{
29+
ClusterNetwork: &ClusterNetwork{
30+
Pods: &NetworkRanges{
31+
CIDRBlocks: podCIDRs,
32+
},
33+
Services: &NetworkRanges{
34+
CIDRBlocks: serviceCIDRs,
35+
},
36+
},
37+
},
38+
}
39+
}
40+
41+
validAndUnambiguous := []struct {
42+
name string
43+
expectRes ClusterIPFamily
44+
c *Cluster
45+
}{
46+
{
47+
name: "pods: ipv4, services: ipv4",
48+
expectRes: IPv4IPFamily,
49+
c: clusterWithNetwork([]string{"192.168.0.0/16"}, []string{"10.128.0.0/12"}),
50+
},
51+
{
52+
name: "pods: ipv4, services: nil",
53+
expectRes: IPv4IPFamily,
54+
c: clusterWithNetwork([]string{"192.168.0.0/16"}, nil),
55+
},
56+
{
57+
name: "pods: ipv6, services: nil",
58+
expectRes: IPv6IPFamily,
59+
c: clusterWithNetwork([]string{"fd00:100:96::/48"}, nil),
60+
},
61+
{
62+
name: "pods: ipv6, services: ipv6",
63+
expectRes: IPv6IPFamily,
64+
c: clusterWithNetwork([]string{"fd00:100:96::/48"}, []string{"fd00:100:64::/108"}),
65+
},
66+
{
67+
name: "pods: dual-stack, services: nil",
68+
expectRes: DualStackIPFamily,
69+
c: clusterWithNetwork([]string{"192.168.0.0/16", "fd00:100:96::/48"}, nil),
70+
},
71+
{
72+
name: "pods: dual-stack, services: ipv4",
73+
expectRes: DualStackIPFamily,
74+
c: clusterWithNetwork([]string{"192.168.0.0/16", "fd00:100:96::/48"}, []string{"10.128.0.0/12"}),
75+
},
76+
{
77+
name: "pods: dual-stack, services: ipv6",
78+
expectRes: DualStackIPFamily,
79+
c: clusterWithNetwork([]string{"192.168.0.0/16", "fd00:100:96::/48"}, []string{"fd00:100:64::/108"}),
80+
},
81+
{
82+
name: "pods: dual-stack, services: dual-stack",
83+
expectRes: DualStackIPFamily,
84+
c: clusterWithNetwork([]string{"192.168.0.0/16", "fd00:100:96::/48"}, []string{"10.128.0.0/12", "fd00:100:64::/108"}),
85+
},
86+
{
87+
name: "pods: nil, services: dual-stack",
88+
expectRes: DualStackIPFamily,
89+
c: clusterWithNetwork(nil, []string{"10.128.0.0/12", "fd00:100:64::/108"}),
90+
},
91+
}
92+
93+
for _, tt := range validAndUnambiguous {
94+
t.Run(tt.name, func(t *testing.T) {
95+
g := NewWithT(t)
96+
ipFamily, err := tt.c.GetIPFamily()
97+
g.Expect(ipFamily).To(Equal(tt.expectRes))
98+
g.Expect(err).NotTo(HaveOccurred())
99+
})
100+
}
101+
102+
validButAmbiguous := []struct {
103+
name string
104+
expectRes ClusterIPFamily
105+
c *Cluster
106+
}{
107+
{
108+
name: "pods: nil, services: nil",
109+
// this could be ipv4, ipv6, or dual-stack; assume ipv4 for now though
110+
expectRes: IPv4IPFamily,
111+
c: clusterWithNetwork(nil, nil),
112+
},
113+
{
114+
name: "pods: nil, services: ipv4",
115+
// this could be a dual-stack; assume ipv4 for now though
116+
expectRes: IPv4IPFamily,
117+
c: clusterWithNetwork(nil, []string{"10.128.0.0/12"}),
118+
},
119+
{
120+
name: "pods: nil, services: ipv6",
121+
// this could be dual-stack; assume ipv6 for now though
122+
expectRes: IPv6IPFamily,
123+
c: clusterWithNetwork(nil, []string{"fd00:100:64::/108"}),
124+
},
125+
}
126+
127+
for _, tt := range validButAmbiguous {
128+
t.Run(tt.name, func(t *testing.T) {
129+
g := NewWithT(t)
130+
ipFamily, err := tt.c.GetIPFamily()
131+
g.Expect(ipFamily).To(Equal(tt.expectRes))
132+
g.Expect(err).NotTo(HaveOccurred())
133+
})
134+
}
135+
136+
invalid := []struct {
137+
name string
138+
expectErr string
139+
c *Cluster
140+
}{
141+
{
142+
name: "pods: ipv4, services: ipv6",
143+
expectErr: "pods and services IP family mismatch",
144+
c: clusterWithNetwork([]string{"192.168.0.0/16"}, []string{"fd00:100:64::/108"}),
145+
},
146+
{
147+
name: "pods: ipv6, services: ipv4",
148+
expectErr: "pods and services IP family mismatch",
149+
c: clusterWithNetwork([]string{"fd00:100:96::/48"}, []string{"10.128.0.0/12"}),
150+
},
151+
{
152+
name: "pods: ipv6, services: dual-stack",
153+
expectErr: "pods and services IP family mismatch",
154+
c: clusterWithNetwork([]string{"fd00:100:96::/48"}, []string{"10.128.0.0/12", "fd00:100:64::/108"}),
155+
},
156+
{
157+
name: "pods: ipv4, services: dual-stack",
158+
expectErr: "pods and services IP family mismatch",
159+
c: clusterWithNetwork([]string{"192.168.0.0/16"}, []string{"10.128.0.0/12", "fd00:100:64::/108"}),
160+
},
161+
{
162+
name: "pods: ipv4, services: dual-stack",
163+
expectErr: "pods and services IP family mismatch",
164+
c: clusterWithNetwork([]string{"192.168.0.0/16"}, []string{"10.128.0.0/12", "fd00:100:64::/108"}),
165+
},
166+
{
167+
name: "pods: bad cidr",
168+
expectErr: "pods: could not parse CIDR",
169+
c: clusterWithNetwork([]string{"foo"}, nil),
170+
},
171+
{
172+
name: "services: bad cidr",
173+
expectErr: "services: could not parse CIDR",
174+
c: clusterWithNetwork([]string{"192.168.0.0/16"}, []string{"foo"}),
175+
},
176+
{
177+
name: "pods: too many cidrs",
178+
expectErr: "pods: too many CIDRs specified",
179+
c: clusterWithNetwork([]string{"192.168.0.0/16", "fd00:100:96::/48", "10.128.0.0/12"}, nil),
180+
},
181+
{
182+
name: "services: too many cidrs",
183+
expectErr: "services: too many CIDRs specified",
184+
c: clusterWithNetwork(nil, []string{"192.168.0.0/16", "fd00:100:96::/48", "10.128.0.0/12"}),
185+
},
186+
}
187+
188+
for _, tt := range invalid {
189+
t.Run(tt.name, func(t *testing.T) {
190+
g := NewWithT(t)
191+
ipFamily, err := tt.c.GetIPFamily()
192+
g.Expect(err).To(HaveOccurred())
193+
g.Expect(err).To(MatchError(ContainSubstring(tt.expectErr)))
194+
g.Expect(ipFamily).To(Equal(InvalidIPFamily))
195+
})
196+
}
197+
}

‎test/e2e/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ cluster-templates-v1alpha4: $(KUSTOMIZE) ## Generate cluster templates for v1alp
6969
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-node-drain --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-node-drain.yaml
7070
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-upgrades --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-upgrades.yaml
7171
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in.yaml
72+
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6 --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6.yaml
7273
## --------------------------------------
7374
## Testing
7475
## --------------------------------------

‎test/e2e/common.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const (
4040
KubernetesVersionUpgradeTo = "KUBERNETES_VERSION_UPGRADE_TO"
4141
EtcdVersionUpgradeTo = "ETCD_VERSION_UPGRADE_TO"
4242
CoreDNSVersionUpgradeTo = "COREDNS_VERSION_UPGRADE_TO"
43+
IPFamily = "IP_FAMILY"
4344
)
4445

4546
func Byf(format string, a ...interface{}) {

‎test/e2e/config/docker.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ providers:
104104
- sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-node-drain.yaml"
105105
- sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-upgrades.yaml"
106106
- sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-kcp-scale-in.yaml"
107+
- sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-ipv6.yaml"
107108
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
108109

109110
variables:
@@ -115,8 +116,8 @@ variables:
115116
KUBERNETES_VERSION_UPGRADE_TO: "v1.19.1"
116117
KUBERNETES_VERSION_UPGRADE_FROM: "v1.18.2"
117118
DOCKER_SERVICE_DOMAIN: "cluster.local"
119+
IP_FAMILY: "IPv4"
118120
DOCKER_SERVICE_CIDRS: "10.128.0.0/12"
119-
# IMPORTANT! This values should match the one used by the CNI provider
120121
DOCKER_POD_CIDRS: "192.168.0.0/16"
121122
CNI: "./data/cni/kindnet/kindnet.yaml"
122123
EXP_CLUSTER_RESOURCE_SET: "true"

‎test/e2e/data/cni/kindnet/kindnet.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ spec:
7878
fieldRef:
7979
fieldPath: status.podIP
8080
- name: POD_SUBNET
81-
value: "192.168.0.0/16"
81+
value: '${DOCKER_POD_CIDRS}'
8282
volumeMounts:
8383
- name: cni-cfg
8484
mountPath: /etc/cni/net.d

‎test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-ipv6/kcp-ipv6.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
kind: KubeadmControlPlane
3+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
4+
metadata:
5+
name: "${CLUSTER_NAME}-control-plane"
6+
spec:
7+
kubeadmConfigSpec:
8+
clusterConfiguration:
9+
apiServer:
10+
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
11+
certSANs: [localhost, "::", "::1", host.docker.internal]
12+
initConfiguration:
13+
localAPIEndpoint:
14+
advertiseAddress: '::'
15+
bindPort: 6443
16+
nodeRegistration:
17+
kubeletExtraArgs:
18+
node-ip: "::"
19+
joinConfiguration:
20+
nodeRegistration:
21+
kubeletExtraArgs:
22+
node-ip: "::"

‎test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-ipv6/kustomization.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
bases:
2+
- ../bases/cluster-with-kcp.yaml
3+
- ../bases/md.yaml
4+
- ../bases/crs.yaml
5+
6+
patchesStrategicMerge:
7+
- ./md-ipv6.yaml
8+
- ./kcp-ipv6.yaml

‎test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-ipv6/md-ipv6.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
3+
kind: KubeadmConfigTemplate
4+
metadata:
5+
name: "${CLUSTER_NAME}-md-0"
6+
spec:
7+
template:
8+
spec:
9+
initConfiguration:
10+
nodeRegistration:
11+
kubeletExtraArgs:
12+
node-ip: "::"
13+
joinConfiguration:
14+
nodeRegistration:
15+
kubeletExtraArgs:
16+
node-ip: "::"

‎test/e2e/e2e_suite_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme,
185185
Name: config.ManagementClusterName,
186186
RequiresDockerSock: config.HasDockerProvider(),
187187
Images: config.Images,
188+
IPFamily: config.GetVariable(IPFamily),
188189
})
189190
Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster")
190191

0 commit comments

Comments
 (0)