Add IPv6/dual-stack support (#350)

## Flags Changes

This adds the proxy-init flag `--iptables-mode` (with possible values `legacy` and `nft`), which supersedes `--firewal-bin-path` and `firewall-save-bin-path` (which still remain supported).
Also the `--ipv6` flag has been added (default `true`).

After the set of rules run via iptables are processed, if `--ipv6` is true (which is the default), the same set of rules will be run via ip6tables.

Analog changes were applied to linkerd-cni as well.

## Backwards-Compatibility

This is backwards-compatible with older control planes and upcoming control planes.
If `--ipv6` is not passed (and thus defaults to true), this doesn't impact operation even if the cluster doesn't support IPv6; the ip6tables rules are applied but they're innocuous.
OTOH if there's no kernel support for IPv6 (which is the case for github runners*) then the ip6tables command will fail but we'll just log the failure and not fail the linkerd-init container (nor the `add` command for linkerd-cni). This avoids having to explicitly set `--ipv6=false`, but it can be set if the user is aware of such limitations and wants to get rid of the errors.

## Testing Improvements

The cni-plugin-integration workflow has been simplified by using a matrix strategy, and enhanced by parameterizing the iptables-mode config.

## Linkerd IPv6 Support

This allows routing IPv6 traffic to the proxy, but is just the first step towards IPv6/dual-stack support. Control plane and proxy changes will come up next.

## (*) Github Runners IPv6 Support

Even though `modinfo` signals support for IPv6, `ip6tables` commands throw modprobe errors. Indeed, according to actions/runner-images#668 support is not there yet.
Also, according to actions/runner#3138 there are issues with hosted runners as well, but that might not affect us if we still expose an IPv4 interface to interact with github. Something to take into account when we get to IPv6 integration testing.

Author: alpeb

.github/workflows/cni-plugin-integration.yml

@@ -12,33 +12,20 @@ on:
       - justfile*
 
 jobs:
-  cni-flannel-test:
-    continue-on-error: true
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
-    steps:
-      - uses: linkerd/dev/actions/setup-tools@v43
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
-      - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-flannel
-  cni-calico-test:
-    continue-on-error: true
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
-    steps:
-      - uses: linkerd/dev/actions/setup-tools@v43
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
-      - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-calico
-  cni-cilium-test:
-    continue-on-error: true
+  cni-test:
+    strategy:
+      matrix:
+        cni: [flannel, calico, cilium]
+        iptables-mode: [legacy, nft]
     timeout-minutes: 15
     runs-on: ubuntu-latest
     steps:
       - uses: linkerd/dev/actions/setup-tools@v43
       - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-cilium
+        env:
+          IPTABLES_MODE: ${{ matrix.iptables-mode }}
+        run: just cni-plugin-test-integration-${{ matrix.cni }}
   ordering-test:
     continue-on-error: true
     timeout-minutes: 15

cni-plugin/integration/manifests/calico/linkerd-cni.yaml

@@ -80,7 +80,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "$IPTABLES_MODE",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/manifests/cilium/linkerd-cni.yaml

@@ -80,7 +80,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "$IPTABLES_MODE",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/manifests/flannel/linkerd-cni.yaml

@@ -84,7 +84,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "$IPTABLES_MODE",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/run.sh

@@ -4,9 +4,8 @@ set -euxo pipefail
 
 cd "${BASH_SOURCE[0]%/*}"
 
-# Integration tests to run. Scenario is passed in as an environment variable.
-# Default is 'flannel'
 SCENARIO=${CNI_TEST_SCENARIO:-flannel}
+IPTABLES_MODE=${IPTABLES_MODE:-legacy}
 
 # Run kubectl with the correct context.
 function k() {
@@ -25,7 +24,10 @@ function create_test_lab() {
     # can enable a testing matrix?
     # Apply all files in scenario directory. For non-flannel CNIs, this will
     # include the CNI manifest itself.
-    k apply -f "manifests/$SCENARIO/"
+    for f in ./manifests/"$SCENARIO"/*.yaml
+    do
+      envsubst < "$f" | k apply -f -
+    done
 }
 
 function cleanup() {

cni-plugin/integration/testutil/test_util.go

@@ -49,8 +49,11 @@ type ProxyInit struct {
 	PortsToRedirect       []int    `json:"ports-to-redirect"`
 	InboundPortsToIgnore  []string `json:"inbound-ports-to-ignore"`
 	OutboundPortsToIgnore []string `json:"outbound-ports-to-ignore"`
+	SubnetsToIgnore       []string `json:"subnets-to-ignore"`
 	Simulate              bool     `json:"simulate"`
 	UseWaitFlag           bool     `json:"use-wait-flag"`
+	IPTablesMode          string   `json:"iptables-mode"`
+	IPv6                  bool     `json:"ipv6"`
 }
 
 // LinkerdPlugin is what we use for CNI configuration in the plugins section

cni-plugin/main.go

@@ -52,6 +52,8 @@ type ProxyInit struct {
 	SubnetsToIgnore       []string `json:"subnets-to-ignore"`
 	Simulate              bool     `json:"simulate"`
 	UseWaitFlag           bool     `json:"use-wait-flag"`
+	IPTablesMode          string   `json:"iptables-mode"`
+	IPv6                  bool     `json:"ipv6"`
 }
 
 // Kubernetes a K8s specific struct to hold config
@@ -219,8 +221,8 @@ func cmdAdd(args *skel.CmdArgs) error {
 				SimulateOnly:          conf.ProxyInit.Simulate,
 				NetNs:                 args.Netns,
 				UseWaitFlag:           conf.ProxyInit.UseWaitFlag,
-				FirewallBinPath:       "iptables-legacy",
-				FirewallSaveBinPath:   "iptables-legacy-save",
+				IPTablesMode:          conf.ProxyInit.IPTablesMode,
+				IPv6:                  conf.ProxyInit.IPv6,
 			}
 
 			// Check if there are any overridden ports to be skipped
@@ -292,17 +294,24 @@ func cmdAdd(args *skel.CmdArgs) error {
 				options.OutboundPortsToIgnore = append(options.OutboundPortsToIgnore, skippedPorts...)
 			}
 
-			firewallConfiguration, err := cmd.BuildFirewallConfiguration(&options)
-			if err != nil {
-				logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options)
-				return err
+			// This ensures BC against linkerd2-cni older versions not yet passing this flag
+			if options.IPTablesMode == "" {
+				options.IPTablesMode = cmd.IPTablesModeLegacy
 			}
 
-			err = iptables.ConfigureFirewall(*firewallConfiguration)
-			if err != nil {
-				logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err)
+			// always trigger the IPv4 rules
+			optIPv4 := options
+			optIPv4.IPv6 = false
+			if err := buildAndConfigure(logEntry, &optIPv4); err != nil {
 				return err
 			}
+
+			// trigger the IPv6 rules
+			if options.IPv6 {
+				if err := buildAndConfigure(logEntry, &options); err != nil {
+					return err
+				}
+			}
 		} else {
 			if containsInitContainer {
 				logEntry.Debug("linkerd-cni: linkerd-init initContainer is present, skipping.")
@@ -353,6 +362,24 @@ func getAPIServerPorts(ctx context.Context, api *kubernetes.Clientset) ([]string
 	return ports, nil
 }
 
+func buildAndConfigure(logEntry *logrus.Entry, options *cmd.RootOptions) error {
+	firewallConfiguration, err := cmd.BuildFirewallConfiguration(options)
+	if err != nil {
+		logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options)
+		return err
+	}
+
+	err = iptables.ConfigureFirewall(*firewallConfiguration)
+	// We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save.
+	// If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged).
+	if !options.IPv6 && err != nil {
+		logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err)
+		return err
+	}
+
+	return nil
+}
+
 func getAnnotationOverride(ctx context.Context, api *kubernetes.Clientset, pod *v1.Pod, key string) (string, error) {
 	// Check if the annotation is present on the pod
 	if override := pod.GetObjectMeta().GetAnnotations()[key]; override != "" {

justfile

@@ -224,6 +224,7 @@ _cni-plugin-setup-cilium:
     echo "Mounted /sys/fs/bpf to cilium-test-server cluster"
     helm repo add cilium https://helm.cilium.io/
     helm install cilium cilium/cilium --version 1.13.0 \
+        --kube-context k3d-l5d-cilium-test \
         --namespace kube-system \
         --set kubeProxyReplacement=partial \
         --set hostServices.enabled=false \

proxy-init/cmd/root.go

@@ -13,6 +13,22 @@ import (
 	"github.com/linkerd/linkerd2-proxy-init/internal/util"
 )
 
+const (
+	// IPTablesModeLegacy signals the usage of the iptables-legacy commands
+	IPTablesModeLegacy = "legacy"
+	// IPTablesModeNFT signals the usage of the iptables-nft commands
+	IPTablesModeNFT = "nft"
+
+	cmdLegacy         = "iptables-legacy"
+	cmdLegacySave     = "iptables-legacy-save"
+	cmdLegacyIPv6     = "ip6tables-legacy"
+	cmdLegacyIPv6Save = "ip6tables-legacy-save"
+	cmdNFT            = "iptables-nft"
+	cmdNFTSave        = "iptables-nft-save"
+	cmdNFTIPv6        = "ip6tables-nft"
+	cmdNFTIPv6Save    = "ip6tables-nft-save"
+)
+
 // RootOptions provides the information that will be used to build a firewall configuration.
 type RootOptions struct {
 	IncomingProxyPort     int
@@ -30,6 +46,8 @@ type RootOptions struct {
 	LogLevel              string
 	FirewallBinPath       string
 	FirewallSaveBinPath   string
+	IPTablesMode          string
+	IPv6                  bool
 }
 
 func newRootOptions() *RootOptions {
@@ -47,8 +65,10 @@ func newRootOptions() *RootOptions {
 		TimeoutCloseWaitSecs:  0,
 		LogFormat:             "plain",
 		LogLevel:              "info",
-		FirewallBinPath:       "iptables-legacy",
-		FirewallSaveBinPath:   "iptables-legacy-save",
+		FirewallBinPath:       "",
+		FirewallSaveBinPath:   "",
+		IPTablesMode:          "",
+		IPv6:                  true,
 	}
 }
 
@@ -61,7 +81,7 @@ func NewRootCmd() *cobra.Command {
 		Use:   "proxy-init",
 		Short: "proxy-init adds a Kubernetes pod to the Linkerd service mesh",
 		Long:  "proxy-init adds a Kubernetes pod to the Linkerd service mesh.",
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 
 			if options.TimeoutCloseWaitSecs != 0 {
 				sysctl := exec.Command("sysctl", "-w",
@@ -75,16 +95,39 @@ func NewRootCmd() *cobra.Command {
 				log.Info(string(out))
 			}
 
-			config, err := BuildFirewallConfiguration(options)
+			log.SetFormatter(getFormatter(options.LogFormat))
+			err := setLogLevel(options.LogLevel)
 			if err != nil {
 				return err
 			}
-			log.SetFormatter(getFormatter(options.LogFormat))
-			err = setLogLevel(options.LogLevel)
+
+			// always trigger the IPv4 rules
+			optIPv4 := *options
+			optIPv4.IPv6 = false
+			config, err := BuildFirewallConfiguration(&optIPv4)
 			if err != nil {
 				return err
 			}
-			return iptables.ConfigureFirewall(*config)
+
+			if err = iptables.ConfigureFirewall(*config); err != nil {
+				return err
+			}
+
+			if !options.IPv6 {
+				return nil
+			}
+
+			// trigger the IPv6 rules
+			config, err = BuildFirewallConfiguration(options)
+			if err != nil {
+				return err
+			}
+
+			// We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save.
+			// If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged).
+			_ = iptables.ConfigureFirewall(*config)
+
+			return nil
 		},
 	}
 
@@ -101,13 +144,32 @@ func NewRootCmd() *cobra.Command {
 	cmd.PersistentFlags().IntVar(&options.TimeoutCloseWaitSecs, "timeout-close-wait-secs", options.TimeoutCloseWaitSecs, "Sets nf_conntrack_tcp_timeout_close_wait")
 	cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", options.LogFormat, "Configure log format ('plain' or 'json')")
 	cmd.PersistentFlags().StringVar(&options.LogLevel, "log-level", options.LogLevel, "Configure log level")
+	cmd.PersistentFlags().StringVar(&options.IPTablesMode, "iptables-mode", options.IPTablesMode, "Variant of iptables command to use (\"legacy\" or \"nft\"); overrides --firewall-bin-path and --firewall-save-bin-path")
+	cmd.PersistentFlags().BoolVar(&options.IPv6, "ipv6", options.IPv6, "Set rules both via iptables and ip6tables to support dual-stack networking")
+
+	// these two flags are kept for backwards-compatibility, but --iptables-mode is preferred
 	cmd.PersistentFlags().StringVar(&options.FirewallBinPath, "firewall-bin-path", options.FirewallBinPath, "Path to iptables binary")
 	cmd.PersistentFlags().StringVar(&options.FirewallSaveBinPath, "firewall-save-bin-path", options.FirewallSaveBinPath, "Path to iptables-save binary")
 	return cmd
 }
 
 // BuildFirewallConfiguration returns an iptables FirewallConfiguration suitable to use to configure iptables.
 func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfiguration, error) {
+	if options.IPTablesMode != "" && options.IPTablesMode != IPTablesModeLegacy && options.IPTablesMode != IPTablesModeNFT {
+		return nil, fmt.Errorf("--iptables-mode valid values are only \"%s\" and \"%s\"", IPTablesModeLegacy, IPTablesModeNFT)
+	}
+
+	if options.IPTablesMode == "" {
+		switch options.FirewallBinPath {
+		case "", cmdLegacy:
+			options.IPTablesMode = IPTablesModeLegacy
+		case cmdNFT:
+			options.IPTablesMode = IPTablesModeNFT
+		default:
+			return nil, fmt.Errorf("--firewall-bin-path valid values are only \"%s\" and \"%s\"", cmdLegacy, cmdNFT)
+		}
+	}
+
 	if !util.IsValidPort(options.IncomingProxyPort) {
 		return nil, fmt.Errorf("--incoming-proxy-port must be a valid TCP port number")
 	}
@@ -116,6 +178,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu
 		return nil, fmt.Errorf("--outgoing-proxy-port must be a valid TCP port number")
 	}
 
+	cmd, cmdSave := getCommands(options)
+
 	sanitizedSubnets := []string{}
 	for _, subnet := range options.SubnetsToIgnore {
 		subnet := strings.TrimSpace(subnet)
@@ -138,8 +202,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu
 		SimulateOnly:           options.SimulateOnly,
 		NetNs:                  options.NetNs,
 		UseWaitFlag:            options.UseWaitFlag,
-		BinPath:                options.FirewallBinPath,
-		SaveBinPath:            options.FirewallSaveBinPath,
+		BinPath:                cmd,
+		SaveBinPath:            cmdSave,
 	}
 
 	if len(options.PortsToRedirect) > 0 {
@@ -160,6 +224,21 @@ func getFormatter(format string) log.Formatter {
 	}
 }
 
+func getCommands(options *RootOptions) (string, string) {
+	if options.IPTablesMode == IPTablesModeLegacy {
+		if options.IPv6 {
+			return cmdLegacyIPv6, cmdLegacyIPv6Save
+		}
+		return cmdLegacy, cmdLegacySave
+	}
+
+	if options.IPv6 {
+		return cmdNFTIPv6, cmdNFTIPv6Save
+	}
+
+	return cmdNFT, cmdNFTSave
+}
+
 func setLogLevel(logLevel string) error {
 	level, err := log.ParseLevel(logLevel)
 	if err != nil {