From 33d61bdb26d0b686cff238d3fb2e86f3404ca7fa Mon Sep 17 00:00:00 2001 From: Michal Rostecki Date: Wed, 23 Feb 2022 12:42:21 +0000 Subject: [PATCH] netpol: Add dual-stack support This change allows to define two cluster CIDRs for compatibility with Kubernetes dual-stuck, with an assumption that two CIDRs are usually IPv4 and IPv6. It does that by levearaging changes in out kube-router fork, with the following downstream release: https://github.com/k3s-io/kube-router/releases/tag/v1.3.2%2Bk3s Signed-off-by: Michal Rostecki --- go.mod | 2 ++ go.sum | 4 ++-- pkg/agent/netpol/netpol.go | 40 ++++++++++++++++++++++++++++++++++++-- pkg/cli/server/server.go | 11 ----------- 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index a10d8ffd4c..19e9a99782 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 replace ( github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.20 github.com/benmoss/go-powershell => github.com/k3s-io/go-powershell v0.0.0-20201118222746-51f4c451fbd7 + github.com/cloudnativelabs/kube-router => github.com/k3s-io/kube-router v1.3.3-0.20220405142336-8ea9a06dc0e3 github.com/containerd/aufs => github.com/containerd/aufs v1.0.0 github.com/containerd/btrfs => github.com/containerd/btrfs v1.0.0 github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.1 @@ -82,6 +83,7 @@ require ( github.com/containerd/containerd v1.6.1 github.com/containerd/fuse-overlayfs-snapshotter v1.0.4 github.com/containerd/stargz-snapshotter v0.11.3 + github.com/coreos/go-iptables v0.6.0 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/docker/docker v20.10.10+incompatible github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 diff --git a/go.sum b/go.sum index 164b9ed603..12c9454d40 100644 --- a/go.sum +++ b/go.sum @@ -172,8 +172,6 @@ github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/cloudnativelabs/kube-router v1.3.2 h1:OBnFEP8IIIiWDAWd25QXDtyXDQi6GxR0DHOP+EXcpNI= -github.com/cloudnativelabs/kube-router v1.3.2/go.mod h1:bu7wbMiNX44Rx7mSCcvgNot2jVHuaBDu/z5ygcEtAJY= github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -654,6 +652,8 @@ github.com/k3s-io/klog v1.0.0-k3s2 h1:yyvD2bQbxG7m85/pvNctLX2bUDmva5kOBvuZ77tTGB github.com/k3s-io/klog v1.0.0-k3s2/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= github.com/k3s-io/klog/v2 v2.9.0-k3s2 h1:8Dzu3wGPFMo1mPEobSEpkHWH+HXqgFXp8R7FbcdgE8k= github.com/k3s-io/klog/v2 v2.9.0-k3s2/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +github.com/k3s-io/kube-router v1.3.3-0.20220405142336-8ea9a06dc0e3 h1:Fm8ZV0dgaoCGshnwWhRgnyWVfFtdVYar+sYU5Ne67fk= +github.com/k3s-io/kube-router v1.3.3-0.20220405142336-8ea9a06dc0e3/go.mod h1:sxZiFDEBgbjXM3SHhxJpV3701TuWIXN+JIQd5k9bDls= github.com/k3s-io/kubernetes v1.22.8-k3s1 h1:cDrypialrN+y2w5fAIT6WQxLQF3md7gkODcdc606Yt4= github.com/k3s-io/kubernetes v1.22.8-k3s1/go.mod h1:657EXnAXbtiipW+npaO0wrcbF2UuXOSyYWt1zi/4VEo= github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.22.8-k3s1 h1:Qb/ZhmRdtKKbfaGOi7QPLYNDNqD9AaEa/vtV6CZ99WU= diff --git a/pkg/agent/netpol/netpol.go b/pkg/agent/netpol/netpol.go index 6df0e727ce..59e7490933 100644 --- a/pkg/agent/netpol/netpol.go +++ b/pkg/agent/netpol/netpol.go @@ -15,8 +15,12 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/healthcheck" "github.com/cloudnativelabs/kube-router/pkg/options" "github.com/cloudnativelabs/kube-router/pkg/utils" + "github.com/coreos/go-iptables/iptables" + "github.com/pkg/errors" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/util" "github.com/sirupsen/logrus" + v1core "k8s.io/api/core/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -50,7 +54,9 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { } krConfig := options.NewKubeRouterConfig() - krConfig.ClusterIPCIDR = nodeConfig.AgentConfig.ServiceCIDR.String() + krConfig.ClusterIPCIDR = util.JoinIPNets(nodeConfig.AgentConfig.ServiceCIDRs) + krConfig.EnableIPv4 = true + krConfig.EnableIPv6 = nodeConfig.AgentConfig.EnableIPv6 krConfig.NodePortRange = strings.ReplaceAll(nodeConfig.AgentConfig.ServiceNodePortRange.String(), "-", ":") krConfig.HostnameOverride = nodeConfig.AgentConfig.NodeName krConfig.MetricsEnabled = false @@ -71,6 +77,35 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { informerFactory.Start(stopCh) informerFactory.WaitForCacheSync(stopCh) + iptablesCmdHandlers := make(map[v1core.IPFamily]utils.IPTablesHandler, 2) + ipSetHandlers := make(map[v1core.IPFamily]utils.IPSetHandler, 2) + + iptHandler, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + if err != nil { + return errors.Wrap(err, "failed to create iptables handler") + } + iptablesCmdHandlers[v1core.IPv4Protocol] = iptHandler + + ipset, err := utils.NewIPSet(false) + if err != nil { + return errors.Wrap(err, "failed to create ipset handler") + } + ipSetHandlers[v1core.IPv4Protocol] = ipset + + if nodeConfig.AgentConfig.EnableIPv6 { + ipt6Handler, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) + if err != nil { + return errors.Wrap(err, "failed to create iptables handler") + } + iptablesCmdHandlers[v1core.IPv6Protocol] = ipt6Handler + + ipset, err := utils.NewIPSet(true) + if err != nil { + return errors.Wrap(err, "failed to create ipset handler") + } + ipSetHandlers[v1core.IPv6Protocol] = ipset + } + // Start kube-router healthcheck server. Netpol requires it hc, err := healthcheck.NewHealthController(krConfig) if err != nil { @@ -83,7 +118,8 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { wg.Add(1) go hc.RunCheck(healthCh, stopCh, &wg) - npc, err := netpol.NewNetworkPolicyController(client, krConfig, podInformer, npInformer, nsInformer, &sync.Mutex{}) + npc, err := netpol.NewNetworkPolicyController(client, krConfig, podInformer, npInformer, nsInformer, &sync.Mutex{}, + iptablesCmdHandlers, ipSetHandlers) if err != nil { return err } diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 3f8691b8f3..588c698605 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -522,22 +522,11 @@ func validateNetworkConfiguration(serverConfig server.Config) error { // Dual-stack operation requires fairly extensive manual configuration at the moment - do some // preflight checks to make sure that the user isn't trying to use flannel/npc, or trying to // enable dual-stack DNS (which we don't currently support since it's not easy to template) - dualCluster, err := utilsnet.IsDualStackCIDRs(serverConfig.ControlConfig.ClusterIPRanges) - if err != nil { - return errors.Wrap(err, "failed to validate cluster-cidr") - } - dualService, err := utilsnet.IsDualStackCIDRs(serverConfig.ControlConfig.ServiceIPRanges) - if err != nil { - return errors.Wrap(err, "failed to validate service-cidr") - } dualDNS, err := utilsnet.IsDualStackIPs(serverConfig.ControlConfig.ClusterDNSs) if err != nil { return errors.Wrap(err, "failed to validate cluster-dns") } - if (serverConfig.ControlConfig.DisableNPC == false) && (dualCluster || dualService) { - return errors.New("network policy enforcement is not compatible with dual-stack operation; server must be restarted with --disable-network-policy") - } if dualDNS == true { return errors.New("dual-stack cluster-dns is not supported") }