From 6736e2467313ac199876669e85b410543f0fb270 Mon Sep 17 00:00:00 2001 From: Segator Date: Tue, 26 Nov 2019 20:27:06 +0100 Subject: [PATCH 1/4] support hostgw --- pkg/agent/flannel/setup.go | 7 ++++++- pkg/daemons/config/types.go | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index dbf5ce8ac0..caf629799f 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -44,6 +44,9 @@ const ( "Backend": %backend% } ` + hostGwBackend = `{ + "Type": "host-gw" +}` vxlanBackend = `{ "Type": "vxlan" @@ -127,6 +130,8 @@ func createFlannelConf(nodeConfig *config.Node) error { } case config.FlannelBackendWireguard: backendConf = wireguardBackend + case config.FlannelBackendHOSTGW: + backendConf = hostGwBackend default: return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend) } @@ -161,4 +166,4 @@ func setupStrongSwan(nodeConfig *config.Node) error { // make new strongswan link return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir) -} +} \ No newline at end of file diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index d1f23fc9d1..ef99343b3f 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -18,6 +18,7 @@ const ( FlannelBackendVXLAN = "vxlan" FlannelBackendIPSEC = "ipsec" FlannelBackendWireguard = "wireguard" + FlannelBackendHOSTGW = "hostgw" ) type Node struct { From c23f12765e04af778846ff019f2be8a2cf196d9a Mon Sep 17 00:00:00 2001 From: Segator Date: Tue, 26 Nov 2019 21:30:12 +0100 Subject: [PATCH 2/4] hostgw flannel support --- pkg/agent/flannel/flannel.go | 1 + pkg/agent/flannel/setup.go | 2 +- pkg/cli/cmds/server.go | 2 +- .../coreos/flannel/backend/hostgw/hostgw.go | 89 ++++++ .../flannel/backend/hostgw/hostgw_windows.go | 253 ++++++++++++++++++ 5 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go create mode 100644 vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go diff --git a/pkg/agent/flannel/flannel.go b/pkg/agent/flannel/flannel.go index 8526772813..6577498f4a 100644 --- a/pkg/agent/flannel/flannel.go +++ b/pkg/agent/flannel/flannel.go @@ -30,6 +30,7 @@ import ( // Backends need to be imported for their init() to get executed and them to register _ "github.com/coreos/flannel/backend/extension" + _ "github.com/coreos/flannel/backend/hostgw" _ "github.com/coreos/flannel/backend/ipsec" _ "github.com/coreos/flannel/backend/vxlan" ) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index caf629799f..809503265c 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -166,4 +166,4 @@ func setupStrongSwan(nodeConfig *config.Node) error { // make new strongswan link return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir) -} \ No newline at end of file +} diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 138d130297..a00ee84930 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -114,7 +114,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { }, cli.StringFlag{ Name: "flannel-backend", - Usage: fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', or 'wireguard'"), + Usage: fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', 'hostgw', or 'wireguard'"), Destination: &ServerConfig.FlannelBackend, Value: "vxlan", }, diff --git a/vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go new file mode 100644 index 0000000000..ebcbfc8eb4 --- /dev/null +++ b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw.go @@ -0,0 +1,89 @@ +// +build !windows + +// Copyright 2015 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +build !windows + +package hostgw + +import ( + "fmt" + + "sync" + + "github.com/coreos/flannel/backend" + "github.com/coreos/flannel/pkg/ip" + "github.com/coreos/flannel/subnet" + "github.com/vishvananda/netlink" + "golang.org/x/net/context" +) + +func init() { + backend.Register("host-gw", New) +} + +type HostgwBackend struct { + sm subnet.Manager + extIface *backend.ExternalInterface +} + +func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) { + if !extIface.ExtAddr.Equal(extIface.IfaceAddr) { + return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend") + } + + be := &HostgwBackend{ + sm: sm, + extIface: extIface, + } + return be, nil +} + +func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, config *subnet.Config) (backend.Network, error) { + n := &backend.RouteNetwork{ + SimpleNetwork: backend.SimpleNetwork{ + ExtIface: be.extIface, + }, + SM: be.sm, + BackendType: "host-gw", + Mtu: be.extIface.Iface.MTU, + LinkIndex: be.extIface.Iface.Index, + } + n.GetRoute = func(lease *subnet.Lease) *netlink.Route { + return &netlink.Route{ + Dst: lease.Subnet.ToIPNet(), + Gw: lease.Attrs.PublicIP.ToIP(), + LinkIndex: n.LinkIndex, + } + } + + attrs := subnet.LeaseAttrs{ + PublicIP: ip.FromIP(be.extIface.ExtAddr), + BackendType: "host-gw", + } + + l, err := be.sm.AcquireLease(ctx, &attrs) + switch err { + case nil: + n.SubnetLease = l + + case context.Canceled, context.DeadlineExceeded: + return nil, err + + default: + return nil, fmt.Errorf("failed to acquire lease: %v", err) + } + + return n, nil +} diff --git a/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go new file mode 100644 index 0000000000..9741f189f7 --- /dev/null +++ b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go @@ -0,0 +1,253 @@ +// Copyright 2018 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostgw + +import ( + "fmt" + "strconv" + "sync" + "time" + + "github.com/Microsoft/hcsshim" + "github.com/coreos/flannel/backend" + "github.com/coreos/flannel/pkg/ip" + "github.com/coreos/flannel/subnet" + log "github.com/golang/glog" + "github.com/juju/errors" + "github.com/rakelkar/gonetsh/netroute" + "github.com/rakelkar/gonetsh/netsh" + "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/wait" + utilexec "k8s.io/utils/exec" +) + +func init() { + backend.Register("host-gw", New) +} + +type HostgwBackend struct { + sm subnet.Manager + extIface *backend.ExternalInterface +} + +func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) { + if !extIface.ExtAddr.Equal(extIface.IfaceAddr) { + return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend") + } + + be := &HostgwBackend{ + sm: sm, + extIface: extIface, + } + + return be, nil +} + +func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, config *subnet.Config) (backend.Network, error) { + // 1. Parse configuration + cfg := struct { + Name string + DNSServerList string + }{} + if len(config.Backend) > 0 { + if err := json.Unmarshal(config.Backend, &cfg); err != nil { + return nil, errors.Annotate(err, "error decoding windows host-gw backend config") + } + } + if len(cfg.Name) == 0 { + cfg.Name = "cbr0" + } + log.Infof("HOST-GW config: %+v", cfg) + + n := &backend.RouteNetwork{ + SimpleNetwork: backend.SimpleNetwork{ + ExtIface: be.extIface, + }, + SM: be.sm, + BackendType: "host-gw", + Mtu: be.extIface.Iface.MTU, + LinkIndex: be.extIface.Iface.Index, + } + n.GetRoute = func(lease *subnet.Lease) *netroute.Route { + return &netroute.Route{ + DestinationSubnet: lease.Subnet.ToIPNet(), + GatewayAddress: lease.Attrs.PublicIP.ToIP(), + LinkIndex: n.LinkIndex, + } + } + + // 2. Acquire the lease form subnet manager + attrs := subnet.LeaseAttrs{ + PublicIP: ip.FromIP(be.extIface.ExtAddr), + BackendType: "host-gw", + } + + l, err := be.sm.AcquireLease(ctx, &attrs) + switch err { + case nil: + n.SubnetLease = l + + case context.Canceled, context.DeadlineExceeded: + return nil, err + + default: + return nil, errors.Annotate(err, "failed to acquire lease") + } + + // 3. Check if the network exists and has the expected settings + netshHelper := netsh.New(utilexec.New()) + createNewNetwork := true + expectedSubnet := n.SubnetLease.Subnet + expectedAddressPrefix := expectedSubnet.String() + expectedGatewayAddress := (expectedSubnet.IP + 1).String() + expectedPodGatewayAddress := expectedSubnet.IP + 2 + networkName := cfg.Name + var waitErr, lastErr error + + existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err == nil { + for _, subnet := range existingNetwork.Subnets { + if subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGatewayAddress { + createNewNetwork = false + log.Infof("Found existing HNSNetwork %s", networkName) + break + } + } + } + + // 4. Create a new HNSNetwork + expectedNetwork := existingNetwork + if createNewNetwork { + if existingNetwork != nil { + if _, err := existingNetwork.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete existing HNSNetwork %s", networkName) + } + log.Infof("Deleted stale HNSNetwork %s", networkName) + } + + expectedNetwork = &hcsshim.HNSNetwork{ + Name: networkName, + Type: "L2Bridge", + DNSServerList: cfg.DNSServerList, + Subnets: []hcsshim.Subnet{ + { + AddressPrefix: expectedAddressPrefix, + GatewayAddress: expectedGatewayAddress, + }, + }, + } + jsonRequest, err := json.Marshal(expectedNetwork) + if err != nil { + return nil, errors.Annotatef(err, "failed to marshal %+v", expectedNetwork) + } + + log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest)) + newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest)) + if err != nil { + return nil, errors.Annotatef(err, "failed to create HNSNetwork %s", networkName) + } + + // Wait for the network to populate Management IP + log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName) + waitErr = wait.Poll(500*time.Millisecond, 30*time.Second, func() (done bool, err error) { + newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetwork.Id, "") + return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(waitErr, "timeout, failed to get management IP from HNSNetwork %s", networkName) + } + + // Wait for the interface with the management IP + log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP) + waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) { + _, lastErr = netshHelper.GetInterfaceByIP(newNetwork.ManagementIP) + return lastErr == nil, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP) + } + + log.Infof("Created HNSNetwork %s", networkName) + expectedNetwork = newNetwork + } + + // 5. Ensure a 1.2 endpoint on this network in the host compartment + createNewBridgeEndpoint := true + bridgeEndpointName := networkName + "_ep" + existingBridgeEndpoint, err := hcsshim.GetHNSEndpointByName(bridgeEndpointName) + if err == nil && existingBridgeEndpoint.IPAddress.String() == expectedPodGatewayAddress.String() { + log.Infof("Found existing bridge HNSEndpoint %s", bridgeEndpointName) + createNewBridgeEndpoint = false + } + + // 6. Create a bridge HNSEndpoint + expectedBridgeEndpoint := existingBridgeEndpoint + if createNewBridgeEndpoint { + if existingBridgeEndpoint != nil { + if _, err = existingBridgeEndpoint.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete existing bridge HNSEndpoint %s", bridgeEndpointName) + } + log.Infof("Deleted stale bridge HNSEndpoint %s", bridgeEndpointName) + } + + expectedBridgeEndpoint = &hcsshim.HNSEndpoint{ + Name: bridgeEndpointName, + IPAddress: expectedPodGatewayAddress.ToIP(), + VirtualNetwork: expectedNetwork.Id, + } + + log.Infof("Attempting to create bridge HNSEndpoint %+v", expectedBridgeEndpoint) + if expectedBridgeEndpoint, err = expectedBridgeEndpoint.Create(); err != nil { + return nil, errors.Annotatef(err, "failed to create bridge HNSEndpoint %s", bridgeEndpointName) + } + + log.Infof("Created bridge HNSEndpoint %s", bridgeEndpointName) + } + + // Wait for the bridgeEndpoint to attach to the host + log.Infof("Waiting to attach bridge endpoint %s to host", bridgeEndpointName) + waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) { + lastErr = expectedBridgeEndpoint.HostAttach(1) + return lastErr == nil, nil + }) + if waitErr == wait.ErrWaitTimeout { + return nil, errors.Annotatef(lastErr, "failed to hot attach bridge HNSEndpoint %s to host compartment", bridgeEndpointName) + } + log.Infof("Attached bridge endpoint %s to host successfully", bridgeEndpointName) + + // 7. Enable forwarding on the host interface and endpoint + for _, interfaceIpAddress := range []string{expectedNetwork.ManagementIP, expectedBridgeEndpoint.IPAddress.String()} { + netInterface, err := netshHelper.GetInterfaceByIP(interfaceIpAddress) + if err != nil { + return nil, errors.Annotatef(err, "failed to find interface for IP Address %s", interfaceIpAddress) + } + log.Infof("Found %+v interface with IP %s", netInterface, interfaceIpAddress) + + // When a new hns network is created, the interface is modified, esp the name, index + if expectedNetwork.ManagementIP == netInterface.IpAddress { + n.LinkIndex = netInterface.Idx + n.Name = netInterface.Name + } + + interfaceIdx := strconv.Itoa(netInterface.Idx) + if err := netshHelper.EnableForwarding(interfaceIdx); err != nil { + return nil, errors.Annotatef(err, "failed to enable forwarding on %s index %s", netInterface.Name, interfaceIdx) + } + log.Infof("Enabled forwarding on %s index %s", netInterface.Name, interfaceIdx) + } + + return n, nil +} From 7675f9f85cdda933775da58f98532884bc2c008c Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Wed, 27 Nov 2019 10:25:55 -0700 Subject: [PATCH 3/4] Clean up host-gw variable names --- pkg/agent/flannel/setup.go | 11 ++++++----- pkg/cli/cmds/server.go | 2 +- pkg/daemons/config/types.go | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index 809503265c..fd84e3689f 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -44,14 +44,15 @@ const ( "Backend": %backend% } ` - hostGwBackend = `{ - "Type": "host-gw" -}` vxlanBackend = `{ "Type": "vxlan" }` + hostGWBackend = `{ + "Type": "host-gw" +}` + ipsecBackend = `{ "Type": "ipsec", "UDPEncap": true, @@ -123,6 +124,8 @@ func createFlannelConf(nodeConfig *config.Node) error { switch nodeConfig.FlannelBackend { case config.FlannelBackendVXLAN: backendConf = vxlanBackend + case config.FlannelBackendHostGW: + backendConf = hostGWBackend case config.FlannelBackendIPSEC: backendConf = strings.Replace(ipsecBackend, "%psk%", nodeConfig.AgentConfig.IPSECPSK, -1) if err := setupStrongSwan(nodeConfig); err != nil { @@ -130,8 +133,6 @@ func createFlannelConf(nodeConfig *config.Node) error { } case config.FlannelBackendWireguard: backendConf = wireguardBackend - case config.FlannelBackendHOSTGW: - backendConf = hostGwBackend default: return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend) } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index a00ee84930..24f366090e 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -114,7 +114,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { }, cli.StringFlag{ Name: "flannel-backend", - Usage: fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', 'hostgw', or 'wireguard'"), + Usage: fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', 'host-gw', or 'wireguard'"), Destination: &ServerConfig.FlannelBackend, Value: "vxlan", }, diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index ef99343b3f..a80f363ff5 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -16,9 +16,9 @@ import ( const ( FlannelBackendNone = "none" FlannelBackendVXLAN = "vxlan" + FlannelBackendHostGW = "host-gw" FlannelBackendIPSEC = "ipsec" FlannelBackendWireguard = "wireguard" - FlannelBackendHOSTGW = "hostgw" ) type Node struct { From 50145002a6fb9fb9d20681862f4f0bd7468ce13d Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Wed, 27 Nov 2019 10:33:04 -0700 Subject: [PATCH 4/4] Update vendor --- .../github.com/coreos/flannel/backend/hostgw/hostgw_windows.go | 2 +- vendor/modules.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go index 9741f189f7..3e441f36ba 100644 --- a/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go +++ b/vendor/github.com/coreos/flannel/backend/hostgw/hostgw_windows.go @@ -24,7 +24,7 @@ import ( "github.com/coreos/flannel/backend" "github.com/coreos/flannel/pkg/ip" "github.com/coreos/flannel/subnet" - log "github.com/golang/glog" + log "k8s.io/klog" "github.com/juju/errors" "github.com/rakelkar/gonetsh/netroute" "github.com/rakelkar/gonetsh/netsh" diff --git a/vendor/modules.txt b/vendor/modules.txt index a37342a728..0e643de35f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -331,6 +331,7 @@ github.com/containernetworking/plugins/pkg/ns # github.com/coreos/flannel v0.11.0 => github.com/rancher/flannel v0.11.0-k3s.1 github.com/coreos/flannel/backend github.com/coreos/flannel/backend/extension +github.com/coreos/flannel/backend/hostgw github.com/coreos/flannel/backend/ipsec github.com/coreos/flannel/backend/vxlan github.com/coreos/flannel/network