flannel/backend/hostgw/hostgw_windows.go

// Copyright 2018 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hostgw

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/Microsoft/hcsshim"
	"github.com/flannel-io/flannel/backend"
	"github.com/flannel-io/flannel/pkg/ip"
	"github.com/flannel-io/flannel/pkg/routing"
	"github.com/flannel-io/flannel/subnet"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"k8s.io/apimachinery/pkg/util/json"
	"k8s.io/apimachinery/pkg/util/wait"
	log "k8s.io/klog"
)

func init() {
	backend.Register("host-gw", New)
}

type HostgwBackend struct {
	sm       subnet.Manager
	extIface *backend.ExternalInterface
}

func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
	if !extIface.ExtAddr.Equal(extIface.IfaceAddr) {
		return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend")
	}

	be := &HostgwBackend{
		sm:       sm,
		extIface: extIface,
	}

	return be, nil
}

func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
	// 1. Parse configuration
	cfg := struct {
		Name          string
		DNSServerList string
	}{}
	if len(config.Backend) > 0 {
		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
			return nil, errors.Wrap(err, "error decoding windows host-gw backend config")
		}
	}
	if len(cfg.Name) == 0 {
		cfg.Name = "cbr0"
	}
	log.Infof("HOST-GW config: %+v", cfg)

	n := &backend.RouteNetwork{
		SimpleNetwork: backend.SimpleNetwork{
			ExtIface: be.extIface,
		},
		SM:          be.sm,
		BackendType: "host-gw",
		Mtu:         be.extIface.Iface.MTU,
		LinkIndex:   be.extIface.Iface.Index,
	}
	n.GetRoute = func(lease *subnet.Lease) *routing.Route {
		return &routing.Route{
			DestinationSubnet: lease.Subnet.ToIPNet(),
			GatewayAddress:    lease.Attrs.PublicIP.ToIP(),
			InterfaceIndex:    n.LinkIndex,
		}
	}

	// 2. Acquire the lease form subnet manager
	attrs := subnet.LeaseAttrs{
		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
		BackendType: "host-gw",
	}

	l, err := be.sm.AcquireLease(ctx, &attrs)
	switch err {
	case nil:
		n.SubnetLease = l

	case context.Canceled, context.DeadlineExceeded:
		return nil, err

	default:
		return nil, errors.Wrap(err, "failed to acquire lease")
	}

	// 3. Check if the network exists and has the expected settings
	createNewNetwork := true
	expectedSubnet := n.SubnetLease.Subnet
	expectedAddressPrefix := expectedSubnet.String()
	expectedGatewayAddress := (expectedSubnet.IP + 1).String()
	expectedPodGatewayAddress := expectedSubnet.IP + 2
	networkName := cfg.Name
	var waitErr, lastErr error

	existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
	if err == nil {
		for _, subnet := range existingNetwork.Subnets {
			if subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGatewayAddress {
				createNewNetwork = false
				log.Infof("Found existing HNSNetwork %s", networkName)
				break
			}
		}
	}

	// 4. Create a new HNSNetwork
	expectedNetwork := existingNetwork
	if createNewNetwork {
		if existingNetwork != nil {
			if _, err := existingNetwork.Delete(); err != nil {
				return nil, errors.Wrapf(err, "failed to delete existing HNSNetwork %s", networkName)
			}
			log.Infof("Deleted stale HNSNetwork %s", networkName)
		}

		expectedNetwork = &hcsshim.HNSNetwork{
			Name:          networkName,
			Type:          "L2Bridge",
			DNSServerList: cfg.DNSServerList,
			Subnets: []hcsshim.Subnet{
				{
					AddressPrefix:  expectedAddressPrefix,
					GatewayAddress: expectedGatewayAddress,
				},
			},
		}
		jsonRequest, err := json.Marshal(expectedNetwork)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal %+v", expectedNetwork)
		}

		log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest))
		newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest))
		if err != nil {
			return nil, errors.Wrapf(err, "failed to create HNSNetwork %s", networkName)
		}

		// Wait for the network to populate Management IP
		log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName)
		var newNetworkID = newNetwork.Id
		waitErr = wait.Poll(500*time.Millisecond, 30*time.Second, func() (done bool, err error) {
			newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetworkID, "")
			return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			// Do not swallow the root cause
			if lastErr != nil {
				waitErr = lastErr
			}
			return nil, errors.Wrapf(waitErr, "timeout, failed to get management IP from HNSNetwork %s", networkName)
		}

		// Wait for the interface with the management IP
		log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
		managementIP, err := ip.ParseIP4(newNetwork.ManagementIP)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to parse management ip (%s)", newNetwork.ManagementIP)
		}

		waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
			_, lastErr = ip.GetInterfaceByIP(managementIP.ToIP())
			return lastErr == nil, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			return nil, errors.Wrapf(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
		}

		log.Infof("Created HNSNetwork %s", networkName)
		expectedNetwork = newNetwork
	}

	// 5. Ensure a 1.2 endpoint on this network in the host compartment
	createNewBridgeEndpoint := true
	bridgeEndpointName := networkName + "_ep"
	existingBridgeEndpoint, err := hcsshim.GetHNSEndpointByName(bridgeEndpointName)
	if err == nil && existingBridgeEndpoint.IPAddress.String() == expectedPodGatewayAddress.String() {
		log.Infof("Found existing bridge HNSEndpoint %s", bridgeEndpointName)
		createNewBridgeEndpoint = false
	}

	// 6. Create a bridge HNSEndpoint
	expectedBridgeEndpoint := existingBridgeEndpoint
	if createNewBridgeEndpoint {
		if existingBridgeEndpoint != nil {
			if _, err = existingBridgeEndpoint.Delete(); err != nil {
				return nil, errors.Wrapf(err, "failed to delete existing bridge HNSEndpoint %s", bridgeEndpointName)
			}
			log.Infof("Deleted stale bridge HNSEndpoint %s", bridgeEndpointName)
		}

		expectedBridgeEndpoint = &hcsshim.HNSEndpoint{
			Name:           bridgeEndpointName,
			IPAddress:      expectedPodGatewayAddress.ToIP(),
			VirtualNetwork: expectedNetwork.Id,
		}

		log.Infof("Attempting to create bridge HNSEndpoint %+v", expectedBridgeEndpoint)
		if expectedBridgeEndpoint, err = expectedBridgeEndpoint.Create(); err != nil {
			return nil, errors.Wrapf(err, "failed to create bridge HNSEndpoint %s", bridgeEndpointName)
		}

		log.Infof("Created bridge HNSEndpoint %s", bridgeEndpointName)
	}

	// Wait for the bridgeEndpoint to attach to the host
	log.Infof("Waiting to attach bridge endpoint %s to host", bridgeEndpointName)
	waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
		lastErr = expectedBridgeEndpoint.HostAttach(1)
		if lastErr == nil {
			return true, nil
		}
		// See https://github.com/flannel-io/flannel/issues/1391 and
		// hcsshim lacks some validations to detect the error, so we judge it by error message.
		if strings.Contains(lastErr.Error(), "This endpoint is already attached to the switch.") {
			return true, nil
		}
		return false, nil
	})
	if waitErr == wait.ErrWaitTimeout {
		return nil, errors.Wrapf(lastErr, "failed to hot attach bridge HNSEndpoint %s to host compartment", bridgeEndpointName)
	}
	log.Infof("Attached bridge endpoint %s to host successfully", bridgeEndpointName)

	// 7. Enable forwarding on the host interface and endpoint
	for _, interfaceIpAddress := range []string{expectedNetwork.ManagementIP, expectedBridgeEndpoint.IPAddress.String()} {
		ipv4, err := ip.ParseIP4(interfaceIpAddress)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to parse expected net interface ip (%s)", interfaceIpAddress)
		}

		netInterface, err := ip.GetInterfaceByIP(ipv4.ToIP())
		if err != nil {
			return nil, errors.Wrapf(err, "failed to find interface for IP Address %s", interfaceIpAddress)
		}
		log.Infof("Found %+v interface with IP %s", netInterface, interfaceIpAddress)

		// When a new hns network is created, the interface is modified, esp the name, index
		if expectedNetwork.ManagementIP == ipv4.String() {
			n.LinkIndex = netInterface.Index
			n.Name = netInterface.Name
		}

		if err := ip.EnableForwardingForInterface(netInterface); err != nil {
			return nil, errors.Wrapf(err, "failed to enable forwarding on %s index %d", netInterface.Name, netInterface.Index)
		}
		log.Infof("Enabled forwarding on %s index %d", netInterface.Name, netInterface.Index)
	}

	return n, nil
}