flannel/backend/vxlan/device_windows.go

// Copyright 2018 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vxlan

import (
	"encoding/json"
	"fmt"
	"github.com/Microsoft/hcsshim"
	"github.com/Microsoft/hcsshim/hcn"
	"github.com/buger/jsonparser"
	"github.com/coreos/flannel/pkg/ip"
	log "github.com/golang/glog"
	"github.com/juju/errors"
	"github.com/rakelkar/gonetsh/netsh"
	"k8s.io/apimachinery/pkg/util/wait"
	utilexec "k8s.io/utils/exec"
	"time"
)

type vxlanDeviceAttrs struct {
	vni           uint32
	name          string
	gbp           bool
	addressPrefix ip.IP4Net
}

type vxlanDevice struct {
	link          *hcsshim.HNSNetwork
	macPrefix     string
	directRouting bool
}

func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {
	hnsNetwork := &hcsshim.HNSNetwork{
		Name:    devAttrs.name,
		Type:    "Overlay",
		Subnets: make([]hcsshim.Subnet, 0, 1),
	}

	hnsNetwork, err := ensureNetwork(hnsNetwork, int64(devAttrs.vni), devAttrs.addressPrefix.String(), (devAttrs.addressPrefix.IP + 1).String())
	if err != nil {
		return nil, err
	}

	return &vxlanDevice{
		link: hnsNetwork,
	}, nil
}

func ensureNetwork(expectedNetwork *hcsshim.HNSNetwork, expectedVSID int64, expectedAddressPrefix, expectedGW string) (*hcsshim.HNSNetwork, error) {
	createNetwork := true
	networkName := expectedNetwork.Name

	// 1. Check if the HNSNetwork exists and has the expected settings
	existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
	if err == nil {
		if existingNetwork.Type == expectedNetwork.Type {
			for _, existingSubnet := range existingNetwork.Subnets {
				if existingSubnet.AddressPrefix == expectedAddressPrefix && existingSubnet.GatewayAddress == expectedGW {
					createNetwork = false
					log.Infof("Found existing HNSNetwork %s", networkName)
					break
				}
			}
		}
	}

	// 2. Create a new HNSNetwork
	if createNetwork {
		if existingNetwork != nil {
			if _, err := existingNetwork.Delete(); err != nil {
				return nil, errors.Annotatef(err, "failed to delete existing HNSNetwork %s", networkName)
			}
			log.Infof("Deleted stale HNSNetwork %s", networkName)
		}

		// Add a VxLan subnet
		expectedNetwork.Subnets = append(expectedNetwork.Subnets, hcsshim.Subnet{
			AddressPrefix:  expectedAddressPrefix,
			GatewayAddress: expectedGW,
			Policies: []json.RawMessage{
				[]byte(fmt.Sprintf(`{"Type":"VSID","VSID":%d}`, expectedVSID)),
			},
		})

		// Config request params
		jsonRequest, err := json.Marshal(expectedNetwork)
		if err != nil {
			return nil, errors.Annotatef(err, "failed to marshal %+v", expectedNetwork)
		}

		log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest))
		newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest))
		if err != nil {
			return nil, errors.Annotatef(err, "failed to create HNSNetwork %s", networkName)
		}

		var waitErr, lastErr error
		// Wait for the network to populate Management IP
		log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName)
		waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
			newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetwork.Id, "")
			return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HNSNetwork %s", networkName)
		}

		// Wait for the interface with the management IP
		netshHelper := netsh.New(utilexec.New())
		log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
		waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
			_, lastErr = netshHelper.GetInterfaceByIP(newNetwork.ManagementIP)
			return lastErr == nil, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
		}

		log.Infof("Created HNSNetwork %s", networkName)
		existingNetwork = newNetwork
	}

	existingNetworkV2, err := hcn.GetNetworkByID(existingNetwork.Id)
	if err != nil {
		return nil, errors.Annotatef(err, "Could not find vxlan0 in V2")
	}

	addHostRoute := true
	for _, policy := range existingNetworkV2.Policies {
		if policy.Type == hcn.HostRoute {
			addHostRoute = false
		}
	}
	if addHostRoute {
		hostRoutePolicy := hcn.NetworkPolicy{
			Type:     hcn.HostRoute,
			Settings: []byte("{}"),
		}

		networkRequest := hcn.PolicyNetworkRequest{
			Policies: []hcn.NetworkPolicy{hostRoutePolicy},
		}
		existingNetworkV2.AddPolicy(networkRequest)
	}

	return existingNetwork, nil
}

type neighbor struct {
	MAC               string
	IP                ip.IP4
	ManagementAddress string
}

func (dev *vxlanDevice) AddEndpoint(n *neighbor) error {
	endpointName := createEndpointName(n.IP)

	// 1. Check if the HNSEndpoint exists and has the expected settings
	existingEndpoint, err := hcsshim.GetHNSEndpointByName(endpointName)
	if err == nil && existingEndpoint.VirtualNetwork == dev.link.Id {
		// Check policies if there is PA type
		targetType := "PA"
		for _, policy := range existingEndpoint.Policies {
			policyType, _ := jsonparser.GetUnsafeString(policy, "Type")
			if policyType == targetType {
				actualPaIP, _ := jsonparser.GetUnsafeString(policy, targetType)
				if actualPaIP == n.ManagementAddress {
					log.Infof("Found existing remote HNSEndpoint %s", endpointName)
					return nil
				}
			}
		}
	}

	// 2. Create a new HNSNetwork
	if existingEndpoint != nil {
		if _, err := existingEndpoint.Delete(); err != nil {
			return errors.Annotatef(err, "failed to delete existing remote HNSEndpoint %s", endpointName)
		}
		log.V(4).Infof("Deleted stale HNSEndpoint %s", endpointName)
	}

	newEndpoint := &hcsshim.HNSEndpoint{
		Name:             endpointName,
		IPAddress:        n.IP.ToIP(),
		MacAddress:       n.MAC,
		VirtualNetwork:   dev.link.Id,
		IsRemoteEndpoint: true,
		Policies: []json.RawMessage{
			[]byte(fmt.Sprintf(`{"Type":"PA","PA":"%s"}`, n.ManagementAddress)),
		},
	}
	if _, err := newEndpoint.Create(); err != nil {
		return errors.Annotatef(err, "failed to create remote HNSEndpoint %s", endpointName)
	}
	log.V(4).Infof("Created HNSEndpoint %s", endpointName)

	return nil
}

func (dev *vxlanDevice) DelEndpoint(n *neighbor) error {
	endpointName := createEndpointName(n.IP)

	existingEndpoint, err := hcsshim.GetHNSEndpointByName(endpointName)
	if err == nil && existingEndpoint.VirtualNetwork == dev.link.Id {
		// Check policies if there is PA type
		targetType := "PA"
		for _, policy := range existingEndpoint.Policies {
			policyType, _ := jsonparser.GetUnsafeString(policy, "Type")
			if policyType == targetType {
				actualPaIP, _ := jsonparser.GetUnsafeString(policy, targetType)
				if actualPaIP == n.ManagementAddress {
					// Found it and delete
					if _, err := existingEndpoint.Delete(); err != nil {
						return errors.Annotatef(err, "failed to delete remote HNSEndpoint %s", endpointName)
					}

					log.V(4).Infof("Deleted HNSEndpoint %s", endpointName)
					break
				}
			}
		}
	}

	return nil
}

func (dev *vxlanDevice) ConjureMac(targetIP ip.IP4) string {
	a, b, c, d := targetIP.Octets()
	return fmt.Sprintf("%v-%02x-%02x-%02x-%02x", dev.macPrefix, a, b, c, d)
}

func createEndpointName(targetIP ip.IP4) string {
	return "remote_" + targetIP.String()
}