flannel/backend/vxlan/device_windows.go

// Copyright 2018 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vxlan

import (
	"encoding/json"
	"github.com/Microsoft/hcsshim/hcn"
	"github.com/coreos/flannel/pkg/ip"
	log "github.com/golang/glog"
	"github.com/juju/errors"
	"github.com/rakelkar/gonetsh/netsh"
	"k8s.io/apimachinery/pkg/util/wait"
	utilexec "k8s.io/utils/exec"
	"time"
)

type vxlanDeviceAttrs struct {
	vni           uint32
	name          string
	gbp           bool
	addressPrefix ip.IP4Net
}

type vxlanDevice struct {
	link          *hcn.HostComputeNetwork
	macPrefix     string
	directRouting bool
}

func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {
	subnet := createSubnet(devAttrs.addressPrefix.String(), (devAttrs.addressPrefix.IP + 1).String(), "0.0.0.0/0")
	network := &hcn.HostComputeNetwork{
		Type: "Overlay",
		Name: devAttrs.name,
		Ipams: []hcn.Ipam{
			{
				Type: "Static",
				Subnets: []hcn.Subnet{
					*subnet,
				},
			},
		},
		Flags: hcn.EnableNonPersistent,
		SchemaVersion: hcn.SchemaVersion{
			Major: 2,
			Minor: 0,
		},
	}

	vsid := &hcn.VsidPolicySetting{
		IsolationId: devAttrs.vni,
	}
	vsidJson, err := json.Marshal(vsid)
	if err != nil {
		return nil, err
	}

	sp := &hcn.SubnetPolicy{
		Type: hcn.VSID,
	}
	sp.Settings = vsidJson

	spJson, err := json.Marshal(sp)
	if err != nil {
		return nil, err
	}

	network.Ipams[0].Subnets[0].Policies = append(network.Ipams[0].Subnets[0].Policies, spJson)

	hnsNetwork, err := ensureNetwork(network, devAttrs.addressPrefix.String())
	if err != nil {
		return nil, err
	}

	return &vxlanDevice{
		link: hnsNetwork,
	}, nil
}

func ensureNetwork(expectedNetwork *hcn.HostComputeNetwork, expectedAddressPrefix string) (*hcn.HostComputeNetwork, error) {
	createNetwork := true
	networkName := expectedNetwork.Name

	// 1. Check if the HostComputeNetwork exists and has the expected settings
	existingNetwork, err := hcn.GetNetworkByName(networkName)
	if err == nil {
		if existingNetwork.Type == expectedNetwork.Type {
			if existingNetwork.Ipams[0].Subnets[0].IpAddressPrefix == expectedAddressPrefix {
				createNetwork = false
				log.Infof("Found existing HostComputeNetwork %s", networkName)
			}
		}
	}

	// 2. Create a new HNSNetwork
	if createNetwork {
		if existingNetwork != nil {
			if err := existingNetwork.Delete(); err != nil {
				return nil, errors.Annotatef(err, "failed to delete existing HostComputeNetwork %s", networkName)
			}
			log.Infof("Deleted stale HostComputeNetwork %s", networkName)
		}

		log.Infof("Attempting to create HostComputeNetwork %v", expectedNetwork)
		newNetwork, err := expectedNetwork.Create()
		if err != nil {
			return nil, errors.Annotatef(err, "failed to create HostComputeNetwork %s", networkName)
		}

		var waitErr, lastErr error
		// Wait for the network to populate Management IP
		log.Infof("Waiting to get ManagementIP from HostComputeNetwork %s", networkName)
		waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
			newNetwork, lastErr = hcn.GetNetworkByID(newNetwork.Id)
			return newNetwork != nil && len(getManagementIP(newNetwork)) != 0, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HostComputeNetwork %s", networkName)
		}

		managementIP := getManagementIP(newNetwork)
		// Wait for the interface with the management IP
		netshHelper := netsh.New(utilexec.New())
		log.Infof("Waiting to get net interface for HostComputeNetwork %s (%s)", networkName, managementIP)
		waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
			_, lastErr = netshHelper.GetInterfaceByIP(managementIP)
			return lastErr == nil, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HostComputeNetwork %s (%s)", networkName, managementIP)
		}

		log.Infof("Created HostComputeNetwork %s", networkName)
		existingNetwork = newNetwork
	}

	addHostRoute := true
	for _, policy := range existingNetwork.Policies {
		if policy.Type == hcn.HostRoute {
			addHostRoute = false
		}
	}
	if addHostRoute {
		hostRoutePolicy := hcn.NetworkPolicy{
			Type:     hcn.HostRoute,
			Settings: []byte("{}"),
		}

		networkRequest := hcn.PolicyNetworkRequest{
			Policies: []hcn.NetworkPolicy{hostRoutePolicy},
		}
		err = existingNetwork.AddPolicy(networkRequest)
		if err != nil {
			log.Infof("Could not apply HostRoute policy for local host to local pod connectivity. This policy requires Windows 18321.1000.19h1_release.190117-1502 or newer")
		}
	}

	return existingNetwork, nil
}

func getManagementIP(network *hcn.HostComputeNetwork) string {
	for _, policy := range network.Policies {
		if policy.Type == hcn.ProviderAddress {
			policySettings := hcn.ProviderAddressEndpointPolicySetting{}
			err := json.Unmarshal(policy.Settings, &policySettings)
			if err != nil {
				return ""
			}
			return policySettings.ProviderAddress
		}
	}
	return ""
}

func createSubnet(AddressPrefix string, NextHop string, DestPrefix string) *hcn.Subnet {
	return &hcn.Subnet{
		IpAddressPrefix: AddressPrefix,
		Routes: []hcn.Route{
			{
				NextHop:           NextHop,
				DestinationPrefix: DestPrefix,
			},
		},
	}
}