Merge pull request #1468 from yuanying/host-gw-ipv6

host-gw (linux): Add dual stack support

Documentation/configuration.md

@@ -83,7 +83,7 @@ Set `healthz-port` to a non-zero value will enable a healthz server for flannel.
 
 ## Dual-stack
 
-Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan backend.
+Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan or host-gw(linux) backend.
 
 Requirements:
 * v1.0 of flannel binary from [containernetworking/plugins](https://github.com/containernetworking/plugins)
@@ -93,13 +93,15 @@ Requirements:
 
 Configuration:
 * Set flanneld daemon with "--kube-subnet-mgr" CLI option
-* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap 
+* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap
 
 If everything works as expected, flanneld should generate a `/run/flannel/subnet.env` file with IPV6 subnet and network. For example:
 
+```bash
 FLANNEL_NETWORK=10.42.0.0/16
 FLANNEL_SUBNET=10.42.0.1/24
 FLANNEL_IPV6_NETWORK=2001:cafe:42::/56
 FLANNEL_IPV6_SUBNET=2001:cafe:42::1/64
 FLANNEL_MTU=1450
 FLANNEL_IPMASQ=true
+```

backend/hostgw/hostgw.go

@@ -59,19 +59,33 @@ func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup
 		Mtu:         be.extIface.Iface.MTU,
 		LinkIndex:   be.extIface.Iface.Index,
 	}
-	n.GetRoute = func(lease *subnet.Lease) *netlink.Route {
-		return &netlink.Route{
-			Dst:       lease.Subnet.ToIPNet(),
-			Gw:        lease.Attrs.PublicIP.ToIP(),
-			LinkIndex: n.LinkIndex,
-		}
-	}
 
 	attrs := subnet.LeaseAttrs{
-		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
 		BackendType: "host-gw",
 	}
 
+	if config.EnableIPv4 {
+		attrs.PublicIP = ip.FromIP(be.extIface.ExtAddr)
+		n.GetRoute = func(lease *subnet.Lease) *netlink.Route {
+			return &netlink.Route{
+				Dst:       lease.Subnet.ToIPNet(),
+				Gw:        lease.Attrs.PublicIP.ToIP(),
+				LinkIndex: n.LinkIndex,
+			}
+		}
+	}
+
+	if config.EnableIPv6 {
+		attrs.PublicIPv6 = ip.FromIP6(be.extIface.ExtV6Addr)
+		n.GetV6Route = func(lease *subnet.Lease) *netlink.Route {
+			return &netlink.Route{
+				Dst:       lease.IPv6Subnet.ToIPNet(),
+				Gw:        lease.Attrs.PublicIPv6.ToIP(),
+				LinkIndex: n.LinkIndex,
+			}
+		}
+	}
+
 	l, err := be.sm.AcquireLease(ctx, &attrs)
 	switch err {
 	case nil:

backend/route_network.go

@@ -36,8 +36,10 @@ type RouteNetwork struct {
 	SimpleNetwork
 	BackendType string
 	routes      []netlink.Route
+	v6Routes    []netlink.Route
 	SM          subnet.Manager
 	GetRoute    func(lease *subnet.Lease) *netlink.Route
+	GetV6Route  func(lease *subnet.Lease) *netlink.Route
 	Mtu         int
 	LinkIndex   int
 }
@@ -82,54 +84,53 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) {
 	for _, evt := range batch {
 		switch evt.Type {
 		case subnet.EventAdded:
-			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
-
 			if evt.Lease.Attrs.BackendType != n.BackendType {
 				log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType)
 				continue
 			}
-			route := n.GetRoute(&evt.Lease)
 
-			n.addToRouteList(*route)
-			// Check if route exists before attempting to add it
-			routeList, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
-			if err != nil {
-				log.Warningf("Unable to list routes: %v", err)
-			}
+			if evt.Lease.EnableIPv4 {
+				log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
 
-			if len(routeList) > 0 && !routeEqual(routeList[0], *route) {
-				// Same Dst different Gw or different link index. Remove it, correct route will be added below.
-				log.Warningf("Replacing existing route to %v via %v dev index %d with %v via %v dev index %d.", evt.Lease.Subnet, routeList[0].Gw, routeList[0].LinkIndex, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex)
-				if err := netlink.RouteDel(&routeList[0]); err != nil {
-					log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
-					continue
-				}
-				n.removeFromRouteList(routeList[0])
+				route := n.GetRoute(&evt.Lease)
+				routeAdd(route, netlink.FAMILY_V4, n.addToRouteList, n.removeFromV4RouteList)
 			}
 
-			if len(routeList) > 0 && routeEqual(routeList[0], *route) {
-				// Same Dst and same Gw, keep it and do not attempt to add it.
-				log.Infof("Route to %v via %v dev index %d already exists, skipping.", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, routeList[0].LinkIndex)
-			} else if err := netlink.RouteAdd(route); err != nil {
-				log.Errorf("Error adding route to %v via %v dev index %d: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex, err)
-				continue
+			if evt.Lease.EnableIPv6 {
+				log.Infof("Subnet added: %v via %v", evt.Lease.IPv6Subnet, evt.Lease.Attrs.PublicIPv6)
+
+				route := n.GetV6Route(&evt.Lease)
+				routeAdd(route, netlink.FAMILY_V6, n.addToV6RouteList, n.removeFromV6RouteList)
 			}
 
 		case subnet.EventRemoved:
-			log.Info("Subnet removed: ", evt.Lease.Subnet)
-
 			if evt.Lease.Attrs.BackendType != n.BackendType {
 				log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType)
 				continue
 			}
 
-			route := n.GetRoute(&evt.Lease)
-			// Always remove the route from the route list.
-			n.removeFromRouteList(*route)
+			if evt.Lease.EnableIPv4 {
+				log.Info("Subnet removed: ", evt.Lease.Subnet)
 
-			if err := netlink.RouteDel(route); err != nil {
-				log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
-				continue
+				route := n.GetRoute(&evt.Lease)
+				// Always remove the route from the route list.
+				n.removeFromV4RouteList(*route)
+
+				if err := netlink.RouteDel(route); err != nil {
+					log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
+				}
+			}
+
+			if evt.Lease.EnableIPv6 {
+				log.Info("Subnet removed: ", evt.Lease.IPv6Subnet)
+
+				route := n.GetV6Route(&evt.Lease)
+				// Always remove the route from the route list.
+				n.removeFromV6RouteList(*route)
+
+				if err := netlink.RouteDel(route); err != nil {
+					log.Errorf("Error deleting route to %v: %v", evt.Lease.IPv6Subnet, err)
+				}
 			}
 
 		default:
@@ -138,22 +139,74 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) {
 	}
 }
 
-func (n *RouteNetwork) addToRouteList(route netlink.Route) {
-	for _, r := range n.routes {
-		if routeEqual(r, route) {
+func routeAdd(route *netlink.Route, ipFamily int, addToRouteList, removeFromRouteList func(netlink.Route)) {
+	addToRouteList(*route)
+	// Check if route exists before attempting to add it
+	routeList, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
+	if err != nil {
+		log.Warningf("Unable to list routes: %v", err)
+	}
+
+	if len(routeList) > 0 && !routeEqual(routeList[0], *route) {
+		// Same Dst different Gw or different link index. Remove it, correct route will be added below.
+		log.Warningf("Replacing existing route to %v with %v", routeList[0], route)
+		if err := netlink.RouteDel(&routeList[0]); err != nil {
+			log.Errorf("Effor deleteing route to %v: %v", routeList[0].Dst, err)
 			return
 		}
+		removeFromRouteList(routeList[0])
+	}
+	routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
+	if err != nil {
+		log.Warningf("Unable to list routes: %v", err)
+	}
+
+	if len(routeList) > 0 && routeEqual(routeList[0], *route) {
+		// Same Dst and same Gw, keep it and do not attempt to add it.
+		log.Infof("Route to %v already exists, skipping.", route)
+	} else if err := netlink.RouteAdd(route); err != nil {
+		log.Errorf("Error adding route to %v", route)
+		return
+	}
+	routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
+	if err != nil {
+		log.Warningf("Unable to list routes: %v", err)
+	}
+}
+
+func (n *RouteNetwork) addToRouteList(route netlink.Route) {
+	n.routes = addToRouteList(&route, n.routes)
+}
+
+func (n *RouteNetwork) addToV6RouteList(route netlink.Route) {
+	n.v6Routes = addToRouteList(&route, n.v6Routes)
+}
+
+func addToRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {
+	for _, r := range routes {
+		if routeEqual(r, *route) {
+			return routes
+		}
 	}
-	n.routes = append(n.routes, route)
+	return append(routes, *route)
 }
 
-func (n *RouteNetwork) removeFromRouteList(route netlink.Route) {
-	for index, r := range n.routes {
-		if routeEqual(r, route) {
-			n.routes = append(n.routes[:index], n.routes[index+1:]...)
-			return
+func (n *RouteNetwork) removeFromV4RouteList(route netlink.Route) {
+	n.routes = n.removeFromRouteList(&route, n.routes)
+}
+
+func (n *RouteNetwork) removeFromV6RouteList(route netlink.Route) {
+	n.v6Routes = n.removeFromRouteList(&route, n.v6Routes)
+}
+
+func (n *RouteNetwork) removeFromRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {
+	for index, r := range routes {
+		if routeEqual(r, *route) {
+			routes = append(routes[:index], routes[index+1:]...)
+			return routes
 		}
 	}
+	return routes
 }
 
 func (n *RouteNetwork) routeCheck(ctx context.Context) {
@@ -162,15 +215,24 @@ func (n *RouteNetwork) routeCheck(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-time.After(routeCheckRetries * time.Second):
-			n.checkSubnetExistInRoutes()
+			n.checkSubnetExistInV4Routes()
+			n.checkSubnetExistInV6Routes()
 		}
 	}
 }
 
-func (n *RouteNetwork) checkSubnetExistInRoutes() {
-	routeList, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+func (n *RouteNetwork) checkSubnetExistInV4Routes() {
+	n.checkSubnetExistInRoutes(n.routes, netlink.FAMILY_V4)
+}
+
+func (n *RouteNetwork) checkSubnetExistInV6Routes() {
+	n.checkSubnetExistInRoutes(n.v6Routes, netlink.FAMILY_V6)
+}
+
+func (n *RouteNetwork) checkSubnetExistInRoutes(routes []netlink.Route, ipFamily int) {
+	routeList, err := netlink.RouteList(nil, ipFamily)
 	if err == nil {
-		for _, route := range n.routes {
+		for _, route := range routes {
 			exist := false
 			for _, r := range routeList {
 				if r.Dst == nil {

backend/route_network_test.go

@@ -57,7 +57,7 @@ func TestRouteCache(t *testing.T) {
 	subnet1 := ip.IP4Net{IP: ip.FromIP(net.ParseIP("192.168.0.0")), PrefixLen: 24}
 	nw.handleSubnetEvents([]subnet.Event{
 		{Type: subnet.EventAdded, Lease: subnet.Lease{
-			Subnet: subnet1, Attrs: subnet.LeaseAttrs{PublicIP: gw1, BackendType: "host-gw"}}},
+			Subnet: subnet1, EnableIPv4: true, Attrs: subnet.LeaseAttrs{PublicIP: gw1, BackendType: "host-gw"}}},
 	})
 	if len(nw.routes) != 1 {
 		t.Fatal(nw.routes)
@@ -68,7 +68,7 @@ func TestRouteCache(t *testing.T) {
 	// change gateway of previous route
 	nw.handleSubnetEvents([]subnet.Event{
 		{Type: subnet.EventAdded, Lease: subnet.Lease{
-			Subnet: subnet1, Attrs: subnet.LeaseAttrs{PublicIP: gw2, BackendType: "host-gw"}}}})
+			Subnet: subnet1, EnableIPv4: true, Attrs: subnet.LeaseAttrs{PublicIP: gw2, BackendType: "host-gw"}}}})
 	if len(nw.routes) != 1 {
 		t.Fatal(nw.routes)
 	}
@@ -76,3 +76,58 @@ func TestRouteCache(t *testing.T) {
 		t.Fatal(nw.routes[0])
 	}
 }
+
+func TestV6RouteCache(t *testing.T) {
+	teardown := ns.SetUpNetlinkTest(t)
+	defer teardown()
+
+	la := netlink.NewLinkAttrs()
+	la.Name = "br"
+	br := &netlink.Bridge{LinkAttrs: la}
+	if err := netlink.LinkAdd(br); err != nil {
+		t.Fatal(err)
+	}
+	if err := netlink.AddrAdd(br, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("2001:db8:1::1"), Mask: net.CIDRMask(64, 128)}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := netlink.LinkSetUp(br); err != nil {
+		t.Fatal(err)
+	}
+
+	nw := RouteNetwork{
+		SimpleNetwork: SimpleNetwork{
+			ExtIface: &ExternalInterface{Iface: &net.Interface{Index: br.Attrs().Index}},
+		},
+		BackendType: "host-gw",
+		LinkIndex:   br.Attrs().Index,
+	}
+	nw.GetV6Route = func(lease *subnet.Lease) *netlink.Route {
+		return &netlink.Route{
+			Dst:       lease.IPv6Subnet.ToIPNet(),
+			Gw:        lease.Attrs.PublicIPv6.ToIP(),
+			LinkIndex: nw.LinkIndex,
+		}
+	}
+	gw1, gw2 := ip.FromIP6(net.ParseIP("2001:db8:1::2")), ip.FromIP6(net.ParseIP("::2"))
+	subnet1 := ip.IP6Net{IP: ip.FromIP6(net.ParseIP("2001:db8:ffff::")), PrefixLen: 64}
+	nw.handleSubnetEvents([]subnet.Event{
+		{Type: subnet.EventAdded, Lease: subnet.Lease{
+			IPv6Subnet: subnet1, EnableIPv6: true, Attrs: subnet.LeaseAttrs{PublicIPv6: gw1, BackendType: "host-gw"}}},
+	})
+	if len(nw.v6Routes) != 1 {
+		t.Fatal(nw.v6Routes)
+	}
+	if !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw1.ToIP(), LinkIndex: br.Attrs().Index}) {
+		t.Fatal(nw.v6Routes[0])
+	}
+	// change gateway of previous route
+	nw.handleSubnetEvents([]subnet.Event{
+		{Type: subnet.EventAdded, Lease: subnet.Lease{
+			IPv6Subnet: subnet1, EnableIPv6: true, Attrs: subnet.LeaseAttrs{PublicIPv6: gw2, BackendType: "host-gw"}}}})
+	if len(nw.v6Routes) != 1 {
+		t.Fatal(nw.v6Routes)
+	}
+	if !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw2.ToIP(), LinkIndex: br.Attrs().Index}) {
+		t.Fatal(nw.v6Routes[0])
+	}
+}

subnet/kube/kube.go

@@ -277,14 +277,15 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 		n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||
 		n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
 		(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String())) ||
-		(n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) ||
-			n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
-			n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() ||
-			n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
-			(n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String())) {
+		(attrs.PublicIPv6 != nil &&
+			(n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) ||
+				n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
+				n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() ||
+				n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
+				(n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String()))) {
 		n.Annotations[ksm.annotations.BackendType] = attrs.BackendType
 
-		//TODO -i only vxlan backend support dual stack now.
+		//TODO -i only vxlan and host-gw backends support dual stack now.
 		if (attrs.BackendType == "vxlan" && string(bd) != "null") || attrs.BackendType != "vxlan" {
 			n.Annotations[ksm.annotations.BackendData] = string(bd)
 			if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
@@ -299,7 +300,7 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 			}
 		}
 
-		if string(v6Bd) != "null" {
+		if (attrs.BackendType == "vxlan" && string(v6Bd) != "null") || (attrs.BackendType == "host-gw" && attrs.PublicIPv6 != nil) {
 			n.Annotations[ksm.annotations.BackendV6Data] = string(v6Bd)
 			if n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" {
 				if n.Annotations[ksm.annotations.BackendPublicIPv6] != n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] {
@@ -354,8 +355,8 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 	if ipv6Cidr != nil {
 		lease.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)
 	}
-	//TODO - only vxlan backend support dual stack now.
-	if attrs.BackendType != "vxlan" {
+	//TODO - only vxlan and host-gw backends support dual stack now.
+	if attrs.BackendType != "vxlan" && attrs.BackendType != "host-gw" {
 		lease.EnableIPv4 = true
 		lease.EnableIPv6 = false
 	}