Merge pull request #1448 from manuelbuil/dualstack-cherryPick

Dual-stack support

Documentation/configuration.md

@@ -80,3 +80,26 @@ Any command line option can be turned into an environment variable by prefixing
 Flannel provides a health check http endpoint `healthz`. Currently this endpoint will blindly
 return http status ok(i.e. 200) when flannel is running. This feature is by default disabled.
 Set `healthz-port` to a non-zero value will enable a healthz server for flannel.
+
+## Dual-stack
+
+Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan backend.
+
+Requirements:
+* v1.0 of flannel binary from [containernetworking/plugins](https://github.com/containernetworking/plugins)
+* Nodes must have an ipv4 and ipv6 address in the main interface
+* Nodes must have an ipv4 and ipv6 address default route
+* vxlan support ipv6 tunnel require kernel version >= 3.12
+
+Configuration:
+* Set flanneld daemon with "--kube-subnet-mgr" CLI option
+* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap 
+
+If everything works as expected, flanneld should generate a `/run/flannel/subnet.env` file with IPV6 subnet and network. For example:
+
+FLANNEL_NETWORK=10.42.0.0/16
+FLANNEL_SUBNET=10.42.0.1/24
+FLANNEL_IPV6_NETWORK=2001:cafe:42::/56
+FLANNEL_IPV6_SUBNET=2001:cafe:42::1/64
+FLANNEL_MTU=1450
+FLANNEL_IPMASQ=true

backend/common.go

@@ -24,9 +24,11 @@ import (
 )
 
 type ExternalInterface struct {
-	Iface     *net.Interface
-	IfaceAddr net.IP
-	ExtAddr   net.IP
+	Iface       *net.Interface
+	IfaceAddr   net.IP
+	IfaceV6Addr net.IP
+	ExtAddr     net.IP
+	ExtV6Addr   net.IP
 }
 
 // Besides the entry points in the Backend interface, the backend's New()

backend/vxlan/device.go

@@ -124,6 +124,18 @@ func (dev *vxlanDevice) Configure(ipa ip.IP4Net, flannelnet ip.IP4Net) error {
 	return nil
 }
 
+func (dev *vxlanDevice) ConfigureIPv6(ipn ip.IP6Net, flannelnet ip.IP6Net) error {
+	if err := ip.EnsureV6AddressOnLink(ipn, flannelnet, dev.link); err != nil {
+		return fmt.Errorf("failed to ensure v6 address of interface %s: %w", dev.link.Attrs().Name, err)
+	}
+
+	if err := netlink.LinkSetUp(dev.link); err != nil {
+		return fmt.Errorf("failed to set v6 interface %s to UP state: %w", dev.link.Attrs().Name, err)
+	}
+
+	return nil
+}
+
 func (dev *vxlanDevice) MACAddr() net.HardwareAddr {
 	return dev.link.HardwareAddr
 }
@@ -131,6 +143,7 @@ func (dev *vxlanDevice) MACAddr() net.HardwareAddr {
 type neighbor struct {
 	MAC net.HardwareAddr
 	IP  ip.IP4
+	IP6 *ip.IP6
 }
 
 func (dev *vxlanDevice) AddFDB(n neighbor) error {
@@ -145,6 +158,18 @@ func (dev *vxlanDevice) AddFDB(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) AddV6FDB(n neighbor) error {
+	log.V(4).Infof("calling AddV6FDB: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighSet(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		State:        netlink.NUD_PERMANENT,
+		Family:       syscall.AF_BRIDGE,
+		Flags:        netlink.NTF_SELF,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func (dev *vxlanDevice) DelFDB(n neighbor) error {
 	log.V(4).Infof("calling DelFDB: %v, %v", n.IP, n.MAC)
 	return netlink.NeighDel(&netlink.Neigh{
@@ -156,6 +181,17 @@ func (dev *vxlanDevice) DelFDB(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) DelV6FDB(n neighbor) error {
+	log.V(4).Infof("calling DelV6FDB: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighDel(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		Family:       syscall.AF_BRIDGE,
+		Flags:        netlink.NTF_SELF,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func (dev *vxlanDevice) AddARP(n neighbor) error {
 	log.V(4).Infof("calling AddARP: %v, %v", n.IP, n.MAC)
 	return netlink.NeighSet(&netlink.Neigh{
@@ -167,6 +203,17 @@ func (dev *vxlanDevice) AddARP(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) AddV6ARP(n neighbor) error {
+	log.V(4).Infof("calling AddV6ARP: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighSet(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		State:        netlink.NUD_PERMANENT,
+		Type:         syscall.RTN_UNICAST,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func (dev *vxlanDevice) DelARP(n neighbor) error {
 	log.V(4).Infof("calling DelARP: %v, %v", n.IP, n.MAC)
 	return netlink.NeighDel(&netlink.Neigh{
@@ -178,6 +225,17 @@ func (dev *vxlanDevice) DelARP(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) DelV6ARP(n neighbor) error {
+	log.V(4).Infof("calling DelV6ARP: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighDel(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		State:        netlink.NUD_PERMANENT,
+		Type:         syscall.RTN_UNICAST,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func vxlanLinksIncompat(l1, l2 netlink.Link) string {
 	if l1.Type() != l2.Type() {
 		return fmt.Sprintf("link type: %v vs %v", l1.Type(), l2.Type())

backend/vxlan/vxlan.go

@@ -88,19 +88,34 @@ func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backen
 	return backend, nil
 }
 
-func newSubnetAttrs(publicIP net.IP, vnid uint16, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
-	data, err := json.Marshal(&vxlanLeaseAttrs{
-		VNI:     vnid,
-		VtepMAC: hardwareAddr(mac)})
-	if err != nil {
-		return nil, err
+func newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, vnid uint16, dev, v6Dev *vxlanDevice) (*subnet.LeaseAttrs, error) {
+	leaseAttrs := &subnet.LeaseAttrs{
+		BackendType: "vxlan",
+	}
+	if publicIP != nil && dev != nil {
+		data, err := json.Marshal(&vxlanLeaseAttrs{
+			VNI:     vnid,
+			VtepMAC: hardwareAddr(dev.MACAddr()),
+		})
+		if err != nil {
+			return nil, err
+		}
+		leaseAttrs.PublicIP = ip.FromIP(publicIP)
+		leaseAttrs.BackendData = json.RawMessage(data)
 	}
 
-	return &subnet.LeaseAttrs{
-		PublicIP:    ip.FromIP(publicIP),
-		BackendType: "vxlan",
-		BackendData: json.RawMessage(data),
-	}, nil
+	if publicIPv6 != nil && v6Dev != nil {
+		data, err := json.Marshal(&vxlanLeaseAttrs{
+			VNI:     vnid,
+			VtepMAC: hardwareAddr(v6Dev.MACAddr()),
+		})
+		if err != nil {
+			return nil, err
+		}
+		leaseAttrs.PublicIPv6 = ip.FromIP6(publicIPv6)
+		leaseAttrs.BackendV6Data = json.RawMessage(data)
+	}
+	return leaseAttrs, nil
 }
 
 func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
@@ -122,23 +137,43 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 	}
 	log.Infof("VXLAN config: VNI=%d Port=%d GBP=%v Learning=%v DirectRouting=%v", cfg.VNI, cfg.Port, cfg.GBP, cfg.Learning, cfg.DirectRouting)
 
-	devAttrs := vxlanDeviceAttrs{
-		vni:       uint32(cfg.VNI),
-		name:      fmt.Sprintf("flannel.%v", cfg.VNI),
-		vtepIndex: be.extIface.Iface.Index,
-		vtepAddr:  be.extIface.IfaceAddr,
-		vtepPort:  cfg.Port,
-		gbp:       cfg.GBP,
-		learning:  cfg.Learning,
-	}
+	var dev, v6Dev *vxlanDevice
+	var err error
+	if config.EnableIPv4 {
+		devAttrs := vxlanDeviceAttrs{
+			vni:       uint32(cfg.VNI),
+			name:      fmt.Sprintf("flannel.%v", cfg.VNI),
+			vtepIndex: be.extIface.Iface.Index,
+			vtepAddr:  be.extIface.IfaceAddr,
+			vtepPort:  cfg.Port,
+			gbp:       cfg.GBP,
+			learning:  cfg.Learning,
+		}
 
-	dev, err := newVXLANDevice(&devAttrs)
-	if err != nil {
-		return nil, err
+		dev, err = newVXLANDevice(&devAttrs)
+		if err != nil {
+			return nil, err
+		}
+		dev.directRouting = cfg.DirectRouting
+	}
+	if config.EnableIPv6 {
+		v6DevAttrs := vxlanDeviceAttrs{
+			vni:       uint32(cfg.VNI),
+			name:      fmt.Sprintf("flannel-v6.%v", cfg.VNI),
+			vtepIndex: be.extIface.Iface.Index,
+			vtepAddr:  be.extIface.IfaceV6Addr,
+			vtepPort:  cfg.Port,
+			gbp:       cfg.GBP,
+			learning:  cfg.Learning,
+		}
+		v6Dev, err = newVXLANDevice(&v6DevAttrs)
+		if err != nil {
+			return nil, err
+		}
+		v6Dev.directRouting = cfg.DirectRouting
 	}
-	dev.directRouting = cfg.DirectRouting
 
-	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), dev.MACAddr())
+	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, uint16(cfg.VNI), dev, v6Dev)
 	if err != nil {
 		return nil, err
 	}
@@ -155,11 +190,17 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 	// Ensure that the device has a /32 address so that no broadcast routes are created.
 	// This IP is just used as a source address for host to workload traffic (so
 	// the return path for the traffic has an address on the flannel network to use as the destination)
-	if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
-		return nil, fmt.Errorf("failed to configure interface %s: %s", dev.link.Attrs().Name, err)
+	if config.EnableIPv4 {
+		if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
+			return nil, fmt.Errorf("failed to configure interface %s: %w", dev.link.Attrs().Name, err)
+		}
 	}
-
-	return newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease)
+	if config.EnableIPv6 {
+		if err := v6Dev.ConfigureIPv6(ip.IP6Net{IP: lease.IPv6Subnet.IP, PrefixLen: 128}, config.IPv6Network); err != nil {
+			return nil, fmt.Errorf("failed to configure interface %s: %w", v6Dev.link.Attrs().Name, err)
+		}
+	}
+	return newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease)
 }
 
 // So we can make it JSON (un)marshalable

backend/vxlan/vxlan_network.go

@@ -33,6 +33,7 @@ import (
 type network struct {
 	backend.SimpleNetwork
 	dev       *vxlanDevice
+	v6Dev     *vxlanDevice
 	subnetMgr subnet.Manager
 }
 
@@ -40,7 +41,7 @@ const (
 	encapOverhead = 50
 )
 
-func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, _ ip.IP4Net, lease *subnet.Lease) (*network, error) {
+func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, v6Dev *vxlanDevice, _ ip.IP4Net, lease *subnet.Lease) (*network, error) {
 	nw := &network{
 		SimpleNetwork: backend.SimpleNetwork{
 			SubnetLease: lease,
@@ -48,6 +49,7 @@ func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, d
 		},
 		subnetMgr: subnetMgr,
 		dev:       dev,
+		v6Dev:     v6Dev,
 	}
 
 	return nw, nil
@@ -91,105 +93,214 @@ type vxlanLeaseAttrs struct {
 func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 	for _, event := range batch {
 		sn := event.Lease.Subnet
+		v6Sn := event.Lease.IPv6Subnet
 		attrs := event.Lease.Attrs
 		if attrs.BackendType != "vxlan" {
-			log.Warningf("ignoring non-vxlan subnet(%s): type=%v", sn, attrs.BackendType)
+			log.Warningf("ignoring non-vxlan v4Subnet(%s) v6Subnet(%s): type=%v", sn, v6Sn, attrs.BackendType)
 			continue
 		}
 
-		var vxlanAttrs vxlanLeaseAttrs
-		if err := json.Unmarshal(attrs.BackendData, &vxlanAttrs); err != nil {
-			log.Error("error decoding subnet lease JSON: ", err)
-			continue
-		}
+		var (
+			vxlanAttrs, v6VxlanAttrs           vxlanLeaseAttrs
+			directRoutingOK, v6DirectRoutingOK bool
+			directRoute, v6DirectRoute         netlink.Route
+			vxlanRoute, v6VxlanRoute           netlink.Route
+		)
 
-		// This route is used when traffic should be vxlan encapsulated
-		vxlanRoute := netlink.Route{
-			LinkIndex: nw.dev.link.Attrs().Index,
-			Scope:     netlink.SCOPE_UNIVERSE,
-			Dst:       sn.ToIPNet(),
-			Gw:        sn.IP.ToIP(),
-		}
-		vxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)
+		if event.Lease.EnableIPv4 && nw.dev != nil {
+			if err := json.Unmarshal(attrs.BackendData, &vxlanAttrs); err != nil {
+				log.Error("error decoding subnet lease JSON: ", err)
+				continue
+			}
 
-		// directRouting is where the remote host is on the same subnet so vxlan isn't required.
-		directRoute := netlink.Route{
-			Dst: sn.ToIPNet(),
-			Gw:  attrs.PublicIP.ToIP(),
+			// This route is used when traffic should be vxlan encapsulated
+			vxlanRoute = netlink.Route{
+				LinkIndex: nw.dev.link.Attrs().Index,
+				Scope:     netlink.SCOPE_UNIVERSE,
+				Dst:       sn.ToIPNet(),
+				Gw:        sn.IP.ToIP(),
+			}
+			vxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)
+
+			// directRouting is where the remote host is on the same subnet so vxlan isn't required.
+			directRoute = netlink.Route{
+				Dst: sn.ToIPNet(),
+				Gw:  attrs.PublicIP.ToIP(),
+			}
+			if nw.dev.directRouting {
+				if dr, err := ip.DirectRouting(attrs.PublicIP.ToIP()); err != nil {
+					log.Error(err)
+				} else {
+					directRoutingOK = dr
+				}
+			}
 		}
-		var directRoutingOK = false
-		if nw.dev.directRouting {
-			if dr, err := ip.DirectRouting(attrs.PublicIP.ToIP()); err != nil {
-				log.Error(err)
-			} else {
-				directRoutingOK = dr
+
+		if event.Lease.EnableIPv6 && nw.v6Dev != nil {
+			if err := json.Unmarshal(attrs.BackendV6Data, &v6VxlanAttrs); err != nil {
+				log.Error("error decoding v6 subnet lease JSON: ", err)
+				continue
+			}
+			if v6Sn.IP != nil && nw.v6Dev != nil {
+				v6VxlanRoute = netlink.Route{
+					LinkIndex: nw.v6Dev.link.Attrs().Index,
+					Scope:     netlink.SCOPE_UNIVERSE,
+					Dst:       v6Sn.ToIPNet(),
+					Gw:        v6Sn.IP.ToIP(),
+				}
+				v6VxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)
+
+				// directRouting is where the remote host is on the same subnet so vxlan isn't required.
+				v6DirectRoute = netlink.Route{
+					Dst: v6Sn.ToIPNet(),
+					Gw:  attrs.PublicIPv6.ToIP(),
+				}
+
+				if nw.v6Dev.directRouting {
+					if v6Dr, err := ip.DirectRouting(attrs.PublicIPv6.ToIP()); err != nil {
+						log.Error(err)
+					} else {
+						v6DirectRoutingOK = v6Dr
+					}
+				}
 			}
 		}
 
 		switch event.Type {
 		case subnet.EventAdded:
-			if directRoutingOK {
-				log.V(2).Infof("Adding direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
+			if event.Lease.EnableIPv4 {
+				if directRoutingOK {
+					log.V(2).Infof("Adding direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
 
-				if err := netlink.RouteReplace(&directRoute); err != nil {
-					log.Errorf("Error adding route to %v via %v: %v", sn, attrs.PublicIP, err)
-					continue
-				}
-			} else {
-				log.V(2).Infof("adding subnet: %s PublicIP: %s VtepMAC: %s", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))
-				if err := nw.dev.AddARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
-					log.Error("AddARP failed: ", err)
-					continue
-				}
+					if err := netlink.RouteReplace(&directRoute); err != nil {
+						log.Errorf("Error adding route to %v via %v: %v", sn, attrs.PublicIP, err)
+						continue
+					}
+				} else {
+					log.V(2).Infof("adding subnet: %s PublicIP: %s VtepMAC: %s", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))
+					if err := nw.dev.AddARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("AddARP failed: ", err)
+						continue
+					}
 
-				if err := nw.dev.AddFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
-					log.Error("AddFDB failed: ", err)
+					if err := nw.dev.AddFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("AddFDB failed: ", err)
 
-					// Try to clean up the ARP entry then continue
-					if err := nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
-						log.Error("DelARP failed: ", err)
+						// Try to clean up the ARP entry then continue
+						if err := nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelARP failed: ", err)
+						}
+
+						continue
 					}
 
-					continue
+					// Set the route - the kernel would ARP for the Gw IP address if it hadn't already been set above so make sure
+					// this is done last.
+					if err := netlink.RouteReplace(&vxlanRoute); err != nil {
+						log.Errorf("failed to add vxlanRoute (%s -> %s): %v", vxlanRoute.Dst, vxlanRoute.Gw, err)
+
+						// Try to clean up both the ARP and FDB entries then continue
+						if err := nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelARP failed: ", err)
+						}
+
+						if err := nw.dev.DelFDB(neighbor{IP: event.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelFDB failed: ", err)
+						}
+
+						continue
+					}
 				}
+			}
+			if event.Lease.EnableIPv6 {
+				if v6DirectRoutingOK {
+					log.V(2).Infof("Adding v6 direct route to v6 subnet: %s PublicIPv6: %s", v6Sn, attrs.PublicIPv6)
+
+					if err := netlink.RouteReplace(&v6DirectRoute); err != nil {
+						log.Errorf("Error adding v6 route to %v via %v: %v", v6Sn, attrs.PublicIPv6, err)
+						continue
+					}
+				} else {
+					log.V(2).Infof("adding v6 subnet: %s PublicIPv6: %s VtepMAC: %s", v6Sn, attrs.PublicIPv6, net.HardwareAddr(v6VxlanAttrs.VtepMAC))
+					if err := nw.v6Dev.AddV6ARP(neighbor{IP6: v6Sn.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("AddV6ARP failed: ", err)
+						continue
+					}
+
+					if err := nw.v6Dev.AddV6FDB(neighbor{IP6: attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("AddV6FDB failed: ", err)
+
+						// Try to clean up the ARP entry then continue
+						if err := nw.v6Dev.DelV6ARP(neighbor{IP6: event.Lease.IPv6Subnet.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelV6ARP failed: ", err)
+						}
 
-				// Set the route - the kernel would ARP for the Gw IP address if it hadn't already been set above so make sure
-				// this is done last.
-				if err := netlink.RouteReplace(&vxlanRoute); err != nil {
-					log.Errorf("failed to add vxlanRoute (%s -> %s): %v", vxlanRoute.Dst, vxlanRoute.Gw, err)
+						continue
+					}
+
+					// Set the route - the kernel would ARP for the Gw IP address if it hadn't already been set above so make sure
+					// this is done last.
+					if err := netlink.RouteReplace(&v6VxlanRoute); err != nil {
+						log.Errorf("failed to add v6 vxlanRoute (%s -> %s): %v", v6VxlanRoute.Dst, v6VxlanRoute.Gw, err)
+
+						// Try to clean up both the ARP and FDB entries then continue
+						if err := nw.v6Dev.DelV6ARP(neighbor{IP6: event.Lease.IPv6Subnet.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelV6ARP failed: ", err)
+						}
+
+						if err := nw.v6Dev.DelV6FDB(neighbor{IP6: event.Lease.Attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelV6FDB failed: ", err)
+						}
+
+						continue
+					}
+				}
+			}
+		case subnet.EventRemoved:
+			if event.Lease.EnableIPv4 {
+				if directRoutingOK {
+					log.V(2).Infof("Removing direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
+					if err := netlink.RouteDel(&directRoute); err != nil {
+						log.Errorf("Error deleting route to %v via %v: %v", sn, attrs.PublicIP, err)
+					}
+				} else {
+					log.V(2).Infof("removing subnet: %s PublicIP: %s VtepMAC: %s", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))
 
-					// Try to clean up both the ARP and FDB entries then continue
-					if err := nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
+					// Try to remove all entries - don't bail out if one of them fails.
+					if err := nw.dev.DelARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
 						log.Error("DelARP failed: ", err)
 					}
 
-					if err := nw.dev.DelFDB(neighbor{IP: event.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
+					if err := nw.dev.DelFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
 						log.Error("DelFDB failed: ", err)
 					}
 
-					continue
+					if err := netlink.RouteDel(&vxlanRoute); err != nil {
+						log.Errorf("failed to delete vxlanRoute (%s -> %s): %v", vxlanRoute.Dst, vxlanRoute.Gw, err)
+					}
 				}
 			}
-		case subnet.EventRemoved:
-			if directRoutingOK {
-				log.V(2).Infof("Removing direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
-				if err := netlink.RouteDel(&directRoute); err != nil {
-					log.Errorf("Error deleting route to %v via %v: %v", sn, attrs.PublicIP, err)
-				}
-			} else {
-				log.V(2).Infof("removing subnet: %s PublicIP: %s VtepMAC: %s", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))
+			if event.Lease.EnableIPv6 {
+				if v6DirectRoutingOK {
+					log.V(2).Infof("Removing v6 direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIPv6)
+					if err := netlink.RouteDel(&directRoute); err != nil {
+						log.Errorf("Error deleting v6 route to %v via %v: %v", v6Sn, attrs.PublicIPv6, err)
+					}
+				} else {
+					log.V(2).Infof("removing v6subnet: %s PublicIPv6: %s VtepMAC: %s", v6Sn, attrs.PublicIPv6, net.HardwareAddr(v6VxlanAttrs.VtepMAC))
 
-				// Try to remove all entries - don't bail out if one of them fails.
-				if err := nw.dev.DelARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
-					log.Error("DelARP failed: ", err)
-				}
+					// Try to remove all entries - don't bail out if one of them fails.
+					if err := nw.v6Dev.DelV6ARP(neighbor{IP6: v6Sn.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("DelV6ARP failed: ", err)
+					}
 
-				if err := nw.dev.DelFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
-					log.Error("DelFDB failed: ", err)
-				}
+					if err := nw.v6Dev.DelV6FDB(neighbor{IP6: attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("DelV6FDB failed: ", err)
+					}
 
-				if err := netlink.RouteDel(&vxlanRoute); err != nil {
-					log.Errorf("failed to delete vxlanRoute (%s -> %s): %v", vxlanRoute.Dst, vxlanRoute.Gw, err)
+					if err := netlink.RouteDel(&v6VxlanRoute); err != nil {
+						log.Errorf("failed to delete v6 vxlanRoute (%s -> %s): %v", v6VxlanRoute.Dst, v6VxlanRoute.Gw, err)
+					}
 				}
 			}
 		default:

go.sum

@@ -73,6 +73,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=

main.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"math/big"
 	"net"
 	"net/http"
 	"os"
@@ -80,6 +81,8 @@ type CmdLineOpts struct {
 	etcdPassword              string
 	help                      bool
 	version                   bool
+	autoDetectIPv4            bool
+	autoDetectIPv6            bool
 	kubeSubnetMgr             bool
 	kubeApiUrl                string
 	kubeAnnotationPrefix      string
@@ -90,6 +93,7 @@ type CmdLineOpts struct {
 	subnetFile                string
 	subnetDir                 string
 	publicIP                  string
+	publicIPv6                string
 	subnetLeaseRenewMargin    int
 	healthzIP                 string
 	healthzPort               int
@@ -108,6 +112,13 @@ var (
 	flannelFlags   = flag.NewFlagSet("flannel", flag.ExitOnError)
 )
 
+const (
+	ipv4Stack int = iota
+	ipv6Stack
+	dualStack
+	noneStack
+)
+
 func init() {
 	flannelFlags.StringVar(&opts.etcdEndpoints, "etcd-endpoints", "http://127.0.0.1:4001,http://127.0.0.1:2379", "a comma-delimited list of etcd endpoints")
 	flannelFlags.StringVar(&opts.etcdPrefix, "etcd-prefix", "/coreos.com/network", "etcd prefix")
@@ -120,6 +131,7 @@ func init() {
 	flannelFlags.Var(&opts.ifaceRegex, "iface-regex", "regex expression to match the first interface to use (IP or name) for inter-host communication. Can be specified multiple times to check each regex in order. Returns the first match found. Regexes are checked after specific interfaces specified by the iface option have already been checked.")
 	flannelFlags.StringVar(&opts.subnetFile, "subnet-file", "/run/flannel/subnet.env", "filename where env variables (subnet, MTU, ... ) will be written to")
 	flannelFlags.StringVar(&opts.publicIP, "public-ip", "", "IP accessible by other nodes for inter-host communication")
+	flannelFlags.StringVar(&opts.publicIPv6, "public-ipv6", "", "IPv6 accessible by other nodes for inter-host communication")
 	flannelFlags.IntVar(&opts.subnetLeaseRenewMargin, "subnet-lease-renew-margin", 60, "subnet lease renewal margin, in minutes, ranging from 1 to 1439")
 	flannelFlags.BoolVar(&opts.ipMasq, "ip-masq", false, "setup IP masquerade rule for traffic destined outside of overlay network")
 	flannelFlags.BoolVar(&opts.kubeSubnetMgr, "kube-subnet-mgr", false, "contact the Kubernetes API for subnet assignment instead of etcd.")
@@ -162,6 +174,17 @@ func usage() {
 	os.Exit(0)
 }
 
+func getIPFamily(autoDetectIPv4, autoDetectIPv6 bool) (int, error) {
+	if autoDetectIPv4 && !autoDetectIPv6 {
+		return ipv4Stack, nil
+	} else if !autoDetectIPv4 && autoDetectIPv6 {
+		return ipv6Stack, nil
+	} else if autoDetectIPv4 && autoDetectIPv6 {
+		return dualStack, nil
+	}
+	return noneStack, errors.New("none defined stack")
+}
+
 func newSubnetManager(ctx context.Context) (subnet.Manager, error) {
 	if opts.kubeSubnetMgr {
 		return kube.NewSubnetManager(ctx, opts.kubeApiUrl, opts.kubeConfigFile, opts.kubeAnnotationPrefix, opts.netConfPath, opts.setNodeNetworkUnavailable)
@@ -200,12 +223,56 @@ func main() {
 		os.Exit(1)
 	}
 
+	// This is the main context that everything should run in.
+	// All spawned goroutines should exit when cancel is called on this context.
+	// Go routines spawned from main.go coordinate using a WaitGroup. This provides a mechanism to allow the shutdownHandler goroutine
+	// to block until all the goroutines return . If those goroutines spawn other goroutines then they are responsible for
+	// blocking and returning only when cancel() is called.
+	ctx, cancel := context.WithCancel(context.Background())
+
+	sm, err := newSubnetManager(ctx)
+	if err != nil {
+		log.Error("Failed to create SubnetManager: ", err)
+		os.Exit(1)
+	}
+	log.Infof("Created subnet manager: %s", sm.Name())
+
+	// Register for SIGINT and SIGTERM
+	log.Info("Installing signal handlers")
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		shutdownHandler(ctx, sigs, cancel)
+		wg.Done()
+	}()
+
+	if opts.healthzPort > 0 {
+		// It's not super easy to shutdown the HTTP server so don't attempt to stop it cleanly
+		go mustRunHealthz()
+	}
+
+	// Fetch the network config (i.e. what backend to use etc..).
+	config, err := getConfig(ctx, sm)
+	if err == errCanceled {
+		wg.Wait()
+		os.Exit(0)
+	}
+
+	// Get ip family stack
+	ipStack, stackErr := getIPFamily(config.EnableIPv4, config.EnableIPv6)
+	if stackErr != nil {
+		log.Error(stackErr.Error())
+		os.Exit(1)
+	}
 	// Work out which interface to use
 	var extIface *backend.ExternalInterface
-	var err error
 	// Check the default interface only if no interfaces are specified
 	if len(opts.iface) == 0 && len(opts.ifaceRegex) == 0 {
-		extIface, err = LookupExtIface(opts.publicIP, "")
+		extIface, err = LookupExtIface(opts.publicIP, "", ipStack)
 		if err != nil {
 			log.Error("Failed to find any valid interface to use: ", err)
 			os.Exit(1)
@@ -213,7 +280,7 @@ func main() {
 	} else {
 		// Check explicitly specified interfaces
 		for _, iface := range opts.iface {
-			extIface, err = LookupExtIface(iface, "")
+			extIface, err = LookupExtIface(iface, "", ipStack)
 			if err != nil {
 				log.Infof("Could not find valid interface matching %s: %s", iface, err)
 			}
@@ -226,7 +293,7 @@ func main() {
 		// Check interfaces that match any specified regexes
 		if extIface == nil {
 			for _, ifaceRegex := range opts.ifaceRegex {
-				extIface, err = LookupExtIface("", ifaceRegex)
+				extIface, err = LookupExtIface("", ifaceRegex, ipStack)
 				if err != nil {
 					log.Infof("Could not find valid interface matching %s: %s", ifaceRegex, err)
 				}
@@ -244,44 +311,7 @@ func main() {
 		}
 	}
 
-	// This is the main context that everything should run in.
-	// All spawned goroutines should exit when cancel is called on this context.
-	// Go routines spawned from main.go coordinate using a WaitGroup. This provides a mechanism to allow the shutdownHandler goroutine
-	// to block until all the goroutines return . If those goroutines spawn other goroutines then they are responsible for
-	// blocking and returning only when cancel() is called.
-	ctx, cancel := context.WithCancel(context.Background())
-
-	sm, err := newSubnetManager(ctx)
-	if err != nil {
-		log.Error("Failed to create SubnetManager: ", err)
-		os.Exit(1)
-	}
-	log.Infof("Created subnet manager: %s", sm.Name())
-
-	// Register for SIGINT and SIGTERM
-	log.Info("Installing signal handlers")
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
 
-	wg := sync.WaitGroup{}
-
-	wg.Add(1)
-	go func() {
-		shutdownHandler(ctx, sigs, cancel)
-		wg.Done()
-	}()
-
-	if opts.healthzPort > 0 {
-		// It's not super easy to shutdown the HTTP server so don't attempt to stop it cleanly
-		go mustRunHealthz()
-	}
-
-	// Fetch the network config (i.e. what backend to use etc..).
-	config, err := getConfig(ctx, sm)
-	if err == errCanceled {
-		wg.Wait()
-		os.Exit(0)
-	}
 
 	// Create a backend manager then use it to create the backend and register the network with it.
 	bm := backend.NewManager(ctx, sm, extIface)
@@ -303,25 +333,44 @@ func main() {
 
 	// Set up ipMasq if needed
 	if opts.ipMasq {
-		if err = recycleIPTables(config.Network, bn.Lease()); err != nil {
-			log.Errorf("Failed to recycle IPTables rules, %v", err)
-			cancel()
-			wg.Wait()
-			os.Exit(1)
+		if config.EnableIPv4 {
+			if err = recycleIPTables(config.Network, bn.Lease()); err != nil {
+				log.Errorf("Failed to recycle IPTables rules, %v", err)
+				cancel()
+				wg.Wait()
+				os.Exit(1)
+			}
+			log.Infof("Setting up masking rules")
+			go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()), opts.iptablesResyncSeconds)
+
+		}
+		if config.EnableIPv6 {
+			if err = recycleIP6Tables(config.IPv6Network, bn.Lease()); err != nil {
+				log.Errorf("Failed to recycle IP6Tables rules, %v", err)
+				cancel()
+				wg.Wait()
+				os.Exit(1)
+			}
+			log.Infof("Setting up masking ip6 rules")
+			go network.SetupAndEnsureIP6Tables(network.MasqIP6Rules(config.IPv6Network, bn.Lease()), opts.iptablesResyncSeconds)
 		}
-		log.Infof("Setting up masking rules")
-		go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()), opts.iptablesResyncSeconds)
 	}
 
 	// Always enables forwarding rules. This is needed for Docker versions >1.13 (https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#container-communication-between-hosts)
 	// In Docker 1.12 and earlier, the default FORWARD chain policy was ACCEPT.
 	// In Docker 1.13 and later, Docker sets the default policy of the FORWARD chain to DROP.
 	if opts.iptablesForwardRules {
-		log.Infof("Changing default FORWARD chain policy to ACCEPT")
-		go network.SetupAndEnsureIPTables(network.ForwardRules(config.Network.String()), opts.iptablesResyncSeconds)
+		if config.EnableIPv4 {
+			log.Infof("Changing default FORWARD chain policy to ACCEPT")
+			go network.SetupAndEnsureIPTables(network.ForwardRules(config.Network.String()), opts.iptablesResyncSeconds)
+		}
+		if config.EnableIPv6 {
+			log.Infof("IPv6: Changing default FORWARD chain policy to ACCEPT")
+			go network.SetupAndEnsureIP6Tables(network.ForwardRules(config.IPv6Network.String()), opts.iptablesResyncSeconds)
+		}
 	}
 
-	if err := WriteSubnetFile(opts.subnetFile, config.Network, opts.ipMasq, bn); err != nil {
+	if err := WriteSubnetFile(opts.subnetFile, config, opts.ipMasq, bn); err != nil {
 		// Continue, even though it failed.
 		log.Warningf("Failed to write subnet file: %s", err)
 	} else {
@@ -370,6 +419,22 @@ func recycleIPTables(nw ip.IP4Net, lease *subnet.Lease) error {
 	return nil
 }
 
+func recycleIP6Tables(nw ip.IP6Net, lease *subnet.Lease) error {
+	prevNetwork := ReadIP6CIDRFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_NETWORK")
+	prevSubnet := ReadIP6CIDRFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_SUBNET")
+	// recycle iptables rules only when network configured or subnet leased is not equal to current one.
+	if prevNetwork.String() != nw.String() && prevSubnet.String() != lease.IPv6Subnet.String() {
+		log.Infof("Current ipv6 network or subnet (%v, %v) is not equal to previous one (%v, %v), trying to recycle old ip6tables rules", nw, lease.IPv6Subnet, prevNetwork, prevSubnet)
+		lease := &subnet.Lease{
+			IPv6Subnet: prevSubnet,
+		}
+		if err := network.DeleteIP6Tables(network.MasqIP6Rules(prevNetwork, lease)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func shutdownHandler(ctx context.Context, sigs chan os.Signal, cancel context.CancelFunc) {
 	// Wait for the context do be Done or for the signal to come in to shutdown.
 	select {
@@ -451,17 +516,48 @@ func MonitorLease(ctx context.Context, sm subnet.Manager, bn backend.Network, wg
 	}
 }
 
-func LookupExtIface(ifname string, ifregex string) (*backend.ExternalInterface, error) {
+func LookupExtIface(ifname string, ifregexS string, ipStack int) (*backend.ExternalInterface, error) {
 	var iface *net.Interface
 	var ifaceAddr net.IP
+	var ifaceV6Addr net.IP
 	var err error
 
+        ifregex, err := regexp.Compile(ifregexS)
+	if err != nil {
+		return nil, fmt.Errorf("could not compile the IP address regex '%s': %w", ifregexS, err)
+	}
+
+	// Check ip family stack
+	if ipStack == noneStack {
+		return nil, fmt.Errorf("none matched ip stack")
+	}
+
 	if len(ifname) > 0 {
 		if ifaceAddr = net.ParseIP(ifname); ifaceAddr != nil {
 			log.Infof("Searching for interface using %s", ifaceAddr)
-			iface, err = ip.GetInterfaceByIP(ifaceAddr)
-			if err != nil {
-				return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
+			switch ipStack {
+			case ipv4Stack:
+				iface, err = ip.GetInterfaceByIP(ifaceAddr)
+				if err != nil {
+					return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
+				}
+			case ipv6Stack:
+				iface, err = ip.GetInterfaceByIP6(ifaceAddr)
+				if err != nil {
+					return nil, fmt.Errorf("error looking up v6 interface %s: %s", ifname, err)
+				}
+			case dualStack:
+				iface, err = ip.GetInterfaceByIP(ifaceAddr)
+				if err != nil {
+					return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
+				}
+				v6Iface, err := ip.GetInterfaceByIP6(ifaceAddr)
+				if err != nil {
+					return nil, fmt.Errorf("error looking up v6 interface %s: %s", ifname, err)
+				}
+				if iface.Name != v6Iface.Name {
+					return nil, fmt.Errorf("v6 interface %s must be the same with v4 interface %s", v6Iface.Name, iface.Name)
+				}
 			}
 		} else {
 			iface, err = net.InterfaceByName(ifname)
@@ -478,30 +574,79 @@ func LookupExtIface(ifname string, ifregex string) (*backend.ExternalInterface,
 
 		// Check IP
 		for _, ifaceToMatch := range ifaces {
-			ifaceIP, err := ip.GetInterfaceIP4Addr(&ifaceToMatch)
-			if err != nil {
-				// Skip if there is no IPv4 address
-				continue
-			}
+			switch ipStack {
+			case ipv4Stack:
+				ifaceIP, err := ip.GetInterfaceIP4Addr(&ifaceToMatch)
+				if err != nil {
+					// Skip if there is no IPv4 address
+					continue
+				}
 
-			matched, err := regexp.MatchString(ifregex, ifaceIP.String())
-			if err != nil {
-				return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String())
-			}
+				matched, err := ifregex.MatchString(ifaceIP.String())
+				if err != nil {
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
+				}
 
-			if matched {
-				ifaceAddr = ifaceIP
-				iface = &ifaceToMatch
-				break
+				if matched {
+					ifaceAddr = ifaceIP
+					iface = &ifaceToMatch
+					break
+				}
+			case ipv6Stack:
+				ifaceIP, err := ip.GetInterfaceIP6Addr(&ifaceToMatch)
+				if err != nil {
+					// Skip if there is no IPv6 address
+					continue
+				}
+
+				matched, err := ifregex.MatchString(ifaceIP.String())
+				if err != nil {
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
+				}
+
+				if matched {
+					ifaceV6Addr = ifaceIP
+					iface = &ifaceToMatch
+					break
+				}
+			case dualStack:
+				ifaceIP, err := ip.GetInterfaceIP4Addr(&ifaceToMatch)
+				if err != nil {
+					// Skip if there is no IPv4 address
+					continue
+				}
+
+				matched, err := ifregex.MatchString(ifaceIP.String())
+				if err != nil {
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
+				}
+
+				ifaceV6IP, err := ip.GetInterfaceIP6Addr(&ifaceToMatch)
+				if err != nil {
+					// Skip if there is no IPv6 address
+					continue
+				}
+
+				v6Matched, err := ifregex.MatchString(ifaceV6IP.String())
+				if err != nil {
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
+				}
+
+				if matched && v6Matched {
+					ifaceAddr = ifaceIP
+					ifaceV6Addr = ifaceV6IP
+					iface = &ifaceToMatch
+					break
+				}
 			}
 		}
 
 		// Check Name
-		if iface == nil && ifaceAddr == nil {
+		if iface == nil && (ifaceAddr == nil || ifaceV6Addr == nil) {
 			for _, ifaceToMatch := range ifaces {
-				matched, err := regexp.MatchString(ifregex, ifaceToMatch.Name)
+				matched, err := ifregex.MatchString(ifaceToMatch.Name)
 				if err != nil {
-					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceToMatch.Name)
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceToMatch.Name)
 				}
 
 				if matched {
@@ -515,33 +660,78 @@ func LookupExtIface(ifname string, ifregex string) (*backend.ExternalInterface,
 		if iface == nil {
 			var availableFaces []string
 			for _, f := range ifaces {
-				ip, _ := ip.GetInterfaceIP4Addr(&f) // We can safely ignore errors. We just won't log any ip
-				availableFaces = append(availableFaces, fmt.Sprintf("%s:%s", f.Name, ip))
+				var ipaddr net.IP
+				switch ipStack {
+				case ipv4Stack, dualStack:
+					ipaddr, _ = ip.GetInterfaceIP4Addr(&f) // We can safely ignore errors. We just won't log any ip
+				case ipv6Stack:
+					ipaddr, _ = ip.GetInterfaceIP6Addr(&f) // We can safely ignore errors. We just won't log any ip
+				}
+				availableFaces = append(availableFaces, fmt.Sprintf("%s:%s", f.Name, ipaddr))
 			}
 
-			return nil, fmt.Errorf("Could not match pattern %s to any of the available network interfaces (%s)", ifregex, strings.Join(availableFaces, ", "))
+			return nil, fmt.Errorf("Could not match pattern %s to any of the available network interfaces (%s)", ifregexS, strings.Join(availableFaces, ", "))
 		}
 	} else {
 		log.Info("Determining IP address of default interface")
-		if iface, err = ip.GetDefaultGatewayInterface(); err != nil {
-			return nil, fmt.Errorf("failed to get default interface: %s", err)
+		switch ipStack {
+		case ipv4Stack:
+			if iface, err = ip.GetDefaultGatewayInterface(); err != nil {
+				return nil, fmt.Errorf("failed to get default interface: %w", err)
+			}
+		case ipv6Stack:
+			if iface, err = ip.GetDefaultV6GatewayInterface(); err != nil {
+				return nil, fmt.Errorf("failed to get default v6 interface: %w", err)
+			}
+		case dualStack:
+			if iface, err = ip.GetDefaultGatewayInterface(); err != nil {
+				return nil, fmt.Errorf("failed to get default interface: %w", err)
+			}
+			v6Iface, err := ip.GetDefaultV6GatewayInterface()
+			if err != nil {
+				return nil, fmt.Errorf("failed to get default v6 interface: %w", err)
+			}
+			if iface.Name != v6Iface.Name {
+				return nil, fmt.Errorf("v6 default route interface %s "+
+					"must be the same with v4 default route interface %s", v6Iface.Name, iface.Name)
+			}
 		}
 	}
 
-	if ifaceAddr == nil {
+	if ipStack == ipv4Stack && ifaceAddr == nil {
+		ifaceAddr, err = ip.GetInterfaceIP4Addr(iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
+		}
+	} else if ipStack == ipv6Stack && ifaceV6Addr == nil {
+		ifaceV6Addr, err = ip.GetInterfaceIP6Addr(iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find IPv6 address for interface %s", iface.Name)
+		}
+	} else if ipStack == dualStack && ifaceAddr == nil && ifaceV6Addr == nil {
 		ifaceAddr, err = ip.GetInterfaceIP4Addr(iface)
 		if err != nil {
 			return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
 		}
+		ifaceV6Addr, err = ip.GetInterfaceIP6Addr(iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find IPv6 address for interface %s", iface.Name)
+		}
 	}
 
-	log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)
+	if ifaceAddr != nil {
+		log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)
+	}
+	if ifaceV6Addr != nil {
+		log.Infof("Using interface with name %s and v6 address %s", iface.Name, ifaceV6Addr)
+	}
 
 	if iface.MTU == 0 {
 		return nil, fmt.Errorf("failed to determine MTU for %s interface", ifaceAddr)
 	}
 
 	var extAddr net.IP
+	var extV6Addr net.IP
 
 	if len(opts.publicIP) > 0 {
 		extAddr = net.ParseIP(opts.publicIP)
@@ -556,30 +746,53 @@ func LookupExtIface(ifname string, ifregex string) (*backend.ExternalInterface,
 		extAddr = ifaceAddr
 	}
 
+	if len(opts.publicIPv6) > 0 {
+		extV6Addr = net.ParseIP(opts.publicIPv6)
+		if extV6Addr == nil {
+			return nil, fmt.Errorf("invalid public IPv6 address: %s", opts.publicIPv6)
+		}
+		log.Infof("Using %s as external address", extV6Addr)
+	}
+
+	if extV6Addr == nil {
+		log.Infof("Defaulting external v6 address to interface address (%s)", ifaceV6Addr)
+		extV6Addr = ifaceV6Addr
+	}
+
 	return &backend.ExternalInterface{
-		Iface:     iface,
-		IfaceAddr: ifaceAddr,
-		ExtAddr:   extAddr,
+		Iface:       iface,
+		IfaceAddr:   ifaceAddr,
+		IfaceV6Addr: ifaceV6Addr,
+		ExtAddr:     extAddr,
+		ExtV6Addr:   extV6Addr,
 	}, nil
 }
 
-func WriteSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network) error {
+func WriteSubnetFile(path string, config *subnet.Config, ipMasq bool, bn backend.Network) error {
 	dir, name := filepath.Split(path)
 	os.MkdirAll(dir, 0755)
-
 	tempFile := filepath.Join(dir, "."+name)
 	f, err := os.Create(tempFile)
 	if err != nil {
 		return err
 	}
+	if config.EnableIPv4 {
+		nw := config.Network
+		// Write out the first usable IP by incrementing
+		// sn.IP by one
+		sn := bn.Lease().Subnet
+		sn.IP += 1
+		fmt.Fprintf(f, "FLANNEL_NETWORK=%s\n", nw)
+		fmt.Fprintf(f, "FLANNEL_SUBNET=%s\n", sn)
+	}
+	if config.EnableIPv6 {
+		ip6Nw := config.IPv6Network
+		ip6Sn := bn.Lease().IPv6Subnet
+		ip6Sn.IP = (*ip.IP6)(big.NewInt(0).Add((*big.Int)(ip6Sn.IP), big.NewInt(1)))
+		fmt.Fprintf(f, "FLANNEL_IPV6_NETWORK=%s\n", ip6Nw)
+		fmt.Fprintf(f, "FLANNEL_IPV6_SUBNET=%s\n", ip6Sn)
+	}
 
-	// Write out the first usable IP by incrementing
-	// sn.IP by one
-	sn := bn.Lease().Subnet
-	sn.IP += 1
-
-	fmt.Fprintf(f, "FLANNEL_NETWORK=%s\n", nw)
-	fmt.Fprintf(f, "FLANNEL_SUBNET=%s\n", sn)
 	fmt.Fprintf(f, "FLANNEL_MTU=%d\n", bn.MTU())
 	_, err = fmt.Fprintf(f, "FLANNEL_IPMASQ=%v\n", ipMasq)
 	f.Close()
@@ -623,3 +836,19 @@ func ReadCIDRFromSubnetFile(path string, CIDRKey string) ip.IP4Net {
 	}
 	return prevCIDR
 }
+
+func ReadIP6CIDRFromSubnetFile(path string, CIDRKey string) ip.IP6Net {
+	var prevCIDR ip.IP6Net
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		prevSubnetVals, err := godotenv.Read(path)
+		if err != nil {
+			log.Errorf("Couldn't fetch previous %s from subnet file at %s: %s", CIDRKey, path, err)
+		} else if prevCIDRString, ok := prevSubnetVals[CIDRKey]; ok {
+			err = prevCIDR.UnmarshalJSON([]byte(prevCIDRString))
+			if err != nil {
+				log.Errorf("Couldn't parse previous %s from subnet file at %s: %s", CIDRKey, path, err)
+			}
+		}
+	}
+	return prevCIDR
+}

network/iptables.go

@@ -77,6 +77,40 @@ func MasqRules(ipn ip.IP4Net, lease *subnet.Lease) []IPTablesRule {
 	}
 }
 
+func MasqIP6Rules(ipn ip.IP6Net, lease *subnet.Lease) []IPTablesRule {
+	n := ipn.String()
+	sn := lease.IPv6Subnet.String()
+	supports_random_fully := false
+	ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	if err == nil {
+		supports_random_fully = ipt.HasRandomFully()
+	}
+
+	if supports_random_fully {
+		return []IPTablesRule{
+			// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0)
+			{"nat", "POSTROUTING", []string{"-s", n, "-d", n, "-j", "RETURN"}},
+			// NAT if it's not multicast traffic
+			{"nat", "POSTROUTING", []string{"-s", n, "!", "-d", "ff00::/8", "-j", "MASQUERADE", "--random-fully"}},
+			// Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", sn, "-j", "RETURN"}},
+			// Masquerade anything headed towards flannel from the host
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", n, "-j", "MASQUERADE", "--random-fully"}},
+		}
+	} else {
+		return []IPTablesRule{
+			// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0)
+			{"nat", "POSTROUTING", []string{"-s", n, "-d", n, "-j", "RETURN"}},
+			// NAT if it's not multicast traffic
+			{"nat", "POSTROUTING", []string{"-s", n, "!", "-d", "ff00::/8", "-j", "MASQUERADE"}},
+			// Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", sn, "-j", "RETURN"}},
+			// Masquerade anything headed towards flannel from the host
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", n, "-j", "MASQUERADE"}},
+		}
+	}
+}
+
 func ForwardRules(flannelNetwork string) []IPTablesRule {
 	return []IPTablesRule{
 		// These rules allow traffic to be forwarded if it is to or from the flannel network range.
@@ -122,6 +156,28 @@ func SetupAndEnsureIPTables(rules []IPTablesRule, resyncPeriod int) {
 	}
 }
 
+func SetupAndEnsureIP6Tables(rules []IPTablesRule, resyncPeriod int) {
+	ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	if err != nil {
+		// if we can't find iptables, give up and return
+		log.Errorf("Failed to setup IP6Tables. iptables binary was not found: %v", err)
+		return
+	}
+
+	defer func() {
+		teardownIPTables(ipt, rules)
+	}()
+
+	for {
+		// Ensure that all the iptables rules exist every 5 seconds
+		if err := ensureIPTables(ipt, rules); err != nil {
+			log.Errorf("Failed to ensure iptables rules: %v", err)
+		}
+
+		time.Sleep(time.Duration(resyncPeriod) * time.Second)
+	}
+}
+
 // DeleteIPTables delete specified iptables rules
 func DeleteIPTables(rules []IPTablesRule) error {
 	ipt, err := iptables.New()
@@ -134,6 +190,18 @@ func DeleteIPTables(rules []IPTablesRule) error {
 	return nil
 }
 
+// DeleteIP6Tables delete specified iptables rules
+func DeleteIP6Tables(rules []IPTablesRule) error {
+	ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	if err != nil {
+		// if we can't find iptables, give up and return
+		log.Errorf("Failed to setup IP6Tables. iptables binary was not found: %v", err)
+		return err
+	}
+	teardownIPTables(ipt, rules)
+	return nil
+}
+
 func ensureIPTables(ipt IPTables, rules []IPTablesRule) error {
 	exists, err := ipTablesRulesExist(ipt, rules)
 	if err != nil {

network/iptables_test.go

@@ -27,9 +27,11 @@ import (
 )
 
 func lease() *subnet.Lease {
+	_, ipv6Net, _ := net.ParseCIDR("fc00::/48")
 	_, net, _ := net.ParseCIDR("192.168.0.0/16")
 	return &subnet.Lease{
-		Subnet: ip.FromIPNet(net),
+		Subnet:     ip.FromIPNet(net),
+		IPv6Subnet: ip.FromIP6Net(ipv6Net),
 	}
 }
 
@@ -113,6 +115,18 @@ func TestDeleteRules(t *testing.T) {
 	}
 }
 
+func TestDeleteIP6Rules(t *testing.T) {
+	ipt := &MockIPTables{t: t}
+	setupIPTables(ipt, MasqIP6Rules(ip.IP6Net{}, lease()))
+	if len(ipt.rules) != 4 {
+		t.Errorf("Should be 4 masqRules, there are actually %d: %#v", len(ipt.rules), ipt.rules)
+	}
+	teardownIPTables(ipt, MasqIP6Rules(ip.IP6Net{}, lease()))
+	if len(ipt.rules) != 0 {
+		t.Errorf("Should be 0 masqRules, there are actually %d: %#v", len(ipt.rules), ipt.rules)
+	}
+}
+
 func TestEnsureRulesError(t *testing.T) {
 	// If an error prevents a rule from being deleted, ensureIPTables should leave the rules as is
 	// rather than potentially re-appending rules in an incorrect order
@@ -135,6 +149,28 @@ func TestEnsureRulesError(t *testing.T) {
 	}
 }
 
+func TestEnsureIP6RulesError(t *testing.T) {
+	// If an error prevents a rule from being deleted, ensureIPTables should leave the rules as is
+	// rather than potentially re-appending rules in an incorrect order
+	ipt_correct := &MockIPTables{t: t}
+	setupIPTables(ipt_correct, MasqIP6Rules(ip.IP6Net{}, lease()))
+	// setup a mock instance where we delete some masqRules and run `ensureIPTables`
+	ipt_recreate := &MockIPTables{t: t}
+	setupIPTables(ipt_recreate, MasqIP6Rules(ip.IP6Net{}, lease()))
+	ipt_recreate.rules = ipt_recreate.rules[0:2]
+
+	rule := ipt_recreate.rules[1]
+	ipt_recreate.failDelete(rule.table, rule.chain, rule.rulespec, false)
+	err := ensureIPTables(ipt_recreate, MasqIP6Rules(ip.IP6Net{}, lease()))
+	if err == nil {
+		t.Errorf("ensureIPTables should have failed but did not.")
+	}
+
+	if len(ipt_recreate.rules) == len(ipt_correct.rules) {
+		t.Errorf("ensureIPTables should not have completed.")
+	}
+}
+
 func TestEnsureRules(t *testing.T) {
 	// If any masqRules are missing, they should be all deleted and recreated in the correct order
 	ipt_correct := &MockIPTables{t: t}
@@ -154,3 +190,23 @@ func TestEnsureRules(t *testing.T) {
 		t.Errorf("iptables masqRules after ensureIPTables are incorrect. Expected: %#v, Actual: %#v", ipt_recreate.rules, ipt_correct.rules)
 	}
 }
+
+func TestEnsureIP6Rules(t *testing.T) {
+	// If any masqRules are missing, they should be all deleted and recreated in the correct order
+	ipt_correct := &MockIPTables{t: t}
+	setupIPTables(ipt_correct, MasqIP6Rules(ip.IP6Net{}, lease()))
+	// setup a mock instance where we delete some masqRules and run `ensureIPTables`
+	ipt_recreate := &MockIPTables{t: t}
+	setupIPTables(ipt_recreate, MasqIP6Rules(ip.IP6Net{}, lease()))
+	ipt_recreate.rules = ipt_recreate.rules[0:2]
+	// set up a normal error that iptables returns when deleting a rule that is already gone
+	deletedRule := ipt_correct.rules[3]
+	ipt_recreate.failDelete(deletedRule.table, deletedRule.chain, deletedRule.rulespec, true)
+	err := ensureIPTables(ipt_recreate, MasqIP6Rules(ip.IP6Net{}, lease()))
+	if err != nil {
+		t.Errorf("ensureIPTables should have completed without errors")
+	}
+	if !reflect.DeepEqual(ipt_recreate.rules, ipt_correct.rules) {
+		t.Errorf("iptables masqIP6Rules after ensureIPTables are incorrect. Expected: %#v, Actual: %#v", ipt_recreate.rules, ipt_correct.rules)
+	}
+}

pkg/ip/iface.go

@@ -36,6 +36,16 @@ func getIfaceAddrs(iface *net.Interface) ([]netlink.Addr, error) {
 	return netlink.AddrList(link, syscall.AF_INET)
 }
 
+func getIfaceV6Addrs(iface *net.Interface) ([]netlink.Addr, error) {
+	link := &netlink.Device{
+		netlink.LinkAttrs{
+			Index: iface.Index,
+		},
+	}
+
+	return netlink.AddrList(link, syscall.AF_INET6)
+}
+
 func GetInterfaceIP4Addr(iface *net.Interface) (net.IP, error) {
 	addrs, err := getIfaceAddrs(iface)
 	if err != nil {
@@ -67,6 +77,37 @@ func GetInterfaceIP4Addr(iface *net.Interface) (net.IP, error) {
 	return nil, errors.New("No IPv4 address found for given interface")
 }
 
+func GetInterfaceIP6Addr(iface *net.Interface) (net.IP, error) {
+	addrs, err := getIfaceV6Addrs(iface)
+	if err != nil {
+		return nil, err
+	}
+
+	// prefer non link-local addr
+	var ll net.IP
+
+	for _, addr := range addrs {
+		if addr.IP.To16() == nil {
+			continue
+		}
+
+		if addr.IP.IsGlobalUnicast() {
+			return addr.IP, nil
+		}
+
+		if addr.IP.IsLinkLocalUnicast() {
+			ll = addr.IP
+		}
+	}
+
+	if ll != nil {
+		// didn't find global but found link-local. it'll do.
+		return ll, nil
+	}
+
+	return nil, errors.New("No IPv6 address found for given interface")
+}
+
 func GetInterfaceIP4AddrMatch(iface *net.Interface, matchAddr net.IP) error {
 	addrs, err := getIfaceAddrs(iface)
 	if err != nil {
@@ -86,6 +127,25 @@ func GetInterfaceIP4AddrMatch(iface *net.Interface, matchAddr net.IP) error {
 	return errors.New("No IPv4 address found for given interface")
 }
 
+func GetInterfaceIP6AddrMatch(iface *net.Interface, matchAddr net.IP) error {
+	addrs, err := getIfaceV6Addrs(iface)
+	if err != nil {
+		return err
+	}
+
+	for _, addr := range addrs {
+		// Attempt to parse the address in CIDR notation
+		// and assert it is IPv6
+		if addr.IP.To16() != nil {
+			if addr.IP.To16().Equal(matchAddr) {
+				return nil
+			}
+		}
+	}
+
+	return errors.New("No IPv6 address found for given interface")
+}
+
 func GetDefaultGatewayInterface() (*net.Interface, error) {
 	routes, err := netlink.RouteList(nil, syscall.AF_INET)
 	if err != nil {
@@ -104,6 +164,24 @@ func GetDefaultGatewayInterface() (*net.Interface, error) {
 	return nil, errors.New("Unable to find default route")
 }
 
+func GetDefaultV6GatewayInterface() (*net.Interface, error) {
+	routes, err := netlink.RouteList(nil, syscall.AF_INET6)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, route := range routes {
+		if route.Dst == nil || route.Dst.String() == "::/0" {
+			if route.LinkIndex <= 0 {
+				return nil, errors.New("Found default v6 route but could not determine interface")
+			}
+			return net.InterfaceByIndex(route.LinkIndex)
+		}
+	}
+
+	return nil, errors.New("Unable to find default v6 route")
+}
+
 func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
 	ifaces, err := net.Interfaces()
 	if err != nil {
@@ -120,6 +198,22 @@ func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
 	return nil, errors.New("No interface with given IP found")
 }
 
+func GetInterfaceByIP6(ip net.IP) (*net.Interface, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, iface := range ifaces {
+		err := GetInterfaceIP6AddrMatch(&iface, ip)
+		if err == nil {
+			return &iface, nil
+		}
+	}
+
+	return nil, errors.New("No interface with given IPv6 found")
+}
+
 func DirectRouting(ip net.IP) (bool, error) {
 	routes, err := netlink.RouteGet(ip)
 	if err != nil {
@@ -164,3 +258,41 @@ func EnsureV4AddressOnLink(ipa IP4Net, ipn IP4Net, link netlink.Link) error {
 
 	return nil
 }
+
+// EnsureV6AddressOnLink ensures that there is only one v6 Addr on `link` and it equals `ipn`.
+// If there exist multiple addresses on link, it returns an error message to tell callers to remove additional address.
+func EnsureV6AddressOnLink(ipa IP6Net, ipn IP6Net, link netlink.Link) error {
+	addr := netlink.Addr{IPNet: ipa.ToIPNet()}
+	existingAddrs, err := netlink.AddrList(link, netlink.FAMILY_V6)
+	if err != nil {
+		return err
+	}
+
+	onlyLinkLocal := true
+	for _, existingAddr := range existingAddrs {
+		if !existingAddr.IP.IsLinkLocalUnicast() {
+			if !existingAddr.Equal(addr) {
+				if err := netlink.AddrDel(link, &existingAddr); err != nil {
+					return fmt.Errorf("failed to remove v6 IP address %s from %s: %w", ipn.String(), link.Attrs().Name, err)
+				}
+				existingAddrs = []netlink.Addr{}
+				onlyLinkLocal = false
+			} else {
+				return nil
+			}
+		}
+	}
+
+	if onlyLinkLocal {
+		existingAddrs = []netlink.Addr{}
+	}
+
+	// Actually add the desired address to the interface if needed.
+	if len(existingAddrs) == 0 {
+		if err := netlink.AddrAdd(link, &addr); err != nil {
+			return fmt.Errorf("failed to add v6 IP address %s to %s: %w", ipn.String(), link.Attrs().Name, err)
+		}
+	}
+
+	return nil
+}

pkg/ip/iface_test.go

@@ -62,3 +62,46 @@ func TestEnsureV4AddressOnLink(t *testing.T) {
 		t.Fatalf("two addresses expected, addrs: %v", addrs)
 	}
 }
+
+func TestEnsureV6AddressOnLink(t *testing.T) {
+	teardown := ns.SetUpNetlinkTest(t)
+	defer teardown()
+	lo, err := netlink.LinkByName("lo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := netlink.LinkSetUp(lo); err != nil {
+		t.Fatal(err)
+	}
+	// check changing address
+	ipn := IP6Net{IP: FromIP6(net.ParseIP("::2")), PrefixLen: 64}
+	if err := EnsureV6AddressOnLink(ipn, ipn, lo); err != nil {
+		t.Fatal(err)
+	}
+	addrs, err := netlink.AddrList(lo, netlink.FAMILY_V6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(addrs) != 1 || addrs[0].String() != "::2/64" {
+		t.Fatalf("v6 addrs %v is not expected", addrs)
+	}
+
+	// check changing address if there exist multiple addresses
+	if err := netlink.AddrAdd(lo, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("2001::4"), Mask: net.CIDRMask(64, 128)}}); err != nil {
+		t.Fatal(err)
+	}
+	addrs, err = netlink.AddrList(lo, netlink.FAMILY_V6)
+	if len(addrs) != 2 {
+		t.Fatalf("two addresses expected, addrs: %v", addrs)
+	}
+	if err := EnsureV6AddressOnLink(ipn, ipn, lo); err != nil {
+		t.Fatal(err)
+	}
+	addrs, err = netlink.AddrList(lo, netlink.FAMILY_V6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(addrs) != 1 {
+		t.Fatalf("only one address expected, addrs: %v", addrs)
+	}
+}

pkg/ip/ip6net.go

@@ -0,0 +1,205 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+)
+
+type IP6 big.Int
+
+func FromIP16Bytes(ip []byte) *IP6 {
+	return (*IP6)(big.NewInt(0).SetBytes(ip))
+}
+
+func FromIP6(ip net.IP) *IP6 {
+	ipv6 := ip.To16()
+
+	if ipv6 == nil {
+		panic("Address is not an IPv6 address")
+	}
+
+	return FromIP16Bytes(ipv6)
+}
+
+func ParseIP6(s string) (*IP6, error) {
+	ip := net.ParseIP(s)
+	if ip == nil {
+		return (*IP6)(big.NewInt(0)), errors.New("Invalid IP address format")
+	}
+	return FromIP6(ip), nil
+}
+
+func Mask(prefixLen int) *big.Int {
+	mask := net.CIDRMask(prefixLen, 128)
+	return big.NewInt(0).SetBytes(mask)
+}
+
+func IsEmpty(subnet *IP6) bool {
+	if subnet == nil || (*big.Int)(subnet).Cmp(big.NewInt(0)) == 0 {
+		return true
+	}
+	return false
+}
+
+func GetIPv6SubnetMin(networkIP *IP6, subnetSize *big.Int) *IP6 {
+	return (*IP6)(big.NewInt(0).Add((*big.Int)(networkIP), subnetSize))
+}
+
+func GetIPv6SubnetMax(networkIP *IP6, subnetSize *big.Int) *IP6 {
+	return (*IP6)(big.NewInt(0).Sub((*big.Int)(networkIP), subnetSize))
+}
+
+func CheckIPv6Subnet(subnetIP *IP6, mask *big.Int) bool {
+	if (*big.Int)(subnetIP).Cmp(big.NewInt(0).And((*big.Int)(subnetIP), mask)) != 0 {
+		return false
+	}
+	return true
+}
+
+func MustParseIP6(s string) *IP6 {
+	ip, err := ParseIP6(s)
+	if err != nil {
+		panic(err)
+	}
+	return ip
+}
+
+func (ip6 *IP6) ToIP() net.IP {
+	ip := net.IP((*big.Int)(ip6).Bytes())
+	if ip.To4() != nil {
+		return ip
+	}
+	a := (*big.Int)(ip6).FillBytes(make([]byte, 16))
+	return a
+}
+
+func (ip6 IP6) String() string {
+	return ip6.ToIP().String()
+}
+
+// MarshalJSON: json.Marshaler impl
+func (ip6 IP6) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, ip6)), nil
+}
+
+// UnmarshalJSON: json.Unmarshaler impl
+func (ip6 *IP6) UnmarshalJSON(j []byte) error {
+	j = bytes.Trim(j, "\"")
+	if val, err := ParseIP6(string(j)); err != nil {
+		return err
+	} else {
+		*ip6 = *val
+		return nil
+	}
+}
+
+// similar to net.IPNet but has uint based representation
+type IP6Net struct {
+	IP        *IP6
+	PrefixLen uint
+}
+
+func (n IP6Net) String() string {
+	if n.IP == nil {
+		n.IP = (*IP6)(big.NewInt(0))
+	}
+	return fmt.Sprintf("%s/%d", n.IP.String(), n.PrefixLen)
+}
+
+func (n IP6Net) StringSep(hexSep, prefixSep string) string {
+	return fmt.Sprintf("%s%s%d", n.IP.String(), prefixSep, n.PrefixLen)
+}
+
+func (n IP6Net) Network() IP6Net {
+	mask := net.CIDRMask(int(n.PrefixLen), 128)
+	return IP6Net{
+		FromIP6(n.IP.ToIP().Mask(mask)),
+		n.PrefixLen,
+	}
+}
+
+func (n IP6Net) Next() IP6Net {
+	return IP6Net{
+		(*IP6)(big.NewInt(0).Add((*big.Int)(n.IP), big.NewInt(0).Lsh(big.NewInt(1), 128-n.PrefixLen))),
+		n.PrefixLen,
+	}
+}
+
+func FromIP6Net(n *net.IPNet) IP6Net {
+	prefixLen, _ := n.Mask.Size()
+	return IP6Net{
+		FromIP6(n.IP),
+		uint(prefixLen),
+	}
+}
+
+func (n IP6Net) ToIPNet() *net.IPNet {
+	return &net.IPNet{
+		IP:   n.IP.ToIP(),
+		Mask: net.CIDRMask(int(n.PrefixLen), 128),
+	}
+}
+
+func (n IP6Net) Overlaps(other IP6Net) bool {
+	var mask *big.Int
+	if n.PrefixLen < other.PrefixLen {
+		mask = n.Mask()
+	} else {
+		mask = other.Mask()
+	}
+	return (IP6)(*big.NewInt(0).And((*big.Int)(n.IP), mask)).String() ==
+		(IP6)(*big.NewInt(0).And((*big.Int)(other.IP), mask)).String()
+}
+
+func (n IP6Net) Equal(other IP6Net) bool {
+	return ((*big.Int)(n.IP).Cmp((*big.Int)(other.IP)) == 0) &&
+		n.PrefixLen == other.PrefixLen
+}
+
+func (n IP6Net) Mask() *big.Int {
+	mask := net.CIDRMask(int(n.PrefixLen), 128)
+	return big.NewInt(0).SetBytes(mask)
+}
+
+func (n IP6Net) Contains(ip *IP6) bool {
+	network := big.NewInt(0).And((*big.Int)(n.IP), n.Mask())
+	subnet := big.NewInt(0).And((*big.Int)(ip), n.Mask())
+	return (IP6)(*network).String() == (IP6)(*subnet).String()
+}
+
+func (n IP6Net) Empty() bool {
+	return n.IP == (*IP6)(big.NewInt(0)) && n.PrefixLen == uint(0)
+}
+
+// MarshalJSON: json.Marshaler impl
+func (n IP6Net) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, n)), nil
+}
+
+// UnmarshalJSON: json.Unmarshaler impl
+func (n *IP6Net) UnmarshalJSON(j []byte) error {
+	j = bytes.Trim(j, "\"")
+	if _, val, err := net.ParseCIDR(string(j)); err != nil {
+		return err
+	} else {
+		*n = FromIP6Net(val)
+		return nil
+	}
+}

pkg/ip/ip6net_test.go

@@ -0,0 +1,113 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"encoding/json"
+	"net"
+	"testing"
+)
+
+func mkIP6Net(s string, plen uint) IP6Net {
+	ip, err := ParseIP6(s)
+	if err != nil {
+		panic(err)
+	}
+	return IP6Net{ip, plen}
+}
+
+func mkIP6(s string) *IP6 {
+	ip, err := ParseIP6(s)
+	if err != nil {
+		panic(err)
+	}
+	return ip
+}
+
+func TestIP6(t *testing.T) {
+	nip := net.ParseIP("fc00::1")
+	ip := FromIP6(nip)
+	ipStr := ip.String()
+	if ipStr != "fc00::1" {
+		t.Error("FromIP6 failed")
+	}
+
+	ip, err := ParseIP6("fc00::1")
+	if err != nil {
+		t.Error("ParseIP6 failed with: ", err)
+	} else {
+		ipStr := ip.String()
+		if ipStr != "fc00::1" {
+			t.Error("ParseIP6 failed")
+		}
+	}
+
+	if ip.ToIP().String() != "fc00::1" {
+		t.Error("ToIP failed")
+	}
+
+	j, err := json.Marshal(ip)
+	if err != nil {
+		t.Error("Marshal of IP6 failed: ", err)
+	} else if string(j) != `"fc00::1"` {
+		t.Error("Marshal of IP6 failed with unexpected value: ", j)
+	}
+}
+
+func TestIP6Net(t *testing.T) {
+	n1 := mkIP6Net("fc00:1::", 64)
+
+	if n1.ToIPNet().String() != "fc00:1::/64" {
+		t.Error("ToIPNet failed")
+	}
+
+	if !n1.Overlaps(n1) {
+		t.Errorf("%s does not overlap %s", n1, n1)
+	}
+
+	n2 := mkIP6Net("fc00::", 16)
+	if !n1.Overlaps(n2) {
+		t.Errorf("%s does not overlap %s", n1, n2)
+	}
+
+	n2 = mkIP6Net("fc00:2::", 64)
+	if n1.Overlaps(n2) {
+		t.Errorf("%s overlaps %s", n1, n2)
+	}
+
+	n2 = mkIP6Net("fb00:2::", 48)
+	if n1.Overlaps(n2) {
+		t.Errorf("%s overlaps %s", n1, n2)
+	}
+
+	if !n1.Contains(mkIP6("fc00:1::")) {
+		t.Error("Contains failed")
+	}
+
+	if !n1.Contains(mkIP6("fc00:1::1")) {
+		t.Error("Contains failed")
+	}
+
+	if n1.Contains(mkIP6("fc00:2::")) {
+		t.Error("Contains failed")
+	}
+
+	j, err := json.Marshal(n1)
+	if err != nil {
+		t.Error("Marshal of IP6Net failed: ", err)
+	} else if string(j) != `"fc00:1::/64"` {
+		t.Error("Marshal of IP6Net failed with unexpected value: ", j)
+	}
+}

subnet/config.go

@@ -18,17 +18,24 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/big"
 
 	"github.com/flannel-io/flannel/pkg/ip"
 )
 
 type Config struct {
-	Network     ip.IP4Net
-	SubnetMin   ip.IP4
-	SubnetMax   ip.IP4
-	SubnetLen   uint
-	BackendType string          `json:"-"`
-	Backend     json.RawMessage `json:",omitempty"`
+	EnableIPv4    bool
+	EnableIPv6    bool
+	Network       ip.IP4Net
+	IPv6Network   ip.IP6Net
+	SubnetMin     ip.IP4
+	SubnetMax     ip.IP4
+	IPv6SubnetMin *ip.IP6
+	IPv6SubnetMax *ip.IP6
+	SubnetLen     uint
+	IPv6SubnetLen uint
+	BackendType   string          `json:"-"`
+	Backend       json.RawMessage `json:",omitempty"`
 }
 
 func parseBackendType(be json.RawMessage) (string, error) {
@@ -47,65 +54,126 @@ func parseBackendType(be json.RawMessage) (string, error) {
 
 func ParseConfig(s string) (*Config, error) {
 	cfg := new(Config)
+	// Enable ipv4 by default
+	cfg.EnableIPv4 = true
 	err := json.Unmarshal([]byte(s), cfg)
 	if err != nil {
 		return nil, err
 	}
 
-	if cfg.SubnetLen > 0 {
-		// SubnetLen needs to allow for a tunnel and bridge device on each host.
-		if cfg.SubnetLen > 30 {
-			return nil, errors.New("SubnetLen must be less than /31")
+	if cfg.EnableIPv4 {
+		if cfg.SubnetLen > 0 {
+			// SubnetLen needs to allow for a tunnel and bridge device on each host.
+			if cfg.SubnetLen > 30 {
+				return nil, errors.New("SubnetLen must be less than /31")
+			}
+
+			// SubnetLen needs to fit _more_ than twice into the Network.
+			// the first subnet isn't used, so splitting into two one only provide one usable host.
+			if cfg.SubnetLen < cfg.Network.PrefixLen+2 {
+				return nil, errors.New("Network must be able to accommodate at least four subnets")
+			}
+		} else {
+			// If the network is smaller than a /28 then the network isn't big enough for flannel so return an error.
+			// Default to giving each host at least a /24 (as long as the network is big enough to support at least four hosts)
+			// Otherwise, if the network is too small to give each host a /24 just split the network into four.
+			if cfg.Network.PrefixLen > 28 {
+				// Each subnet needs at least four addresses (/30) and the network needs to accommodate at least four
+				// since the first subnet isn't used, so splitting into two would only provide one usable host.
+				// So the min useful PrefixLen is /28
+				return nil, errors.New("Network is too small. Minimum useful network prefix is /28")
+			} else if cfg.Network.PrefixLen <= 22 {
+				// Network is big enough to give each host a /24
+				cfg.SubnetLen = 24
+			} else {
+				// Use +2 to provide four hosts per subnet.
+				cfg.SubnetLen = cfg.Network.PrefixLen + 2
+			}
 		}
 
-		// SubnetLen needs to fit _more_ than twice into the Network.
-		// the first subnet isn't used, so splitting into two one only provide one usable host.
-		if cfg.SubnetLen < cfg.Network.PrefixLen+2 {
-			return nil, errors.New("Network must be able to accommodate at least four subnets")
+		subnetSize := ip.IP4(1 << (32 - cfg.SubnetLen))
+
+		if cfg.SubnetMin == ip.IP4(0) {
+			// skip over the first subnet otherwise it causes problems. e.g.
+			// if Network is 10.100.0.0/16, having an interface with 10.0.0.0
+			// conflicts with the broadcast address.
+			cfg.SubnetMin = cfg.Network.IP + subnetSize
+		} else if !cfg.Network.Contains(cfg.SubnetMin) {
+			return nil, errors.New("SubnetMin is not in the range of the Network")
 		}
-	} else {
-		// If the network is smaller than a /28 then the network isn't big enough for flannel so return an error.
-		// Default to giving each host at least a /24 (as long as the network is big enough to support at least four hosts)
-		// Otherwise, if the network is too small to give each host a /24 just split the network into four.
-		if cfg.Network.PrefixLen > 28 {
-			// Each subnet needs at least four addresses (/30) and the network needs to accommodate at least four
-			// since the first subnet isn't used, so splitting into two would only provide one usable host.
-			// So the min useful PrefixLen is /28
-			return nil, errors.New("Network is too small. Minimum useful network prefix is /28")
-		} else if cfg.Network.PrefixLen <= 22 {
-			// Network is big enough to give each host a /24
-			cfg.SubnetLen = 24
-		} else {
-			// Use +2 to provide four hosts per subnet.
-			cfg.SubnetLen = cfg.Network.PrefixLen + 2
+
+		if cfg.SubnetMax == ip.IP4(0) {
+			cfg.SubnetMax = cfg.Network.Next().IP - subnetSize
+		} else if !cfg.Network.Contains(cfg.SubnetMax) {
+			return nil, errors.New("SubnetMax is not in the range of the Network")
 		}
-	}
 
-	subnetSize := ip.IP4(1 << (32 - cfg.SubnetLen))
+		// The SubnetMin and SubnetMax need to be aligned to a SubnetLen boundary
+		mask := ip.IP4(0xFFFFFFFF << (32 - cfg.SubnetLen))
+		if cfg.SubnetMin != cfg.SubnetMin&mask {
+			return nil, fmt.Errorf("SubnetMin is not on a SubnetLen boundary: %v", cfg.SubnetMin)
+		}
 
-	if cfg.SubnetMin == ip.IP4(0) {
-		// skip over the first subnet otherwise it causes problems. e.g.
-		// if Network is 10.100.0.0/16, having an interface with 10.0.0.0
-		// conflicts with the broadcast address.
-		cfg.SubnetMin = cfg.Network.IP + subnetSize
-	} else if !cfg.Network.Contains(cfg.SubnetMin) {
-		return nil, errors.New("SubnetMin is not in the range of the Network")
+		if cfg.SubnetMax != cfg.SubnetMax&mask {
+			return nil, fmt.Errorf("SubnetMax is not on a SubnetLen boundary: %v", cfg.SubnetMax)
+		}
 	}
+	if cfg.EnableIPv6 {
+		if cfg.IPv6SubnetLen > 0 {
+			// SubnetLen needs to allow for a tunnel and bridge device on each host.
+			if cfg.IPv6SubnetLen > 126 {
+				return nil, errors.New("SubnetLen must be less than /127")
+			}
+
+			// SubnetLen needs to fit _more_ than twice into the Network.
+			// the first subnet isn't used, so splitting into two one only provide one usable host.
+			if cfg.IPv6SubnetLen < cfg.IPv6Network.PrefixLen+2 {
+				return nil, errors.New("Network must be able to accommodate at least four subnets")
+			}
+		} else {
+			// If the network is smaller than a /124 then the network isn't big enough for flannel so return an error.
+			// Default to giving each host at least a /64 (as long as the network is big enough to support at least four hosts)
+			// Otherwise, if the network is too small to give each host a /64 just split the network into four.
+			if cfg.IPv6Network.PrefixLen > 124 {
+				// Each subnet needs at least four addresses (/126) and the network needs to accommodate at least four
+				// since the first subnet isn't used, so splitting into two would only provide one usable host.
+				// So the min useful PrefixLen is /124
+				return nil, errors.New("IPv6Network is too small. Minimum useful network prefix is /124")
+			} else if cfg.IPv6Network.PrefixLen <= 62 {
+				// Network is big enough to give each host a /64
+				cfg.IPv6SubnetLen = 64
+			} else {
+				// Use +2 to provide four hosts per subnet.
+				cfg.IPv6SubnetLen = cfg.IPv6Network.PrefixLen + 2
+			}
+		}
 
-	if cfg.SubnetMax == ip.IP4(0) {
-		cfg.SubnetMax = cfg.Network.Next().IP - subnetSize
-	} else if !cfg.Network.Contains(cfg.SubnetMax) {
-		return nil, errors.New("SubnetMax is not in the range of the Network")
-	}
+		ipv6SubnetSize := big.NewInt(0).Lsh(big.NewInt(1), 128-cfg.IPv6SubnetLen)
 
-	// The SubnetMin and SubnetMax need to be aligned to a SubnetLen boundary
-	mask := ip.IP4(0xFFFFFFFF << (32 - cfg.SubnetLen))
-	if cfg.SubnetMin != cfg.SubnetMin&mask {
-		return nil, fmt.Errorf("SubnetMin is not on a SubnetLen boundary: %v", cfg.SubnetMin)
-	}
+		if ip.IsEmpty(cfg.IPv6SubnetMin) {
+			// skip over the first subnet otherwise it causes problems. e.g.
+			// if Network is fc00::/48, having an interface with fc00::
+			// conflicts with the broadcast address.
+			cfg.IPv6SubnetMin = ip.GetIPv6SubnetMin(cfg.IPv6Network.IP, ipv6SubnetSize)
+		} else if !cfg.IPv6Network.Contains(cfg.IPv6SubnetMin) {
+			return nil, errors.New("IPv6SubnetMin is not in the range of the IPv6Network")
+		}
+
+		if ip.IsEmpty(cfg.IPv6SubnetMax) {
+			cfg.IPv6SubnetMax = ip.GetIPv6SubnetMax(cfg.IPv6Network.Next().IP, ipv6SubnetSize)
+		} else if !cfg.IPv6Network.Contains(cfg.IPv6SubnetMax) {
+			return nil, errors.New("IPv6SubnetMax is not in the range of the IPv6Network")
+		}
+
+		// The SubnetMin and SubnetMax need to be aligned to a SubnetLen boundary
+		mask := ip.Mask(int(cfg.IPv6SubnetLen))
+		if !ip.CheckIPv6Subnet(cfg.IPv6SubnetMin, mask) {
+			return nil, fmt.Errorf("IPv6SubnetMin is not on a SubnetLen boundary: %v", cfg.IPv6SubnetMin)
+		}
 
-	if cfg.SubnetMax != cfg.SubnetMax&mask {
-		return nil, fmt.Errorf("SubnetMax is not on a SubnetLen boundary: %v", cfg.SubnetMax)
+		if !ip.CheckIPv6Subnet(cfg.IPv6SubnetMax, mask) {
+			return nil, fmt.Errorf("IPv6SubnetMax is not on a SubnetLen boundary: %v", cfg.IPv6SubnetMax)
+		}
 	}
 
 	bt, err := parseBackendType(cfg.Backend)

subnet/config_test.go

@@ -44,6 +44,32 @@ func TestConfigDefaults(t *testing.T) {
 	}
 }
 
+func TestIPv6ConfigDefaults(t *testing.T) {
+	s := `{ "enableIPv6": true, "ipv6Network": "fc00::/48" }`
+
+	cfg, err := ParseConfig(s)
+	if err != nil {
+		t.Fatalf("ParseConfig failed: %s", err)
+	}
+
+	expectedNet := "fc00::/48"
+	if cfg.IPv6Network.String() != expectedNet {
+		t.Errorf("IPv6Network mismatch: expected %s, got %s", expectedNet, cfg.IPv6Network)
+	}
+
+	if cfg.IPv6SubnetMin.String() != "fc00:0:0:1::" {
+		t.Errorf("IPv6SubnetMin mismatch, expected fc00:0:0:1::, got %s", cfg.IPv6SubnetMin)
+	}
+
+	if cfg.IPv6SubnetMax.String() != "fc00:0:0:ffff::" {
+		t.Errorf("IPv6SubnetMax mismatch, expected fc00:0:0:ffff::, got %s", cfg.IPv6SubnetMax)
+	}
+
+	if cfg.IPv6SubnetLen != 64 {
+		t.Errorf("IPv6SubnetLen mismatch: expected 64, got %d", cfg.IPv6SubnetLen)
+	}
+}
+
 func TestConfigOverrides(t *testing.T) {
 	s := `{ "Network": "10.3.0.0/16", "SubnetMin": "10.3.5.0", "SubnetMax": "10.3.8.0", "SubnetLen": 28 }`
 
@@ -69,3 +95,29 @@ func TestConfigOverrides(t *testing.T) {
 		t.Errorf("SubnetLen mismatch: expected 28, got %d", cfg.SubnetLen)
 	}
 }
+
+func TestIPv6ConfigOverrides(t *testing.T) {
+	s := `{ "EnableIPv6": true, "IPv6Network": "fc00::/48", "IPv6SubnetMin": "fc00:0:0:1::", "IPv6SubnetMax": "fc00:0:0:f::", "IPv6SubnetLen": 124 }`
+
+	cfg, err := ParseConfig(s)
+	if err != nil {
+		t.Fatalf("ParseConfig failed: %s", err)
+	}
+
+	expectedNet := "fc00::/48"
+	if cfg.IPv6Network.String() != expectedNet {
+		t.Errorf("IPv6Network mismatch: expected %s, got %s", expectedNet, cfg.IPv6Network)
+	}
+
+	if cfg.IPv6SubnetMin.String() != "fc00:0:0:1::" {
+		t.Errorf("IPv6SubnetMin mismatch: expected fc00:0:0:1::, got %s", cfg.IPv6SubnetMin)
+	}
+
+	if cfg.IPv6SubnetMax.String() != "fc00:0:0:f::" {
+		t.Errorf("IPv6SubnetMax mismatch: expected fc00:0:0:f::, got %s", cfg.IPv6SubnetMax)
+	}
+
+	if cfg.IPv6SubnetLen != 124 {
+		t.Errorf("IPv6SubnetLen mismatch: expected 124, got %d", cfg.IPv6SubnetLen)
+	}
+}

subnet/etcdv2/local_manager.go

@@ -103,6 +103,9 @@ func (m *LocalManager) AcquireLease(ctx context.Context, attrs *LeaseAttrs) (*Le
 		l, err := m.tryAcquireLease(ctx, config, attrs.PublicIP, attrs)
 		switch err {
 		case nil:
+			//TODO only vxlan backend and kube subnet manager support dual stack now.
+			l.EnableIPv4 = true
+			l.EnableIPv6 = false
 			return l, nil
 		case errTryAgain:
 			continue
@@ -288,6 +291,10 @@ func (m *LocalManager) leaseWatchReset(ctx context.Context, sn ip.IP4Net) (Lease
 		return LeaseWatchResult{}, err
 	}
 
+	//TODO only vxlan backend and kube subnet manager support dual stack now.
+	l.EnableIPv4 = true
+	l.EnableIPv6 = false
+
 	return LeaseWatchResult{
 		Snapshot: []Lease{*l},
 		Cursor:   watchCursor{index},
@@ -308,6 +315,9 @@ func (m *LocalManager) WatchLease(ctx context.Context, sn ip.IP4Net, cursor inte
 
 	switch {
 	case err == nil:
+		//TODO only vxlan backend and kube subnet manager support dual stack now.
+		evt.Lease.EnableIPv4 = true
+		evt.Lease.EnableIPv6 = false
 		return LeaseWatchResult{
 			Events: []Event{evt},
 			Cursor: watchCursor{index},
@@ -333,9 +343,11 @@ func (m *LocalManager) WatchLeases(ctx context.Context, cursor interface{}) (Lea
 	}
 
 	evt, index, err := m.registry.watchSubnets(ctx, nextIndex)
-
 	switch {
 	case err == nil:
+		//TODO only vxlan backend and kube subnet manager support dual stack now.
+		evt.Lease.EnableIPv4 = true
+		evt.Lease.EnableIPv6 = false
 		return LeaseWatchResult{
 			Events: []Event{evt},
 			Cursor: watchCursor{index},

subnet/etcdv2/registry.go

@@ -314,6 +314,9 @@ func nodeToLease(node *etcd.Node) (*Lease, error) {
 	}
 
 	lease := Lease{
+		//TODO only vxlan backend and kube subnet manager support dual stack now.
+		EnableIPv4: true,
+		EnableIPv6: false,
 		Subnet:     *sn,
 		Attrs:      *attrs,
 		Expiration: exp,

subnet/etcdv2/registry_test.go

@@ -150,7 +150,7 @@ func TestEtcdRegistry(t *testing.T) {
 	if resp == nil || resp.Node == nil {
 		t.Fatal("Failed to retrive node in subnet lease")
 	}
-	if resp.Node.Value != "{\"PublicIP\":\"1.2.3.4\"}" {
+	if resp.Node.Value != "{\"PublicIP\":\"1.2.3.4\",\"PublicIPv6\":null}" {
 		t.Fatalf("Unexpected subnet lease node %s value %s", resp.Node.Key, resp.Node.Value)
 	}
 

subnet/etcdv2/subnet_test.go

@@ -35,13 +35,13 @@ func newDummyRegistry() *MockSubnetRegistry {
 
 	subnets := []Lease{
 		// leases within SubnetMin-SubnetMax range
-		{ip.IP4Net{ip.MustParseIP4("10.3.1.0"), 24}, attrs, exp, 10},
-		{ip.IP4Net{ip.MustParseIP4("10.3.2.0"), 24}, attrs, exp, 11},
-		{ip.IP4Net{ip.MustParseIP4("10.3.4.0"), 24}, attrs, exp, 12},
-		{ip.IP4Net{ip.MustParseIP4("10.3.5.0"), 24}, attrs, exp, 13},
+		{true, false, ip.IP4Net{ip.MustParseIP4("10.3.1.0"), 24}, ip.IP6Net{}, attrs, exp, 10},
+		{true, false, ip.IP4Net{ip.MustParseIP4("10.3.2.0"), 24}, ip.IP6Net{}, attrs, exp, 11},
+		{true, false, ip.IP4Net{ip.MustParseIP4("10.3.4.0"), 24}, ip.IP6Net{}, attrs, exp, 12},
+		{true, false, ip.IP4Net{ip.MustParseIP4("10.3.5.0"), 24}, ip.IP6Net{}, attrs, exp, 13},
 
 		// hand created lease outside the range of subnetMin-SubnetMax for testing removal
-		{ip.IP4Net{ip.MustParseIP4("10.3.31.0"), 24}, attrs, exp, 13},
+		{true, false, ip.IP4Net{ip.MustParseIP4("10.3.31.0"), 24}, ip.IP6Net{}, attrs, exp, 13},
 	}
 
 	config := `{ "Network": "10.3.0.0/16", "SubnetMin": "10.3.1.0", "SubnetMax": "10.3.25.0" }`

subnet/kube/annotations.go

@@ -21,11 +21,14 @@ import (
 )
 
 type annotations struct {
-	SubnetKubeManaged        string
-	BackendData              string
-	BackendType              string
-	BackendPublicIP          string
-	BackendPublicIPOverwrite string
+	SubnetKubeManaged          string
+	BackendData                string
+	BackendV6Data              string
+	BackendType                string
+	BackendPublicIP            string
+	BackendPublicIPv6          string
+	BackendPublicIPOverwrite   string
+	BackendPublicIPv6Overwrite string
 }
 
 func newAnnotations(prefix string) (annotations, error) {
@@ -55,11 +58,14 @@ func newAnnotations(prefix string) (annotations, error) {
 	}
 
 	a := annotations{
-		SubnetKubeManaged:        prefix + "kube-subnet-manager",
-		BackendData:              prefix + "backend-data",
-		BackendType:              prefix + "backend-type",
-		BackendPublicIP:          prefix + "public-ip",
-		BackendPublicIPOverwrite: prefix + "public-ip-overwrite",
+		SubnetKubeManaged:          prefix + "kube-subnet-manager",
+		BackendData:                prefix + "backend-data",
+		BackendV6Data:              prefix + "backend-v6-data",
+		BackendType:                prefix + "backend-type",
+		BackendPublicIP:            prefix + "public-ip",
+		BackendPublicIPOverwrite:   prefix + "public-ip-overwrite",
+		BackendPublicIPv6:          prefix + "public-ipv6",
+		BackendPublicIPv6Overwrite: prefix + "public-ipv6-overwrite",
 	}
 
 	return a, nil

subnet/kube/kube.go

@@ -51,6 +51,8 @@ const (
 )
 
 type kubeSubnetManager struct {
+	enableIPv4                bool
+	enableIPv6                bool
 	annotations               annotations
 	client                    clientset.Interface
 	nodeName                  string
@@ -134,6 +136,8 @@ func newKubeSubnetManager(ctx context.Context, c clientset.Interface, sc *subnet
 	if err != nil {
 		return nil, err
 	}
+	ksm.enableIPv4 = sc.EnableIPv4
+	ksm.enableIPv6 = sc.EnableIPv6
 	ksm.client = c
 	ksm.nodeName = nodeName
 	ksm.subnetConf = sc
@@ -200,9 +204,20 @@ func (ksm *kubeSubnetManager) handleUpdateLeaseEvent(oldObj, newObj interface{})
 	if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
 		return
 	}
-	if o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&
+	var changed = true
+	if ksm.enableIPv4 && o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&
 		o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&
 		o.Annotations[ksm.annotations.BackendPublicIP] == n.Annotations[ksm.annotations.BackendPublicIP] {
+		changed = false
+	}
+
+	if ksm.enableIPv6 && o.Annotations[ksm.annotations.BackendV6Data] == n.Annotations[ksm.annotations.BackendV6Data] &&
+		o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&
+		o.Annotations[ksm.annotations.BackendPublicIPv6] == n.Annotations[ksm.annotations.BackendPublicIPv6] {
+		changed = false
+	}
+
+	if !changed {
 		return // No change to lease
 	}
 
@@ -228,30 +243,74 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 	if n.Spec.PodCIDR == "" {
 		return nil, fmt.Errorf("node %q pod cidr not assigned", ksm.nodeName)
 	}
-	bd, err := attrs.BackendData.MarshalJSON()
+
+	var bd, v6Bd []byte
+	bd, err = attrs.BackendData.MarshalJSON()
 	if err != nil {
 		return nil, err
 	}
-	_, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
+
+	v6Bd, err = attrs.BackendV6Data.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+
+	var cidr, ipv6Cidr *net.IPNet
+	_, cidr, err = net.ParseCIDR(n.Spec.PodCIDR)
 	if err != nil {
 		return nil, err
 	}
-	if n.Annotations[ksm.annotations.BackendData] != string(bd) ||
+
+	for _, podCidr := range n.Spec.PodCIDRs {
+		_, parseCidr, err := net.ParseCIDR(podCidr)
+		if err != nil {
+			return nil, err
+		}
+		if len(parseCidr.IP) == net.IPv6len {
+			ipv6Cidr = parseCidr
+			break
+		}
+	}
+
+	if (n.Annotations[ksm.annotations.BackendData] != string(bd) ||
 		n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
 		n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||
 		n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
-		(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String()) {
+		(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String())) ||
+		(n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) ||
+			n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
+			n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() ||
+			n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
+			(n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String())) {
 		n.Annotations[ksm.annotations.BackendType] = attrs.BackendType
-		n.Annotations[ksm.annotations.BackendData] = string(bd)
-		if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
-			if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {
-				log.Infof("Overriding public ip with '%s' from node annotation '%s'",
-					n.Annotations[ksm.annotations.BackendPublicIPOverwrite],
-					ksm.annotations.BackendPublicIPOverwrite)
-				n.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite]
+
+		//TODO -i only vxlan backend support dual stack now.
+		if (attrs.BackendType == "vxlan" && string(bd) != "null") || attrs.BackendType != "vxlan" {
+			n.Annotations[ksm.annotations.BackendData] = string(bd)
+			if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
+				if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {
+					log.Infof("Overriding public ip with '%s' from node annotation '%s'",
+						n.Annotations[ksm.annotations.BackendPublicIPOverwrite],
+						ksm.annotations.BackendPublicIPOverwrite)
+					n.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite]
+				}
+			} else {
+				n.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String()
+			}
+		}
+
+		if string(v6Bd) != "null" {
+			n.Annotations[ksm.annotations.BackendV6Data] = string(v6Bd)
+			if n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" {
+				if n.Annotations[ksm.annotations.BackendPublicIPv6] != n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] {
+					log.Infof("Overriding public ipv6 with '%s' from node annotation '%s'",
+						n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite],
+						ksm.annotations.BackendPublicIPv6Overwrite)
+					n.Annotations[ksm.annotations.BackendPublicIPv6] = n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite]
+				}
+			} else {
+				n.Annotations[ksm.annotations.BackendPublicIPv6] = attrs.PublicIPv6.String()
 			}
-		} else {
-			n.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String()
 		}
 		n.Annotations[ksm.annotations.SubnetKubeManaged] = "true"
 
@@ -284,11 +343,23 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 	} else {
 		log.Infoln("Skip setting NodeNetworkUnavailable")
 	}
-	return &subnet.Lease{
-		Subnet:     ip.FromIPNet(cidr),
+
+	lease := &subnet.Lease{
 		Attrs:      *attrs,
 		Expiration: time.Now().Add(24 * time.Hour),
-	}, nil
+	}
+	if cidr != nil {
+		lease.Subnet = ip.FromIPNet(cidr)
+	}
+	if ipv6Cidr != nil {
+		lease.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)
+	}
+	//TODO - only vxlan backend support dual stack now.
+	if attrs.BackendType != "vxlan" {
+		lease.EnableIPv4 = true
+		lease.EnableIPv6 = false
+	}
+	return lease, nil
 }
 
 func (ksm *kubeSubnetManager) WatchLeases(ctx context.Context, cursor interface{}) (subnet.LeaseWatchResult, error) {
@@ -308,20 +379,43 @@ func (ksm *kubeSubnetManager) Run(ctx context.Context) {
 }
 
 func (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l subnet.Lease, err error) {
-	l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP])
-	if err != nil {
-		return l, err
+	if ksm.enableIPv4 {
+		l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP])
+		if err != nil {
+			return l, err
+		}
+		l.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData])
+
+		_, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
+		if err != nil {
+			return l, err
+		}
+		l.Subnet = ip.FromIPNet(cidr)
+		l.EnableIPv4 = ksm.enableIPv4
 	}
 
-	l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]
-	l.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData])
+	if ksm.enableIPv6 {
+		l.Attrs.PublicIPv6, err = ip.ParseIP6(n.Annotations[ksm.annotations.BackendPublicIPv6])
+		if err != nil {
+			return l, err
+		}
+		l.Attrs.BackendV6Data = json.RawMessage(n.Annotations[ksm.annotations.BackendV6Data])
 
-	_, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
-	if err != nil {
-		return l, err
+		ipv6Cidr := new(net.IPNet)
+		for _, podCidr := range n.Spec.PodCIDRs {
+			_, parseCidr, err := net.ParseCIDR(podCidr)
+			if err != nil {
+				return l, err
+			}
+			if len(parseCidr.IP) == net.IPv6len {
+				ipv6Cidr = parseCidr
+				break
+			}
+		}
+		l.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)
+		l.EnableIPv6 = ksm.enableIPv6
 	}
-
-	l.Subnet = ip.FromIPNet(cidr)
+	l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]
 	return l, nil
 }
 

subnet/subnet.go

@@ -34,13 +34,18 @@ var (
 )
 
 type LeaseAttrs struct {
-	PublicIP    ip.IP4
-	BackendType string          `json:",omitempty"`
-	BackendData json.RawMessage `json:",omitempty"`
+	PublicIP      ip.IP4
+	PublicIPv6    *ip.IP6
+	BackendType   string          `json:",omitempty"`
+	BackendData   json.RawMessage `json:",omitempty"`
+	BackendV6Data json.RawMessage `json:",omitempty"`
 }
 
 type Lease struct {
+	EnableIPv4 bool
+	EnableIPv6 bool
 	Subnet     ip.IP4Net
+	IPv6Subnet ip.IP6Net
 	Attrs      LeaseAttrs
 	Expiration time.Time
 

subnet/watch.go

@@ -76,13 +76,39 @@ func (lw *leaseWatcher) reset(leases []Lease) []Event {
 	batch := []Event{}
 
 	for _, nl := range leases {
-		if lw.ownLease != nil && nl.Subnet.Equal(lw.ownLease.Subnet) {
+		if lw.ownLease != nil && nl.EnableIPv4 && !nl.EnableIPv6 &&
+			nl.Subnet.Equal(lw.ownLease.Subnet) {
+			continue
+		} else if lw.ownLease != nil && !nl.EnableIPv4 && nl.EnableIPv6 &&
+			nl.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && nl.EnableIPv4 && nl.EnableIPv6 &&
+			nl.Subnet.Equal(lw.ownLease.Subnet) &&
+			nl.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && !nl.EnableIPv4 && !nl.EnableIPv6 &&
+			nl.Subnet.Equal(lw.ownLease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			continue
 		}
 
 		found := false
 		for i, ol := range lw.leases {
-			if ol.Subnet.Equal(nl.Subnet) {
+			if ol.EnableIPv4 && !ol.EnableIPv6 && ol.Subnet.Equal(nl.Subnet) {
+				lw.leases = deleteLease(lw.leases, i)
+				found = true
+				break
+			} else if ol.EnableIPv4 && !ol.EnableIPv6 && ol.IPv6Subnet.Equal(nl.IPv6Subnet) {
+				lw.leases = deleteLease(lw.leases, i)
+				found = true
+				break
+			} else if ol.EnableIPv4 && ol.EnableIPv6 && ol.Subnet.Equal(nl.Subnet) &&
+				ol.IPv6Subnet.Equal(nl.IPv6Subnet) {
+				lw.leases = deleteLease(lw.leases, i)
+				found = true
+				break
+			} else if !ol.EnableIPv4 && !ol.EnableIPv6 && ol.Subnet.Equal(nl.Subnet) {
+				//TODO - dual-stack temporarily only compatible with kube subnet manager
 				lw.leases = deleteLease(lw.leases, i)
 				found = true
 				break
@@ -97,7 +123,19 @@ func (lw *leaseWatcher) reset(leases []Lease) []Event {
 
 	// everything left in sm.leases has been deleted
 	for _, l := range lw.leases {
-		if lw.ownLease != nil && l.Subnet.Equal(lw.ownLease.Subnet) {
+		if lw.ownLease != nil && l.EnableIPv4 && !l.EnableIPv6 &&
+			l.Subnet.Equal(lw.ownLease.Subnet) {
+			continue
+		} else if lw.ownLease != nil && !l.EnableIPv4 && l.EnableIPv6 &&
+			l.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && l.EnableIPv4 && l.EnableIPv6 &&
+			l.Subnet.Equal(lw.ownLease.Subnet) &&
+			l.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && !l.EnableIPv4 && !l.EnableIPv6 &&
+			l.Subnet.Equal(lw.ownLease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			continue
 		}
 		batch = append(batch, Event{EventRemoved, l})
@@ -114,7 +152,19 @@ func (lw *leaseWatcher) update(events []Event) []Event {
 	batch := []Event{}
 
 	for _, e := range events {
-		if lw.ownLease != nil && e.Lease.Subnet.Equal(lw.ownLease.Subnet) {
+		if lw.ownLease != nil && e.Lease.EnableIPv4 && !e.Lease.EnableIPv6 &&
+			e.Lease.Subnet.Equal(lw.ownLease.Subnet) {
+			continue
+		} else if lw.ownLease != nil && !e.Lease.EnableIPv4 && e.Lease.EnableIPv6 &&
+			e.Lease.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && e.Lease.EnableIPv4 && e.Lease.EnableIPv6 &&
+			e.Lease.Subnet.Equal(lw.ownLease.Subnet) &&
+			e.Lease.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && !e.Lease.EnableIPv4 && !e.Lease.EnableIPv6 &&
+			e.Lease.Subnet.Equal(lw.ownLease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			continue
 		}
 
@@ -132,12 +182,22 @@ func (lw *leaseWatcher) update(events []Event) []Event {
 
 func (lw *leaseWatcher) add(lease *Lease) Event {
 	for i, l := range lw.leases {
-		if l.Subnet.Equal(lease.Subnet) {
+		if l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			lw.leases[i] = *lease
+			return Event{EventAdded, lw.leases[i]}
+		} else if !l.EnableIPv4 && l.EnableIPv6 && l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases[i] = *lease
+			return Event{EventAdded, lw.leases[i]}
+		} else if l.EnableIPv4 && l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) &&
+			l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases[i] = *lease
+			return Event{EventAdded, lw.leases[i]}
+		} else if !l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			lw.leases[i] = *lease
 			return Event{EventAdded, lw.leases[i]}
 		}
 	}
-
 	lw.leases = append(lw.leases, *lease)
 
 	return Event{EventAdded, lw.leases[len(lw.leases)-1]}
@@ -145,13 +205,24 @@ func (lw *leaseWatcher) add(lease *Lease) Event {
 
 func (lw *leaseWatcher) remove(lease *Lease) Event {
 	for i, l := range lw.leases {
-		if l.Subnet.Equal(lease.Subnet) {
+		if l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			lw.leases = deleteLease(lw.leases, i)
+			return Event{EventRemoved, l}
+		} else if !l.EnableIPv4 && l.EnableIPv6 && l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases = deleteLease(lw.leases, i)
+			return Event{EventRemoved, l}
+		} else if l.EnableIPv4 && l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) &&
+			l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases = deleteLease(lw.leases, i)
+			return Event{EventRemoved, l}
+		} else if !l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			lw.leases = deleteLease(lw.leases, i)
 			return Event{EventRemoved, l}
 		}
 	}
 
-	log.Errorf("Removed subnet (%s) was not found", lease.Subnet)
+	log.Errorf("Removed subnet (%s) and ipv6 subnet (%s) were not found", lease.Subnet, lease.IPv6Subnet)
 	return Event{EventRemoved, *lease}
 }