Add PublicIP address as an option

README.md

@@ -175,6 +175,7 @@ $ flanneld --remote=10.0.0.3:8888 --networks=blue,green
 ## Key command line options
 
 ```
+--public-ip="": IP accessible by other nodes for inter-host communication. Defaults to the IP of the interface being used for communication.
 --etcd-endpoints=http://127.0.0.1:4001: a comma-delimited list of etcd endpoints.
 --etcd-prefix=/coreos.com/network: etcd prefix.
 --etcd-keyfile="": SSL key file used to secure etcd communication.

backend/alloc/alloc.go

@@ -29,9 +29,9 @@ func New(sm subnet.Manager, network string) backend.Backend {
 	}
 }
 
-func (m *AllocBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
+func (m *AllocBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) {
 	attrs := subnet.LeaseAttrs{
-		PublicIP: ip.FromIP(extIP),
+		PublicIP: ip.FromIP(extEaddr),
 	}
 
 	l, err := m.sm.AcquireLease(m.ctx, m.network, &attrs)

backend/awsvpc/awsvpc.go

@@ -54,7 +54,7 @@ func New(sm subnet.Manager, network string, config *subnet.Config) backend.Backe
 	return &be
 }
 
-func (m *AwsVpcBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
+func (m *AwsVpcBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) {
 	// Parse our configuration
 	if len(m.config.Backend) > 0 {
 		if err := json.Unmarshal(m.config.Backend, &m.cfg); err != nil {
@@ -64,7 +64,7 @@ func (m *AwsVpcBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.Su
 
 	// Acquire the lease form subnet manager
 	attrs := subnet.LeaseAttrs{
-		PublicIP: ip.FromIP(extIP),
+		PublicIP: ip.FromIP(extEaddr),
 	}
 
 	l, err := m.sm.AcquireLease(m.ctx, m.network, &attrs)

backend/common.go

@@ -26,7 +26,7 @@ type SubnetDef struct {
 }
 
 type Backend interface {
-	Init(extIface *net.Interface, extIP net.IP) (*SubnetDef, error)
+	Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*SubnetDef, error)
 	Run()
 	Stop()
 	Name() string

backend/gce/gce.go

@@ -86,9 +86,9 @@ func New(sm subnet.Manager, network string, config *subnet.Config) backend.Backe
 	return &gb
 }
 
-func (g *GCEBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
+func (g *GCEBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) {
 	attrs := subnet.LeaseAttrs{
-		PublicIP: ip.FromIP(extIP),
+		PublicIP: ip.FromIP(extEaddr),
 	}
 
 	l, err := g.sm.AcquireLease(g.ctx, g.network, &attrs)

backend/hostgw/hostgw.go

@@ -38,7 +38,7 @@ type HostgwBackend struct {
 	network  string
 	lease    *subnet.Lease
 	extIface *net.Interface
-	extIP    net.IP
+	extIaddr net.IP
 	ctx      context.Context
 	cancel   context.CancelFunc
 	wg       sync.WaitGroup
@@ -57,12 +57,16 @@ func New(sm subnet.Manager, network string) backend.Backend {
 	return b
 }
 
-func (rb *HostgwBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
+func (rb *HostgwBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) {
 	rb.extIface = extIface
-	rb.extIP = extIP
+	rb.extIaddr = extIaddr
+
+	if extIaddr.Equal(extEaddr) {
+		return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend")
+	}
 
 	attrs := subnet.LeaseAttrs{
-		PublicIP:    ip.FromIP(extIP),
+		PublicIP:    ip.FromIP(extIaddr),
 		BackendType: "host-gw",
 	}
 
@@ -145,7 +149,7 @@ func (rb *HostgwBackend) handleSubnetEvents(batch []subnet.Event) {
 				Gw:        evt.Lease.Attrs.PublicIP.ToIP(),
 				LinkIndex: rb.extIface.Index,
 			}
-			if rb.extIP.Equal(route.Gw) {
+			if rb.extIaddr.Equal(route.Gw) {
 				continue
 			}
 			if err := netlink.RouteAdd(&route); err != nil {

backend/udp/udp.go

@@ -69,7 +69,7 @@ func New(sm subnet.Manager, network string, config *subnet.Config) backend.Backe
 	return &be
 }
 
-func (m *UdpBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
+func (m *UdpBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) {
 	// Parse our configuration
 	if len(m.config.Backend) > 0 {
 		if err := json.Unmarshal(m.config.Backend, &m.cfg); err != nil {
@@ -79,7 +79,7 @@ func (m *UdpBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.Subne
 
 	// Acquire the lease form subnet manager
 	attrs := subnet.LeaseAttrs{
-		PublicIP: ip.FromIP(extIP),
+		PublicIP: ip.FromIP(extEaddr),
 	}
 
 	l, err := m.sm.AcquireLease(m.ctx, m.network, &attrs)

backend/vxlan/vxlan.go

@@ -66,20 +66,20 @@ func New(sm subnet.Manager, network string, config *subnet.Config) backend.Backe
 	return vb
 }
 
-func newSubnetAttrs(pubIP net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
+func newSubnetAttrs(extEaddr net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
 	data, err := json.Marshal(&vxlanLeaseAttrs{hardwareAddr(mac)})
 	if err != nil {
 		return nil, err
 	}
 
 	return &subnet.LeaseAttrs{
-		PublicIP:    ip.FromIP(pubIP),
+		PublicIP:    ip.FromIP(extEaddr),
 		BackendType: "vxlan",
 		BackendData: json.RawMessage(data),
 	}, nil
 }
 
-func (vb *VXLANBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
+func (vb *VXLANBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) {
 	// Parse our configuration
 	if len(vb.config.Backend) > 0 {
 		if err := json.Unmarshal(vb.config.Backend, &vb.cfg); err != nil {
@@ -91,7 +91,7 @@ func (vb *VXLANBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.Su
 		vni:       uint32(vb.cfg.VNI),
 		name:      fmt.Sprintf("flannel.%v", vb.cfg.VNI),
 		vtepIndex: extIface.Index,
-		vtepAddr:  extIP,
+		vtepAddr:  extIaddr,
 		vtepPort:  vb.cfg.Port,
 	}
 
@@ -109,7 +109,7 @@ func (vb *VXLANBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.Su
 		}
 	}
 
-	sa, err := newSubnetAttrs(extIP, vb.dev.MACAddr())
+	sa, err := newSubnetAttrs(extEaddr, vb.dev.MACAddr())
 	if err != nil {
 		return nil, err
 	}

main.go

@@ -36,6 +36,7 @@ import (
 )
 
 type CmdLineOpts struct {
+	publicIP       string
 	etcdEndpoints  string
 	etcdPrefix     string
 	etcdKeyfile    string
@@ -58,6 +59,7 @@ type CmdLineOpts struct {
 var opts CmdLineOpts
 
 func init() {
+	flag.StringVar(&opts.publicIP, "public-ip", "", "IP accessible by other nodes for inter-host communication")
 	flag.StringVar(&opts.etcdEndpoints, "etcd-endpoints", "http://127.0.0.1:4001,http://127.0.0.1:2379", "a comma-delimited list of etcd endpoints")
 	flag.StringVar(&opts.etcdPrefix, "etcd-prefix", "/coreos.com/network", "etcd prefix")
 	flag.StringVar(&opts.etcdKeyfile, "etcd-keyfile", "", "SSL key file used to secure etcd communication")
@@ -128,12 +130,12 @@ func writeSubnetFile(path string, sn *backend.SubnetDef) error {
 
 func lookupIface() (*net.Interface, net.IP, error) {
 	var iface *net.Interface
-	var ipaddr net.IP
+	var iaddr net.IP
 	var err error
 
 	if len(opts.iface) > 0 {
-		if ipaddr = net.ParseIP(opts.iface); ipaddr != nil {
-			iface, err = ip.GetInterfaceByIP(ipaddr)
+		if iaddr = net.ParseIP(opts.iface); iaddr != nil {
+			iface, err = ip.GetInterfaceByIP(iaddr)
 			if err != nil {
 				return nil, nil, fmt.Errorf("Error looking up interface %s: %s", opts.iface, err)
 			}
@@ -150,14 +152,14 @@ func lookupIface() (*net.Interface, net.IP, error) {
 		}
 	}
 
-	if ipaddr == nil {
-		ipaddr, err = ip.GetIfaceIP4Addr(iface)
+	if iaddr == nil {
+		iaddr, err = ip.GetIfaceIP4Addr(iface)
 		if err != nil {
 			return nil, nil, fmt.Errorf("Failed to find IPv4 address for interface %s", iface.Name)
 		}
 	}
 
-	return iface, ipaddr, nil
+	return iface, iaddr, nil
 }
 
 func isMultiNetwork() bool {
@@ -181,18 +183,29 @@ func newSubnetManager() (subnet.Manager, error) {
 }
 
 func initAndRun(ctx context.Context, sm subnet.Manager, netnames []string) {
-	iface, ipaddr, err := lookupIface()
+	iface, iaddr, err := lookupIface()
 	if err != nil {
 		log.Error(err)
 		return
 	}
 
 	if iface.MTU == 0 {
-		log.Errorf("Failed to determine MTU for %s interface", ipaddr)
+		log.Errorf("Failed to determine MTU for %s interface", iaddr)
 		return
 	}
 
-	log.Infof("Using %s as external interface", ipaddr)
+	var eaddr net.IP
+
+	if len(opts.publicIP) > 0 {
+		eaddr = net.ParseIP(opts.publicIP)
+	}
+
+	if eaddr == nil {
+		eaddr = iaddr
+	}
+
+	log.Infof("Using %s as external interface", iaddr)
+	log.Infof("Using %s as external endpoint", eaddr)
 
 	nets := []*network.Network{}
 	for _, n := range netnames {
@@ -206,7 +219,7 @@ func initAndRun(ctx context.Context, sm subnet.Manager, netnames []string) {
 			wg.Add(1)
 			defer wg.Done()
 
-			sn := n.Init(ctx, iface, ipaddr)
+			sn := n.Init(ctx, iface, iaddr, eaddr)
 			if sn != nil {
 				if isMultiNetwork() {
 					path := filepath.Join(opts.subnetDir, n.Name) + ".env"

network/network.go

@@ -41,7 +41,7 @@ func New(sm subnet.Manager, name string, ipMasq bool) *Network {
 	}
 }
 
-func (n *Network) Init(ctx context.Context, iface *net.Interface, ipaddr net.IP) *backend.SubnetDef {
+func (n *Network) Init(ctx context.Context, iface *net.Interface, iaddr net.IP, eaddr net.IP) *backend.SubnetDef {
 	var cfg *subnet.Config
 	var be backend.Backend
 	var sn *backend.SubnetDef
@@ -66,7 +66,7 @@ func (n *Network) Init(ctx context.Context, iface *net.Interface, ipaddr net.IP)
 		},
 
 		func() (err error) {
-			sn, err = be.Init(iface, ipaddr)
+			sn, err = be.Init(iface, iaddr, eaddr)
 			if err != nil {
 				log.Errorf("Failed to initialize network %v (type %v): %v", n.Name, be.Name(), err)
 			}

subnet/etcd.go

@@ -117,7 +117,7 @@ func findLeaseByIP(leases []Lease, pubIP ip.IP4) *Lease {
 	return nil
 }
 
-func (m *EtcdManager) tryAcquireLease(ctx context.Context, network string, config *Config, extIP ip.IP4, attrs *LeaseAttrs) (*Lease, error) {
+func (m *EtcdManager) tryAcquireLease(ctx context.Context, network string, config *Config, extIaddr ip.IP4, attrs *LeaseAttrs) (*Lease, error) {
 	var err error
 	leases, _, err := m.getLeases(ctx, network)
 	if err != nil {
@@ -130,10 +130,10 @@ func (m *EtcdManager) tryAcquireLease(ctx context.Context, network string, confi
 	}
 
 	// try to reuse a subnet if there's one that matches our IP
-	if l := findLeaseByIP(leases, extIP); l != nil {
+	if l := findLeaseByIP(leases, extIaddr); l != nil {
 		// make sure the existing subnet is still within the configured network
 		if isSubnetConfigCompat(config, l.Subnet) {
-			log.Infof("Found lease (%v) for current IP (%v), reusing", l.Subnet, extIP)
+			log.Infof("Found lease (%v) for current IP (%v), reusing", l.Subnet, extIaddr)
 			resp, err := m.registry.updateSubnet(ctx, network, l.Key(), string(attrBytes), subnetTTL)
 			if err != nil {
 				return nil, err
@@ -143,7 +143,7 @@ func (m *EtcdManager) tryAcquireLease(ctx context.Context, network string, confi
 			l.Expiration = *resp.Node.Expiration
 			return l, nil
 		} else {
-			log.Infof("Found lease (%v) for current IP (%v) but not compatible with current config, deleting", l.Subnet, extIP)
+			log.Infof("Found lease (%v) for current IP (%v) but not compatible with current config, deleting", l.Subnet, extIaddr)
 			if _, err := m.registry.deleteSubnet(ctx, network, l.Key()); err != nil {
 				return nil, err
 			}

subnet/subnet_test.go

@@ -43,9 +43,9 @@ func TestAcquireLease(t *testing.T) {
 	msr := newDummyRegistry(1000)
 	sm := newEtcdManager(msr)
 
-	extIP, _ := ip.ParseIP4("1.2.3.4")
+	extIaddr, _ := ip.ParseIP4("1.2.3.4")
 	attrs := LeaseAttrs{
-		PublicIP: extIP,
+		PublicIP: extIaddr,
 	}
 
 	l, err := sm.AcquireLease(context.Background(), "", &attrs)
@@ -71,9 +71,9 @@ func TestConfigChanged(t *testing.T) {
 	msr := newDummyRegistry(1000)
 	sm := newEtcdManager(msr)
 
-	extIP, _ := ip.ParseIP4("1.2.3.4")
+	extIaddr, _ := ip.ParseIP4("1.2.3.4")
 	attrs := LeaseAttrs{
-		PublicIP: extIP,
+		PublicIP: extIaddr,
 	}
 
 	l, err := sm.AcquireLease(context.Background(), "", &attrs)
@@ -188,9 +188,9 @@ func TestRenewLease(t *testing.T) {
 	sm := newEtcdManager(msr)
 
 	// Create LeaseAttrs
-	extIP, _ := ip.ParseIP4("1.2.3.4")
+	extIaddr, _ := ip.ParseIP4("1.2.3.4")
 	attrs := LeaseAttrs{
-		PublicIP:    extIP,
+		PublicIP:    extIaddr,
 		BackendType: "vxlan",
 	}