Refactor the backend interfaces for multi-networks

Make backends be aware of multiple networks but creating
a separate backend.Network interface.

Also fixes bugs related to flannel exiting and networks
being deleted.

backend/alloc/alloc.go

@@ -2,7 +2,6 @@ package alloc
 
 import (
 	"fmt"
-	"net"
 
 	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/coreos/flannel/backend"
@@ -10,34 +9,38 @@ import (
 	"github.com/coreos/flannel/subnet"
 )
 
+func init() {
+	backend.Register("alloc", New)
+}
+
 type AllocBackend struct {
 	sm       subnet.Manager
-	publicIP ip.IP4
-	mtu      int
-	lease    *subnet.Lease
+	extIface *backend.ExternalInterface
 }
 
-func New(sm subnet.Manager, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error) {
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
 	be := AllocBackend{
-		sm:      sm,
-		publicIP: ip.FromIP(extEaddr),
-		mtu:      extIface.MTU,
+		sm:       sm,
+		extIface: extIface,
 	}
 	return &be, nil
 }
 
-func (m *AllocBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (*backend.SubnetDef, error) {
+func (_ *AllocBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func (be *AllocBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (backend.Network, error) {
 	attrs := subnet.LeaseAttrs{
-		PublicIP: m.publicIP,
+		PublicIP: ip.FromIP(be.extIface.ExtAddr),
 	}
 
-	l, err := m.sm.AcquireLease(ctx, network, &attrs)
+	l, err := be.sm.AcquireLease(ctx, network, &attrs)
 	switch err {
 	case nil:
-		m.lease = l
-		return &backend.SubnetDef{
-			Lease: l,
-			MTU:   m.mtu,
+		return &backend.SimpleNetwork{
+			SubnetLease: l,
+			ExtIface:    be.extIface,
 		}, nil
 
 	case context.Canceled, context.DeadlineExceeded:
@@ -47,9 +50,3 @@ func (m *AllocBackend) RegisterNetwork(ctx context.Context, network string, conf
 		return nil, fmt.Errorf("failed to acquire lease: %v", err)
 	}
 }
-
-func (m *AllocBackend) Run(ctx context.Context) {
-}
-
-func (m *AllocBackend) UnregisterNetwork(ctx context.Context, name string) {
-}

backend/awsvpc/awsvpc.go

@@ -17,7 +17,6 @@ package awsvpc
 import (
 	"encoding/json"
 	"fmt"
-	"net"
 
 	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws"
 	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws/awserr"
@@ -31,42 +30,47 @@ import (
 	"github.com/coreos/flannel/subnet"
 )
 
+func init() {
+	backend.Register("aws-vpc", New)
+}
+
 type AwsVpcBackend struct {
 	sm       subnet.Manager
-	publicIP ip.IP4
-	mtu      int
-	cfg      struct {
-		RouteTableID string
-	}
-	lease *subnet.Lease
+	extIface *backend.ExternalInterface
 }
 
-func New(sm subnet.Manager, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error) {
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
 	be := AwsVpcBackend{
 		sm:       sm,
-		publicIP: ip.FromIP(extEaddr),
-		mtu:      extIface.MTU,
+		extIface: extIface,
 	}
 	return &be, nil
 }
 
-func (m *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (*backend.SubnetDef, error) {
+func (be *AwsVpcBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (backend.Network, error) {
 	// Parse our configuration
+	cfg := struct {
+		RouteTableID string
+	}{}
+
 	if len(config.Backend) > 0 {
-		if err := json.Unmarshal(config.Backend, &m.cfg); err != nil {
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
 			return nil, fmt.Errorf("error decoding VPC backend config: %v", err)
 		}
 	}
 
 	// Acquire the lease form subnet manager
 	attrs := subnet.LeaseAttrs{
-		PublicIP: m.publicIP,
+		PublicIP: ip.FromIP(be.extIface.ExtAddr),
 	}
 
-	l, err := m.sm.AcquireLease(ctx, network, &attrs)
+	l, err := be.sm.AcquireLease(ctx, network, &attrs)
 	switch err {
 	case nil:
-		m.lease = l
 
 	case context.Canceled, context.DeadlineExceeded:
 		return nil, err
@@ -88,20 +92,20 @@ func (m *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, con
 
 	ec2c := ec2.New(&aws.Config{Region: aws.String(region)})
 
-	if _, err = m.disableSrcDestCheck(instanceID, ec2c); err != nil {
+	if _, err = be.disableSrcDestCheck(instanceID, ec2c); err != nil {
 		log.Infof("Warning- disabling source destination check failed: %v", err)
 	}
 
-	if m.cfg.RouteTableID == "" {
+	if cfg.RouteTableID == "" {
 		log.Infof("RouteTableID not passed as config parameter, detecting ...")
-		if err := m.detectRouteTableID(instanceID, ec2c); err != nil {
+		if cfg.RouteTableID, err = be.detectRouteTableID(instanceID, ec2c); err != nil {
 			return nil, err
 		}
 	}
 
-	log.Info("RouteRouteTableID: ", m.cfg.RouteTableID)
+	log.Info("RouteRouteTableID: ", cfg.RouteTableID)
 
-	matchingRouteFound, err := m.checkMatchingRoutes(instanceID, l.Subnet.String(), ec2c)
+	matchingRouteFound, err := be.checkMatchingRoutes(cfg.RouteTableID, instanceID, l.Subnet.String(), ec2c)
 	if err != nil {
 		log.Errorf("Error describing route tables: %v", err)
 
@@ -114,7 +118,7 @@ func (m *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, con
 
 	if !matchingRouteFound {
 		cidrBlock := l.Subnet.String()
-		deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &m.cfg.RouteTableID, DestinationCidrBlock: &cidrBlock}
+		deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &cfg.RouteTableID, DestinationCidrBlock: &cidrBlock}
 		if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil {
 			if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" {
 				// an error other than the route not already existing occurred
@@ -123,25 +127,25 @@ func (m *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, con
 		}
 
 		// Add the route for this machine's subnet
-		if _, err := m.createRoute(instanceID, l.Subnet.String(), ec2c); err != nil {
+		if _, err := be.createRoute(cfg.RouteTableID, instanceID, l.Subnet.String(), ec2c); err != nil {
 			return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err)
 		}
 	}
 
-	return &backend.SubnetDef{
-		Lease: l,
-		MTU:   m.mtu,
+	return &backend.SimpleNetwork{
+		SubnetLease: l,
+		ExtIface:    be.extIface,
 	}, nil
 }
 
-func (m *AwsVpcBackend) checkMatchingRoutes(instanceID, subnet string, ec2c *ec2.EC2) (bool, error) {
+func (be *AwsVpcBackend) checkMatchingRoutes(routeTableID, instanceID, subnet string, ec2c *ec2.EC2) (bool, error) {
 	matchingRouteFound := false
 
 	filter := newFilter()
 	filter.Add("route.destination-cidr-block", subnet)
 	filter.Add("route.state", "active")
 
-	input := ec2.DescribeRouteTablesInput{Filters: filter, RouteTableIds: []*string{&m.cfg.RouteTableID}}
+	input := ec2.DescribeRouteTablesInput{Filters: filter, RouteTableIds: []*string{&routeTableID}}
 
 	resp, err := ec2c.DescribeRouteTables(&input)
 	if err != nil {
@@ -165,16 +169,17 @@ func (m *AwsVpcBackend) checkMatchingRoutes(instanceID, subnet string, ec2c *ec2
 	return matchingRouteFound, nil
 }
 
-func (m *AwsVpcBackend) createRoute(instanceID, subnet string, ec2c *ec2.EC2) (*ec2.CreateRouteOutput, error) {
+func (be *AwsVpcBackend) createRoute(routeTableID, instanceID, subnet string, ec2c *ec2.EC2) (*ec2.CreateRouteOutput, error) {
 	route := &ec2.CreateRouteInput{
-		RouteTableId:         &m.cfg.RouteTableID,
+		RouteTableId:         &routeTableID,
 		InstanceId:           &instanceID,
 		DestinationCidrBlock: &subnet,
 	}
 
 	return ec2c.CreateRoute(route)
 }
-func (m *AwsVpcBackend) disableSrcDestCheck(instanceID string, ec2c *ec2.EC2) (*ec2.ModifyInstanceAttributeOutput, error) {
+
+func (be *AwsVpcBackend) disableSrcDestCheck(instanceID string, ec2c *ec2.EC2) (*ec2.ModifyInstanceAttributeOutput, error) {
 	modifyAttributes := &ec2.ModifyInstanceAttributeInput{
 		InstanceId:      aws.String(instanceID),
 		SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(false)},
@@ -183,22 +188,22 @@ func (m *AwsVpcBackend) disableSrcDestCheck(instanceID string, ec2c *ec2.EC2) (*
 	return ec2c.ModifyInstanceAttribute(modifyAttributes)
 }
 
-func (m *AwsVpcBackend) detectRouteTableID(instanceID string, ec2c *ec2.EC2) error {
+func (be *AwsVpcBackend) detectRouteTableID(instanceID string, ec2c *ec2.EC2) (string, error) {
 	instancesInput := &ec2.DescribeInstancesInput{
 		InstanceIds: []*string{&instanceID},
 	}
 
 	resp, err := ec2c.DescribeInstances(instancesInput)
 	if err != nil {
-		return fmt.Errorf("error getting instance info: %v", err)
+		return "", fmt.Errorf("error getting instance info: %v", err)
 	}
 
 	if len(resp.Reservations) == 0 {
-		return fmt.Errorf("no reservations found")
+		return "", fmt.Errorf("no reservations found")
 	}
 
 	if len(resp.Reservations[0].Instances) == 0 {
-		return fmt.Errorf("no matching instance found with id: %v", instanceID)
+		return "", fmt.Errorf("no matching instance found with id: %v", instanceID)
 	}
 
 	subnetID := resp.Reservations[0].Instances[0].SubnetId
@@ -216,12 +221,11 @@ func (m *AwsVpcBackend) detectRouteTableID(instanceID string, ec2c *ec2.EC2) err
 
 	res, err := ec2c.DescribeRouteTables(routeTablesInput)
 	if err != nil {
-		return fmt.Errorf("error describing routeTables for subnetID %s: %v", *subnetID, err)
+		return "", fmt.Errorf("error describing routeTables for subnetID %s: %v", *subnetID, err)
 	}
 
 	if len(res.RouteTables) != 0 {
-		m.cfg.RouteTableID = *res.RouteTables[0].RouteTableId
-		return nil
+		return *res.RouteTables[0].RouteTableId, nil
 	}
 
 	filter = newFilter()
@@ -238,16 +242,8 @@ func (m *AwsVpcBackend) detectRouteTableID(instanceID string, ec2c *ec2.EC2) err
 	}
 
 	if len(res.RouteTables) == 0 {
-		return fmt.Errorf("main route table not found")
+		return "", fmt.Errorf("main route table not found")
 	}
 
-	m.cfg.RouteTableID = *res.RouteTables[0].RouteTableId
-
-	return nil
-}
-
-func (m *AwsVpcBackend) Run(ctx context.Context) {
-}
-
-func (m *AwsVpcBackend) UnregisterNetwork(ctx context.Context, name string) {
+	return *res.RouteTables[0].RouteTableId, nil
 }

backend/common.go

@@ -15,14 +15,17 @@
 package backend
 
 import (
+	"net"
+
 	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
 
 	"github.com/coreos/flannel/subnet"
 )
 
-type SubnetDef struct {
-	Lease *subnet.Lease
-	MTU   int
+type ExternalInterface struct {
+	Iface     *net.Interface
+	IfaceAddr net.IP
+	ExtAddr   net.IP
 }
 
 // Besides the entry points in the Backend interface, the backend's New()
@@ -37,11 +40,33 @@ type SubnetDef struct {
 // since multiple RegisterNetwork() and Run() calls may be in-flight at any
 // given time for a singleton backend, it must protect these calls with a mutex.
 type Backend interface {
+	// Called first to start the necessary event loops and such
+	Run(ctx context.Context)
 	// Called when the backend should create or begin managing a new network
-	RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (*SubnetDef, error)
-	// Called after the backend's first network has been registered to
-	// allow the plugin to watch dynamic events
+	RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (Network, error)
+}
+
+type Network interface {
+	Lease() *subnet.Lease
+	MTU() int
 	Run(ctx context.Context)
-	// Called to clean up any network resources or operations
-	UnregisterNetwork(ctx context.Context, network string)
+}
+
+type BackendCtor func(sm subnet.Manager, ei *ExternalInterface) (Backend, error)
+
+type SimpleNetwork struct {
+	SubnetLease *subnet.Lease
+	ExtIface    *ExternalInterface
+}
+
+func (n *SimpleNetwork) Lease() *subnet.Lease {
+	return n.SubnetLease
+}
+
+func (n *SimpleNetwork) MTU() int {
+	return n.ExtIface.Iface.MTU
+}
+
+func (_ *SimpleNetwork) Run(ctx context.Context) {
+	<-ctx.Done()
 }

backend/gce/api.go

@@ -0,0 +1,117 @@
+package gce
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/coreos/flannel/Godeps/_workspace/src/code.google.com/p/goauth2/compute/serviceaccount"
+	"github.com/coreos/flannel/Godeps/_workspace/src/code.google.com/p/google-api-go-client/compute/v1"
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+)
+
+type gceAPI struct {
+	project        string
+	computeService *compute.Service
+	gceNetwork     *compute.Network
+	gceInstance    *compute.Instance
+}
+
+func newAPI() (*gceAPI, error) {
+	client, err := serviceaccount.NewClient(&serviceaccount.Options{})
+	if err != nil {
+		return nil, fmt.Errorf("error creating client: %v", err)
+	}
+
+	cs, err := compute.New(client)
+	if err != nil {
+		return nil, fmt.Errorf("error creating compute service: %v", err)
+	}
+
+	networkName, err := networkFromMetadata()
+	if err != nil {
+		return nil, fmt.Errorf("error getting network metadata: %v", err)
+	}
+
+	prj, err := projectFromMetadata()
+	if err != nil {
+		return nil, fmt.Errorf("error getting project: %v", err)
+	}
+
+	instanceName, err := instanceNameFromMetadata()
+	if err != nil {
+		return nil, fmt.Errorf("error getting instance name: %v", err)
+	}
+
+	instanceZone, err := instanceZoneFromMetadata()
+	if err != nil {
+		return nil, fmt.Errorf("error getting instance zone: %v", err)
+	}
+
+	gn, err := cs.Networks.Get(prj, networkName).Do()
+	if err != nil {
+		return nil, fmt.Errorf("error getting network from compute service: %v", err)
+	}
+
+	gi, err := cs.Instances.Get(prj, instanceZone, instanceName).Do()
+	if err != nil {
+		return nil, fmt.Errorf("error getting instance from compute service: %v", err)
+	}
+
+	return &gceAPI{
+		project:        prj,
+		computeService: cs,
+		gceNetwork:     gn,
+		gceInstance:    gi,
+	}, nil
+}
+
+func (api *gceAPI) getRoute(subnet string) (*compute.Route, error) {
+	routeName := formatRouteName(subnet)
+	return api.computeService.Routes.Get(api.project, routeName).Do()
+}
+
+func (api *gceAPI) deleteRoute(subnet string) (*compute.Operation, error) {
+	routeName := formatRouteName(subnet)
+	return api.computeService.Routes.Delete(api.project, routeName).Do()
+}
+
+func (api *gceAPI) insertRoute(subnet string) (*compute.Operation, error) {
+	log.Infof("Inserting route for subnet: %v", subnet)
+	route := &compute.Route{
+		Name:            formatRouteName(subnet),
+		DestRange:       subnet,
+		Network:         api.gceNetwork.SelfLink,
+		NextHopInstance: api.gceInstance.SelfLink,
+		Priority:        1000,
+		Tags:            []string{},
+	}
+	return api.computeService.Routes.Insert(api.project, route).Do()
+}
+
+func (api *gceAPI) pollOperationStatus(operationName string) error {
+	for i := 0; i < 100; i++ {
+		operation, err := api.computeService.GlobalOperations.Get(api.project, operationName).Do()
+		if err != nil {
+			return fmt.Errorf("error fetching operation status: %v", err)
+		}
+
+		if operation.Error != nil {
+			return fmt.Errorf("error running operation: %v", operation.Error)
+		}
+
+		if i%5 == 0 {
+			log.Infof("%v operation status: %v waiting for completion...", operation.OperationType, operation.Status)
+		}
+
+		if operation.Status == "DONE" {
+			return nil
+		}
+		time.Sleep(time.Second)
+	}
+
+	return fmt.Errorf("timeout waiting for operation to finish")
+}
+
+func formatRouteName(subnet string) string {
+	return fmt.Sprintf("flannel-%s", replacer.Replace(subnet))
+}

backend/gce/gce.go

@@ -39,12 +39,9 @@ package gce
 
 import (
 	"fmt"
-	"net"
 	"strings"
-	"time"
+	"sync"
 
-	"github.com/coreos/flannel/Godeps/_workspace/src/code.google.com/p/goauth2/compute/serviceaccount"
-	"github.com/coreos/flannel/Godeps/_workspace/src/code.google.com/p/google-api-go-client/compute/v1"
 	"github.com/coreos/flannel/Godeps/_workspace/src/code.google.com/p/google-api-go-client/googleapi"
 	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
 	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
@@ -54,39 +51,49 @@ import (
 	"github.com/coreos/flannel/subnet"
 )
 
+func init() {
+	backend.Register("gce", New)
+}
+
 var metadataEndpoint = "http://169.254.169.254/computeMetadata/v1"
 
 var replacer = strings.NewReplacer(".", "-", "/", "-")
 
 type GCEBackend struct {
-	sm             subnet.Manager
-	publicIP       ip.IP4
-	mtu            int
-	project        string
-	lease          *subnet.Lease
-	computeService *compute.Service
-	gceNetwork     *compute.Network
-	gceInstance    *compute.Instance
+	sm       subnet.Manager
+	extIface *backend.ExternalInterface
+	apiInit  sync.Once
+	api      *gceAPI
 }
 
-func New(sm subnet.Manager, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error) {
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
 	gb := GCEBackend{
 		sm:       sm,
-		publicIP: ip.FromIP(extEaddr),
-		mtu:      extIface.MTU,
+		extIface: extIface,
 	}
 	return &gb, nil
 }
 
-func (g *GCEBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (*backend.SubnetDef, error) {
+func (g *GCEBackend) ensureAPI() error {
+	var err error
+	g.apiInit.Do(func() {
+		g.api, err = newAPI()
+	})
+	return err
+}
+
+func (g *GCEBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func (g *GCEBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (backend.Network, error) {
 	attrs := subnet.LeaseAttrs{
-		PublicIP: g.publicIP,
+		PublicIP: ip.FromIP(g.extIface.ExtAddr),
 	}
 
 	l, err := g.sm.AcquireLease(ctx, network, &attrs)
 	switch err {
 	case nil:
-		g.lease = l
 
 	case context.Canceled, context.DeadlineExceeded:
 		return nil, err
@@ -95,44 +102,8 @@ func (g *GCEBackend) RegisterNetwork(ctx context.Context, network string, config
 		return nil, fmt.Errorf("failed to acquire lease: %v", err)
 	}
 
-	client, err := serviceaccount.NewClient(&serviceaccount.Options{})
-	if err != nil {
-		return nil, fmt.Errorf("error creating client: %v", err)
-	}
-
-	g.computeService, err = compute.New(client)
-	if err != nil {
-		return nil, fmt.Errorf("error creating compute service: %v", err)
-	}
-
-	networkName, err := networkFromMetadata()
-	if err != nil {
-		return nil, fmt.Errorf("error getting network metadata: %v", err)
-	}
-
-	g.project, err = projectFromMetadata()
-	if err != nil {
-		return nil, fmt.Errorf("error getting project: %v", err)
-	}
-
-	instanceName, err := instanceNameFromMetadata()
-	if err != nil {
-		return nil, fmt.Errorf("error getting instance name: %v", err)
-	}
-
-	instanceZone, err := instanceZoneFromMetadata()
-	if err != nil {
-		return nil, fmt.Errorf("error getting instance zone: %v", err)
-	}
-
-	g.gceNetwork, err = g.computeService.Networks.Get(g.project, networkName).Do()
-	if err != nil {
-		return nil, fmt.Errorf("error getting network from compute service: %v", err)
-	}
-
-	g.gceInstance, err = g.computeService.Instances.Get(g.project, instanceZone, instanceName).Do()
-	if err != nil {
-		return nil, fmt.Errorf("error getting instance from compute service: %v", err)
+	if err = g.ensureAPI(); err != nil {
+		return nil, err
 	}
 
 	found, err := g.handleMatchingRoute(l.Subnet.String())
@@ -141,56 +112,26 @@ func (g *GCEBackend) RegisterNetwork(ctx context.Context, network string, config
 	}
 
 	if !found {
-		operation, err := g.insertRoute(l.Subnet.String())
+		operation, err := g.api.insertRoute(l.Subnet.String())
 		if err != nil {
 			return nil, fmt.Errorf("error inserting route: %v", err)
 		}
 
-		err = g.pollOperationStatus(operation.Name)
+		err = g.api.pollOperationStatus(operation.Name)
 		if err != nil {
 			return nil, fmt.Errorf("insert operaiton failed: ", err)
 		}
 	}
 
-	return &backend.SubnetDef{
-		Lease: l,
-		MTU:   g.mtu,
+	return &backend.SimpleNetwork{
+		SubnetLease: l,
+		ExtIface:    g.extIface,
 	}, nil
 }
 
-func (g *GCEBackend) Run(ctx context.Context) {
-}
-
-func (g *GCEBackend) UnregisterNetwork(ctx context.Context, name string) {
-}
-
-func (g *GCEBackend) pollOperationStatus(operationName string) error {
-	for i := 0; i < 100; i++ {
-		operation, err := g.computeService.GlobalOperations.Get(g.project, operationName).Do()
-		if err != nil {
-			return fmt.Errorf("error fetching operation status: %v", err)
-		}
-
-		if operation.Error != nil {
-			return fmt.Errorf("error running operation: %v", operation.Error)
-		}
-
-		if i%5 == 0 {
-			log.Infof("%v operation status: %v waiting for completion...", operation.OperationType, operation.Status)
-		}
-
-		if operation.Status == "DONE" {
-			return nil
-		}
-		time.Sleep(time.Second)
-	}
-
-	return fmt.Errorf("timeout waiting for operation to finish")
-}
-
 //returns true if an exact matching rule is found
 func (g *GCEBackend) handleMatchingRoute(subnet string) (bool, error) {
-	matchingRoute, err := g.getRoute(subnet)
+	matchingRoute, err := g.api.getRoute(subnet)
 	if err != nil {
 		if apiError, ok := err.(*googleapi.Error); ok {
 			if apiError.Code != 404 {
@@ -201,49 +142,21 @@ func (g *GCEBackend) handleMatchingRoute(subnet string) (bool, error) {
 		return false, fmt.Errorf("error getting googleapi: %v", err)
 	}
 
-	if matchingRoute.NextHopInstance == g.gceInstance.SelfLink {
+	if matchingRoute.NextHopInstance == g.api.gceInstance.SelfLink {
 		log.Info("Exact pre-existing route found")
 		return true, nil
 	}
 
 	log.Info("Deleting conflicting route")
-	operation, err := g.deleteRoute(subnet)
+	operation, err := g.api.deleteRoute(subnet)
 	if err != nil {
 		return false, fmt.Errorf("error deleting conflicting route : %v", err)
 	}
 
-	err = g.pollOperationStatus(operation.Name)
+	err = g.api.pollOperationStatus(operation.Name)
 	if err != nil {
 		return false, fmt.Errorf("delete operation failed: %v", err)
 	}
 
 	return false, nil
-
-}
-
-func (g *GCEBackend) getRoute(subnet string) (*compute.Route, error) {
-	routeName := formatRouteName(subnet)
-	return g.computeService.Routes.Get(g.project, routeName).Do()
-}
-
-func (g *GCEBackend) deleteRoute(subnet string) (*compute.Operation, error) {
-	routeName := formatRouteName(subnet)
-	return g.computeService.Routes.Delete(g.project, routeName).Do()
-}
-
-func (g *GCEBackend) insertRoute(subnet string) (*compute.Operation, error) {
-	log.Infof("Inserting route for subnet: %v", subnet)
-	route := &compute.Route{
-		Name:            formatRouteName(subnet),
-		DestRange:       subnet,
-		Network:         g.gceNetwork.SelfLink,
-		NextHopInstance: g.gceInstance.SelfLink,
-		Priority:        1000,
-		Tags:            []string{},
-	}
-	return g.computeService.Routes.Insert(g.project, route).Do()
-}
-
-func formatRouteName(subnet string) string {
-	return fmt.Sprintf("flannel-%s", replacer.Replace(subnet))
 }

backend/hostgw/hostgw.go

@@ -15,62 +15,62 @@
 package hostgw
 
 import (
-	"bytes"
 	"fmt"
-	"net"
-	"sync"
-	"time"
 
-	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
-	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
 	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/coreos/flannel/backend"
 	"github.com/coreos/flannel/pkg/ip"
 	"github.com/coreos/flannel/subnet"
 )
 
+func init() {
+	backend.Register("host-gw", New)
+}
+
 const (
 	routeCheckRetries = 10
 )
 
 type HostgwBackend struct {
 	sm       subnet.Manager
-	publicIP ip.IP4
-	network  string
-	lease    *subnet.Lease
-	extIface *net.Interface
-	extIaddr net.IP
-	mtu      int
-	rl       []netlink.Route
+	extIface *backend.ExternalInterface
+	networks map[string]*network
 }
 
-func New(sm subnet.Manager, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error) {
-	if !extIaddr.Equal(extEaddr) {
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
+	if !extIface.ExtAddr.Equal(extIface.IfaceAddr) {
 		return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend")
 	}
 
-	b := &HostgwBackend{
+	be := &HostgwBackend{
 		sm:       sm,
-		publicIP: ip.FromIP(extEaddr),
-		mtu:      extIface.MTU,
 		extIface: extIface,
-		extIaddr: extIaddr,
+		networks: make(map[string]*network),
 	}
-	return b, nil
+
+	return be, nil
 }
 
-func (rb *HostgwBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (*backend.SubnetDef, error) {
-	rb.network = network
+func (_ *HostgwBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func (be *HostgwBackend) RegisterNetwork(ctx context.Context, netname string, config *subnet.Config) (backend.Network, error) {
+	n := &network{
+		name:     netname,
+		extIface: be.extIface,
+		sm:       be.sm,
+	}
 
 	attrs := subnet.LeaseAttrs{
-		PublicIP:    rb.publicIP,
+		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
 		BackendType: "host-gw",
 	}
 
-	l, err := rb.sm.AcquireLease(ctx, rb.network, &attrs)
+	l, err := be.sm.AcquireLease(ctx, netname, &attrs)
 	switch err {
 	case nil:
-		rb.lease = l
+		n.lease = l
 
 	case context.Canceled, context.DeadlineExceeded:
 		return nil, err
@@ -81,148 +81,7 @@ func (rb *HostgwBackend) RegisterNetwork(ctx context.Context, network string, co
 
 	/* NB: docker will create the local route to `sn` */
 
-	return &backend.SubnetDef{
-		Lease: l,
-		MTU:   rb.mtu,
-	}, nil
-}
-
-func (rb *HostgwBackend) Run(ctx context.Context) {
-	wg := sync.WaitGroup{}
-
-	log.Info("Watching for new subnet leases")
-	evts := make(chan []subnet.Event)
-	wg.Add(1)
-	go func() {
-		subnet.WatchLeases(ctx, rb.sm, rb.network, rb.lease, evts)
-		wg.Done()
-	}()
-
-	rb.rl = make([]netlink.Route, 0, 10)
-	wg.Add(1)
-	go func() {
-		rb.routeCheck(ctx)
-		wg.Done()
-	}()
-
-	defer wg.Wait()
-
-	for {
-		select {
-		case evtBatch := <-evts:
-			rb.handleSubnetEvents(evtBatch)
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
+	be.networks[netname] = n
 
-func (rb *HostgwBackend) UnregisterNetwork(ctx context.Context, name string) {
-}
-
-func (rb *HostgwBackend) handleSubnetEvents(batch []subnet.Event) {
-	for _, evt := range batch {
-		switch evt.Type {
-		case subnet.EventAdded:
-			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
-
-			if evt.Lease.Attrs.BackendType != "host-gw" {
-				log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
-				continue
-			}
-
-			route := netlink.Route{
-				Dst:       evt.Lease.Subnet.ToIPNet(),
-				Gw:        evt.Lease.Attrs.PublicIP.ToIP(),
-				LinkIndex: rb.extIface.Index,
-			}
-			if err := netlink.RouteAdd(&route); err != nil {
-				log.Errorf("Error adding route to %v via %v: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, err)
-				continue
-			}
-			rb.addToRouteList(route)
-
-		case subnet.EventRemoved:
-			log.Info("Subnet removed: ", evt.Lease.Subnet)
-
-			if evt.Lease.Attrs.BackendType != "host-gw" {
-				log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
-				continue
-			}
-
-			route := netlink.Route{
-				Dst:       evt.Lease.Subnet.ToIPNet(),
-				Gw:        evt.Lease.Attrs.PublicIP.ToIP(),
-				LinkIndex: rb.extIface.Index,
-			}
-			if err := netlink.RouteDel(&route); err != nil {
-				log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
-				continue
-			}
-			rb.removeFromRouteList(route)
-
-		default:
-			log.Error("Internal error: unknown event type: ", int(evt.Type))
-		}
-	}
-}
-
-func (rb *HostgwBackend) addToRouteList(route netlink.Route) {
-	rb.rl = append(rb.rl, route)
-}
-
-func (rb *HostgwBackend) removeFromRouteList(route netlink.Route) {
-	for index, r := range rb.rl {
-		if routeEqual(r, route) {
-			rb.rl = append(rb.rl[:index], rb.rl[index+1:]...)
-			return
-		}
-	}
-}
-
-func (rb *HostgwBackend) routeCheck(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(routeCheckRetries * time.Second):
-			rb.checkSubnetExistInRoutes()
-		}
-	}
-}
-
-func (rb *HostgwBackend) checkSubnetExistInRoutes() {
-	routeList, err := netlink.RouteList(nil, netlink.FAMILY_V4)
-	if err == nil {
-		for _, route := range rb.rl {
-			exist := false
-			for _, r := range routeList {
-				if r.Dst == nil {
-					continue
-				}
-				if routeEqual(r, route) {
-					exist = true
-					break
-				}
-			}
-			if !exist {
-				if err := netlink.RouteAdd(&route); err != nil {
-					if nerr, ok := err.(net.Error); !ok {
-						log.Errorf("Error recovering route to %v: %v, %v", route.Dst, route.Gw, nerr)
-					}
-					continue
-				} else {
-					log.Infof("Route recovered %v : %v", route.Dst, route.Gw)
-				}
-			}
-		}
-	}
-}
-
-func routeEqual(x, y netlink.Route) bool {
-	if x.Dst.IP.Equal(y.Dst.IP) && x.Gw.Equal(y.Gw) && bytes.Equal(x.Dst.Mask, y.Dst.Mask) {
-		return true
-	}
-	return false
+	return n, nil
 }

backend/hostgw/network.go

@@ -0,0 +1,183 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostgw
+
+import (
+	"bytes"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
+	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/subnet"
+)
+
+type network struct {
+	name      string
+	extIface  *backend.ExternalInterface
+	linkIndex int
+	rl        []netlink.Route
+	lease     *subnet.Lease
+	sm        subnet.Manager
+}
+
+func (n *network) Lease() *subnet.Lease {
+	return n.lease
+}
+
+func (n *network) MTU() int {
+	return n.extIface.Iface.MTU
+}
+
+func (n *network) Run(ctx context.Context) {
+	wg := sync.WaitGroup{}
+
+	log.Info("Watching for new subnet leases")
+	evts := make(chan []subnet.Event)
+	wg.Add(1)
+	go func() {
+		subnet.WatchLeases(ctx, n.sm, n.name, n.lease, evts)
+		wg.Done()
+	}()
+
+	n.rl = make([]netlink.Route, 0, 10)
+	wg.Add(1)
+	go func() {
+		n.routeCheck(ctx)
+		wg.Done()
+	}()
+
+	defer wg.Wait()
+
+	for {
+		select {
+		case evtBatch := <-evts:
+			n.handleSubnetEvents(evtBatch)
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (n *network) handleSubnetEvents(batch []subnet.Event) {
+	for _, evt := range batch {
+		switch evt.Type {
+		case subnet.EventAdded:
+			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
+
+			if evt.Lease.Attrs.BackendType != "host-gw" {
+				log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			route := netlink.Route{
+				Dst:       evt.Lease.Subnet.ToIPNet(),
+				Gw:        evt.Lease.Attrs.PublicIP.ToIP(),
+				LinkIndex: n.linkIndex,
+			}
+			if err := netlink.RouteAdd(&route); err != nil {
+				log.Errorf("Error adding route to %v via %v: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, err)
+				continue
+			}
+			n.addToRouteList(route)
+
+		case subnet.EventRemoved:
+			log.Info("Subnet removed: ", evt.Lease.Subnet)
+
+			if evt.Lease.Attrs.BackendType != "host-gw" {
+				log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			route := netlink.Route{
+				Dst:       evt.Lease.Subnet.ToIPNet(),
+				Gw:        evt.Lease.Attrs.PublicIP.ToIP(),
+				LinkIndex: n.linkIndex,
+			}
+			if err := netlink.RouteDel(&route); err != nil {
+				log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
+				continue
+			}
+			n.removeFromRouteList(route)
+
+		default:
+			log.Error("Internal error: unknown event type: ", int(evt.Type))
+		}
+	}
+}
+
+func (n *network) addToRouteList(route netlink.Route) {
+	n.rl = append(n.rl, route)
+}
+
+func (n *network) removeFromRouteList(route netlink.Route) {
+	for index, r := range n.rl {
+		if routeEqual(r, route) {
+			n.rl = append(n.rl[:index], n.rl[index+1:]...)
+			return
+		}
+	}
+}
+
+func (n *network) routeCheck(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(routeCheckRetries * time.Second):
+			n.checkSubnetExistInRoutes()
+		}
+	}
+}
+
+func (n *network) checkSubnetExistInRoutes() {
+	routeList, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+	if err == nil {
+		for _, route := range n.rl {
+			exist := false
+			for _, r := range routeList {
+				if r.Dst == nil {
+					continue
+				}
+				if routeEqual(r, route) {
+					exist = true
+					break
+				}
+			}
+			if !exist {
+				if err := netlink.RouteAdd(&route); err != nil {
+					if nerr, ok := err.(net.Error); !ok {
+						log.Errorf("Error recovering route to %v: %v, %v", route.Dst, route.Gw, nerr)
+					}
+					continue
+				} else {
+					log.Infof("Route recovered %v : %v", route.Dst, route.Gw)
+				}
+			}
+		}
+	}
+}
+
+func routeEqual(x, y netlink.Route) bool {
+	if x.Dst.IP.Equal(y.Dst.IP) && x.Gw.Equal(y.Gw) && bytes.Equal(x.Dst.Mask, y.Dst.Mask) {
+		return true
+	}
+	return false
+}

backend/manager.go

@@ -0,0 +1,85 @@
+package backend
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
+
+	"github.com/coreos/flannel/subnet"
+)
+
+var backendCtors map[string]BackendCtor = make(map[string]BackendCtor)
+
+type Manager interface {
+	GetBackend(backendType string) (Backend, error)
+	Wait()
+}
+
+type manager struct {
+	ctx      context.Context
+	sm       subnet.Manager
+	extIface *ExternalInterface
+	mux      sync.Mutex
+	active   map[string]Backend
+	wg       sync.WaitGroup
+}
+
+func NewManager(ctx context.Context, sm subnet.Manager, extIface *ExternalInterface) Manager {
+	return &manager{
+		ctx:      ctx,
+		sm:       sm,
+		extIface: extIface,
+	}
+}
+
+func (bm *manager) GetBackend(backendType string) (Backend, error) {
+	bm.mux.Lock()
+	defer bm.mux.Unlock()
+
+	betype := strings.ToLower(backendType)
+	// see if one is already running
+	if be, ok := bm.active[betype]; ok {
+		return be, nil
+	}
+
+	// first request, need to create and run it
+	befunc, ok := backendCtors[betype]
+	if !ok {
+		return nil, fmt.Errorf("unknown backend type: %v", betype)
+	}
+
+	be, err := befunc(bm.sm, bm.extIface)
+	if err != nil {
+		return nil, err
+	}
+
+	bm.wg.Add(1)
+	go func() {
+		be.Run(bm.ctx)
+
+		// TODO(eyakubovich): this obviosly introduces a race.
+		// GetBackend() could get called while we are here.
+		// Currently though, all backends' Run exit only
+		// on shutdown
+
+		bm.mux.Lock()
+		delete(bm.active, betype)
+		bm.mux.Unlock()
+
+		bm.wg.Done()
+	}()
+
+	return be, nil
+}
+
+func (bm *manager) Wait() {
+	bm.wg.Wait()
+}
+
+func Register(name string, ctor BackendCtor) {
+	log.Infof("Register: %v", name)
+	backendCtors[name] = ctor
+}

backend/udp/network.go

@@ -0,0 +1,204 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package udp
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"sync"
+	"syscall"
+
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
+	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+)
+
+const (
+	encapOverhead = 28 // 20 bytes IP hdr + 8 bytes UDP hdr
+)
+
+type network struct {
+	backend.SimpleNetwork
+	name   string
+	port   int
+	ctl    *os.File
+	ctl2   *os.File
+	tun    *os.File
+	conn   *net.UDPConn
+	tunNet ip.IP4Net
+	sm     subnet.Manager
+}
+
+func newNetwork(name string, sm subnet.Manager, extIface *backend.ExternalInterface, port int, nw ip.IP4Net, l *subnet.Lease) (*network, error) {
+	n := &network{
+		SimpleNetwork: backend.SimpleNetwork{
+			SubnetLease: l,
+			ExtIface:    extIface,
+		},
+		name: name,
+		port: port,
+		sm:   sm,
+	}
+
+	n.tunNet = nw
+
+	if err := n.initTun(); err != nil {
+		return nil, err
+	}
+
+	var err error
+	n.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, fmt.Errorf("failed to start listening on UDP socket: %v", err)
+	}
+
+	n.ctl, n.ctl2, err = newCtlSockets()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create control socket: %v", err)
+	}
+
+	return n, nil
+}
+
+func (n *network) Run(ctx context.Context) {
+	defer func() {
+		n.tun.Close()
+		n.conn.Close()
+		n.ctl.Close()
+		n.ctl2.Close()
+	}()
+
+	// one for each goroutine below
+	wg := sync.WaitGroup{}
+	defer wg.Wait()
+
+	wg.Add(1)
+	go func() {
+		runCProxy(n.tun, n.conn, n.ctl2, n.tunNet.IP, n.MTU())
+		wg.Done()
+	}()
+
+	log.Info("Watching for new subnet leases")
+
+	evts := make(chan []subnet.Event)
+
+	wg.Add(1)
+	go func() {
+		subnet.WatchLeases(ctx, n.sm, n.name, n.SubnetLease, evts)
+		wg.Done()
+	}()
+
+	for {
+		select {
+		case evtBatch := <-evts:
+			n.processSubnetEvents(evtBatch)
+
+		case <-ctx.Done():
+			stopProxy(n.ctl)
+			return
+		}
+	}
+}
+
+func (n *network) MTU() int {
+	return n.ExtIface.Iface.MTU - encapOverhead
+}
+
+func newCtlSockets() (*os.File, *os.File, error) {
+	fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_SEQPACKET, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	f1 := os.NewFile(uintptr(fds[0]), "ctl")
+	f2 := os.NewFile(uintptr(fds[1]), "ctl")
+	return f1, f2, nil
+}
+
+func (n *network) initTun() error {
+	var tunName string
+	var err error
+
+	n.tun, tunName, err = ip.OpenTun("flannel%d")
+	if err != nil {
+		return fmt.Errorf("failed to open TUN device: %v", err)
+	}
+
+	err = configureIface(tunName, n.tunNet, n.MTU())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func configureIface(ifname string, ipn ip.IP4Net, mtu int) error {
+	iface, err := netlink.LinkByName(ifname)
+	if err != nil {
+		return fmt.Errorf("failed to lookup interface %v", ifname)
+	}
+
+	err = netlink.AddrAdd(iface, &netlink.Addr{ipn.ToIPNet(), ""})
+	if err != nil {
+		return fmt.Errorf("failed to add IP address %v to %v: %v", ipn.String(), ifname, err)
+	}
+
+	err = netlink.LinkSetMTU(iface, mtu)
+	if err != nil {
+		return fmt.Errorf("failed to set MTU for %v: %v", ifname, err)
+	}
+
+	err = netlink.LinkSetUp(iface)
+	if err != nil {
+		return fmt.Errorf("failed to set interface %v to UP state: %v", ifname, err)
+	}
+
+	// explicitly add a route since there might be a route for a subnet already
+	// installed by Docker and then it won't get auto added
+	err = netlink.RouteAdd(&netlink.Route{
+		LinkIndex: iface.Attrs().Index,
+		Scope:     netlink.SCOPE_UNIVERSE,
+		Dst:       ipn.Network().ToIPNet(),
+	})
+	if err != nil && err != syscall.EEXIST {
+		return fmt.Errorf("failed to add route (%v -> %v): %v", ipn.Network().String(), ifname, err)
+	}
+
+	return nil
+}
+
+func (n *network) processSubnetEvents(batch []subnet.Event) {
+	for _, evt := range batch {
+		switch evt.Type {
+		case subnet.EventAdded:
+			log.Info("Subnet added: ", evt.Lease.Subnet)
+
+			setRoute(n.ctl, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, n.port)
+
+		case subnet.EventRemoved:
+			log.Info("Subnet removed: ", evt.Lease.Subnet)
+
+			removeRoute(n.ctl, evt.Lease.Subnet)
+
+		default:
+			log.Error("Internal error: unknown event type: ", int(evt.Type))
+		}
+	}
+}

backend/udp/udp.go

@@ -17,13 +17,7 @@ package udp
 import (
 	"encoding/json"
 	"fmt"
-	"net"
-	"os"
-	"sync"
-	"syscall"
 
-	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
-	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
 	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
 
 	"github.com/coreos/flannel/backend"
@@ -31,57 +25,49 @@ import (
 	"github.com/coreos/flannel/subnet"
 )
 
+func init() {
+	backend.Register("udp", New)
+}
+
 const (
-	encapOverhead = 28 // 20 bytes IP hdr + 8 bytes UDP hdr
-	defaultPort   = 8285
+	defaultPort = 8285
 )
 
 type UdpBackend struct {
 	sm       subnet.Manager
-	network  string
-	publicIP ip.IP4
-	cfg      struct {
-		Port int
-	}
-	lease  *subnet.Lease
-	ctl    *os.File
-	ctl2   *os.File
-	tun    *os.File
-	conn   *net.UDPConn
-	mtu    int
-	tunNet ip.IP4Net
+	extIface *backend.ExternalInterface
 }
 
-func New(sm subnet.Manager, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error) {
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
 	be := UdpBackend{
 		sm:       sm,
-		publicIP: ip.FromIP(extEaddr),
-		// TUN MTU will be smaller b/c of encap (IP+UDP hdrs)
-		mtu: extIface.MTU - encapOverhead,
+		extIface: extIface,
 	}
-	be.cfg.Port = defaultPort
 	return &be, nil
 }
 
-func (m *UdpBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (*backend.SubnetDef, error) {
-	m.network = network
+func (be *UdpBackend) RegisterNetwork(ctx context.Context, netname string, config *subnet.Config) (backend.Network, error) {
+	cfg := struct {
+		Port int
+	}{
+		Port: defaultPort,
+	}
 
 	// Parse our configuration
 	if len(config.Backend) > 0 {
-		if err := json.Unmarshal(config.Backend, &m.cfg); err != nil {
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
 			return nil, fmt.Errorf("error decoding UDP backend config: %v", err)
 		}
 	}
 
 	// Acquire the lease form subnet manager
 	attrs := subnet.LeaseAttrs{
-		PublicIP: m.publicIP,
+		PublicIP: ip.FromIP(be.extIface.ExtAddr),
 	}
 
-	l, err := m.sm.AcquireLease(ctx, m.network, &attrs)
+	l, err := be.sm.AcquireLease(ctx, netname, &attrs)
 	switch err {
 	case nil:
-		m.lease = l
 
 	case context.Canceled, context.DeadlineExceeded:
 		return nil, err
@@ -92,146 +78,14 @@ func (m *UdpBackend) RegisterNetwork(ctx context.Context, network string, config
 
 	// Tunnel's subnet is that of the whole overlay network (e.g. /16)
 	// and not that of the individual host (e.g. /24)
-	m.tunNet = ip.IP4Net{
+	tunNet := ip.IP4Net{
 		IP:        l.Subnet.IP,
 		PrefixLen: config.Network.PrefixLen,
 	}
 
-	if err = m.initTun(); err != nil {
-		return nil, err
-	}
-
-	m.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: m.cfg.Port})
-	if err != nil {
-		return nil, fmt.Errorf("failed to start listening on UDP socket: %v", err)
-	}
-
-	m.ctl, m.ctl2, err = newCtlSockets()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create control socket: %v", err)
-	}
-
-	return &backend.SubnetDef{
-		Lease: l,
-		MTU:   m.mtu,
-	}, nil
-}
-
-func (m *UdpBackend) Run(ctx context.Context) {
-	// one for each goroutine below
-	wg := sync.WaitGroup{}
-
-	wg.Add(1)
-	go func() {
-		runCProxy(m.tun, m.conn, m.ctl2, m.tunNet.IP, m.mtu)
-		wg.Done()
-	}()
-
-	log.Info("Watching for new subnet leases")
-
-	evts := make(chan []subnet.Event)
-
-	wg.Add(1)
-	go func() {
-		subnet.WatchLeases(ctx, m.sm, m.network, m.lease, evts)
-		wg.Done()
-	}()
-
-	for {
-		select {
-		case evtBatch := <-evts:
-			m.processSubnetEvents(evtBatch)
-
-		case <-ctx.Done():
-			stopProxy(m.ctl)
-			break
-		}
-	}
-
-	wg.Wait()
+	return newNetwork(netname, be.sm, be.extIface, cfg.Port, tunNet, l)
 }
 
-func (m *UdpBackend) UnregisterNetwork(ctx context.Context, name string) {
-}
-
-func newCtlSockets() (*os.File, *os.File, error) {
-	fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_SEQPACKET, 0)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	f1 := os.NewFile(uintptr(fds[0]), "ctl")
-	f2 := os.NewFile(uintptr(fds[1]), "ctl")
-	return f1, f2, nil
-}
-
-func (m *UdpBackend) initTun() error {
-	var tunName string
-	var err error
-
-	m.tun, tunName, err = ip.OpenTun("flannel%d")
-	if err != nil {
-		return fmt.Errorf("Failed to open TUN device: %v", err)
-	}
-
-	err = configureIface(tunName, m.tunNet, m.mtu)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func configureIface(ifname string, ipn ip.IP4Net, mtu int) error {
-	iface, err := netlink.LinkByName(ifname)
-	if err != nil {
-		return fmt.Errorf("failed to lookup interface %v", ifname)
-	}
-
-	err = netlink.AddrAdd(iface, &netlink.Addr{ipn.ToIPNet(), ""})
-	if err != nil {
-		return fmt.Errorf("failed to add IP address %v to %v: %v", ipn.String(), ifname, err)
-	}
-
-	err = netlink.LinkSetMTU(iface, mtu)
-	if err != nil {
-		return fmt.Errorf("failed to set MTU for %v: %v", ifname, err)
-	}
-
-	err = netlink.LinkSetUp(iface)
-	if err != nil {
-		return fmt.Errorf("failed to set interface %v to UP state: %v", ifname, err)
-	}
-
-	// explicitly add a route since there might be a route for a subnet already
-	// installed by Docker and then it won't get auto added
-	err = netlink.RouteAdd(&netlink.Route{
-		LinkIndex: iface.Attrs().Index,
-		Scope:     netlink.SCOPE_UNIVERSE,
-		Dst:       ipn.Network().ToIPNet(),
-	})
-	if err != nil && err != syscall.EEXIST {
-		return fmt.Errorf("Failed to add route (%v -> %v): %v", ipn.Network().String(), ifname, err)
-	}
-
-	return nil
-}
-
-func (m *UdpBackend) processSubnetEvents(batch []subnet.Event) {
-	for _, evt := range batch {
-		switch evt.Type {
-		case subnet.EventAdded:
-			log.Info("Subnet added: ", evt.Lease.Subnet)
-
-			setRoute(m.ctl, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, m.cfg.Port)
-
-		case subnet.EventRemoved:
-			log.Info("Subnet removed: ", evt.Lease.Subnet)
-
-			removeRoute(m.ctl, evt.Lease.Subnet)
-
-		default:
-			log.Error("Internal error: unknown event type: ", int(evt.Type))
-		}
-	}
+func (_ *UdpBackend) Run(ctx context.Context) {
+	<-ctx.Done()
 }

backend/vxlan/network.go

@@ -0,0 +1,238 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vxlan
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
+	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+)
+
+type network struct {
+	backend.SimpleNetwork
+	name     string
+	extIface *backend.ExternalInterface
+	dev      *vxlanDevice
+	rts      routes
+	sm       subnet.Manager
+}
+
+func newNetwork(name string, sm subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, nw ip.IP4Net, l *subnet.Lease) (*network, error) {
+	n := &network{
+		SimpleNetwork: backend.SimpleNetwork{
+			SubnetLease: l,
+			ExtIface:    extIface,
+		},
+		name: name,
+		sm:   sm,
+		dev:  dev,
+	}
+
+	return n, nil
+}
+
+func (n *network) Run(ctx context.Context) {
+	log.Info("Watching for L3 misses")
+	misses := make(chan *netlink.Neigh, 100)
+	// Unfrtunately MonitorMisses does not take a cancel channel
+	// as there's no wait to interrupt netlink socket recv
+	go n.dev.MonitorMisses(misses)
+
+	wg := sync.WaitGroup{}
+
+	log.Info("Watching for new subnet leases")
+	evts := make(chan []subnet.Event)
+	wg.Add(1)
+	go func() {
+		subnet.WatchLeases(ctx, n.sm, n.name, n.SubnetLease, evts)
+		log.Info("WatchLeases exited")
+		wg.Done()
+	}()
+
+	defer wg.Wait()
+	initialEvtsBatch := <-evts
+	for {
+		err := n.handleInitialSubnetEvents(initialEvtsBatch)
+		if err == nil {
+			break
+		}
+		log.Error(err, " About to retry")
+		time.Sleep(time.Second)
+	}
+
+	for {
+		select {
+		case miss := <-misses:
+			n.handleMiss(miss)
+
+		case evtBatch := <-evts:
+			n.handleSubnetEvents(evtBatch)
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (n *network) MTU() int {
+	return n.dev.MTU()
+}
+
+type vxlanLeaseAttrs struct {
+	VtepMAC hardwareAddr
+}
+
+func (n *network) handleSubnetEvents(batch []subnet.Event) {
+	for _, evt := range batch {
+		switch evt.Type {
+		case subnet.EventAdded:
+			log.Info("Subnet added: ", evt.Lease.Subnet)
+
+			if evt.Lease.Attrs.BackendType != "vxlan" {
+				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			var attrs vxlanLeaseAttrs
+			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
+				log.Error("Error decoding subnet lease JSON: ", err)
+				continue
+			}
+			n.rts.set(evt.Lease.Subnet, net.HardwareAddr(attrs.VtepMAC))
+			n.dev.AddL2(neigh{IP: evt.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(attrs.VtepMAC)})
+
+		case subnet.EventRemoved:
+			log.Info("Subnet removed: ", evt.Lease.Subnet)
+
+			if evt.Lease.Attrs.BackendType != "vxlan" {
+				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			var attrs vxlanLeaseAttrs
+			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
+				log.Error("Error decoding subnet lease JSON: ", err)
+				continue
+			}
+
+			if len(attrs.VtepMAC) > 0 {
+				n.dev.DelL2(neigh{IP: evt.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(attrs.VtepMAC)})
+			}
+			n.rts.remove(evt.Lease.Subnet)
+
+		default:
+			log.Error("Internal error: unknown event type: ", int(evt.Type))
+		}
+	}
+}
+
+func (n *network) handleInitialSubnetEvents(batch []subnet.Event) error {
+	log.Infof("Handling initial subnet events")
+	fdbTable, err := n.dev.GetL2List()
+	if err != nil {
+		return fmt.Errorf("error fetching L2 table: %v", err)
+	}
+
+	for _, fdbEntry := range fdbTable {
+		log.Infof("fdb already populated with: %s %s ", fdbEntry.IP, fdbEntry.HardwareAddr)
+	}
+
+	evtMarker := make([]bool, len(batch))
+	leaseAttrsList := make([]vxlanLeaseAttrs, len(batch))
+	fdbEntryMarker := make([]bool, len(fdbTable))
+
+	for i, evt := range batch {
+		if evt.Lease.Attrs.BackendType != "vxlan" {
+			log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
+			evtMarker[i] = true
+			continue
+		}
+
+		if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &leaseAttrsList[i]); err != nil {
+			log.Error("Error decoding subnet lease JSON: ", err)
+			evtMarker[i] = true
+			continue
+		}
+
+		for j, fdbEntry := range fdbTable {
+			if evt.Lease.Attrs.PublicIP.ToIP().Equal(fdbEntry.IP) && bytes.Equal([]byte(leaseAttrsList[i].VtepMAC), []byte(fdbEntry.HardwareAddr)) {
+				evtMarker[i] = true
+				fdbEntryMarker[j] = true
+				break
+			}
+		}
+		n.rts.set(evt.Lease.Subnet, net.HardwareAddr(leaseAttrsList[i].VtepMAC))
+	}
+
+	for j, marker := range fdbEntryMarker {
+		if !marker && fdbTable[j].IP != nil {
+			err := n.dev.DelL2(neigh{IP: ip.FromIP(fdbTable[j].IP), MAC: fdbTable[j].HardwareAddr})
+			if err != nil {
+				log.Error("Delete L2 failed: ", err)
+			}
+		}
+	}
+
+	for i, marker := range evtMarker {
+		if !marker {
+			err := n.dev.AddL2(neigh{IP: batch[i].Lease.Attrs.PublicIP, MAC: net.HardwareAddr(leaseAttrsList[i].VtepMAC)})
+			if err != nil {
+				log.Error("Add L2 failed: ", err)
+			}
+
+		}
+	}
+	return nil
+}
+
+func (n *network) handleMiss(miss *netlink.Neigh) {
+	switch {
+	case len(miss.IP) == 0 && len(miss.HardwareAddr) == 0:
+		log.Info("Ignoring nil miss")
+
+	case len(miss.HardwareAddr) == 0:
+		n.handleL3Miss(miss)
+
+	default:
+		log.Infof("Ignoring not a miss: %v, %v", miss.HardwareAddr, miss.IP)
+	}
+}
+
+func (n *network) handleL3Miss(miss *netlink.Neigh) {
+	log.Infof("L3 miss: %v", miss.IP)
+
+	rt := n.rts.findByNetwork(ip.FromIP(miss.IP))
+	if rt == nil {
+		log.Infof("Route for %v not found", miss.IP)
+		return
+	}
+
+	if err := n.dev.AddL3(neigh{IP: ip.FromIP(miss.IP), MAC: rt.vtepMAC}); err != nil {
+		log.Errorf("AddL3 failed: %v", err)
+	} else {
+		log.Info("AddL3 succeeded")
+	}
+}

backend/vxlan/vxlan.go

@@ -15,15 +15,10 @@
 package vxlan
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"net"
-	"sync"
-	"time"
 
-	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
-	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
 	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
 
 	"github.com/coreos/flannel/backend"
@@ -31,35 +26,26 @@ import (
 	"github.com/coreos/flannel/subnet"
 )
 
+func init() {
+	backend.Register("vxlan", New)
+}
+
 const (
 	defaultVNI = 1
 )
 
 type VXLANBackend struct {
-	sm      subnet.Manager
-	network string
-	cfg     struct {
-		VNI  int
-		Port int
-	}
-	extIndex int
-	extIaddr net.IP
-	extEaddr net.IP
-	lease    *subnet.Lease
-	dev      *vxlanDevice
-	rts      routes
+	sm       subnet.Manager
+	extIface *backend.ExternalInterface
 }
 
-func New(sm subnet.Manager, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error) {
-	vb := &VXLANBackend{
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
+	be := &VXLANBackend{
 		sm:       sm,
-		extIndex: extIface.Index,
-		extIaddr: extIaddr,
-		extEaddr: extEaddr,
+		extIface: extIface,
 	}
-	vb.cfg.VNI = defaultVNI
 
-	return vb, nil
+	return be, nil
 }
 
 func newSubnetAttrs(extEaddr net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
@@ -75,39 +61,46 @@ func newSubnetAttrs(extEaddr net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs,
 	}, nil
 }
 
-func (vb *VXLANBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (*backend.SubnetDef, error) {
-	vb.network = network
+func (be *VXLANBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
 
+func (be *VXLANBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (backend.Network, error) {
 	// Parse our configuration
+	cfg := struct {
+		VNI  int
+		Port int
+	}{
+		VNI: defaultVNI,
+	}
+
 	if len(config.Backend) > 0 {
-		if err := json.Unmarshal(config.Backend, &vb.cfg); err != nil {
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
 			return nil, fmt.Errorf("error decoding VXLAN backend config: %v", err)
 		}
 	}
 
 	devAttrs := vxlanDeviceAttrs{
-		vni:       uint32(vb.cfg.VNI),
-		name:      fmt.Sprintf("flannel.%v", vb.cfg.VNI),
-		vtepIndex: vb.extIndex,
-		vtepAddr:  vb.extIaddr,
-		vtepPort:  vb.cfg.Port,
+		vni:       uint32(cfg.VNI),
+		name:      fmt.Sprintf("flannel.%v", cfg.VNI),
+		vtepIndex: be.extIface.Iface.Index,
+		vtepAddr:  be.extIface.IfaceAddr,
+		vtepPort:  cfg.Port,
 	}
 
-	var err error
-	vb.dev, err = newVXLANDevice(&devAttrs)
+	dev, err := newVXLANDevice(&devAttrs)
 	if err != nil {
 		return nil, err
 	}
 
-	sa, err := newSubnetAttrs(vb.extEaddr, vb.dev.MACAddr())
+	sa, err := newSubnetAttrs(be.extIface.ExtAddr, dev.MACAddr())
 	if err != nil {
 		return nil, err
 	}
 
-	l, err := vb.sm.AcquireLease(ctx, vb.network, sa)
+	l, err := be.sm.AcquireLease(ctx, network, sa)
 	switch err {
 	case nil:
-		vb.lease = l
 
 	case context.Canceled, context.DeadlineExceeded:
 		return nil, err
@@ -122,60 +115,11 @@ func (vb *VXLANBackend) RegisterNetwork(ctx context.Context, network string, con
 		IP:        l.Subnet.IP,
 		PrefixLen: config.Network.PrefixLen,
 	}
-	if err = vb.dev.Configure(vxlanNet); err != nil {
+	if err = dev.Configure(vxlanNet); err != nil {
 		return nil, err
 	}
 
-	return &backend.SubnetDef{
-		Lease: l,
-		MTU:   vb.dev.MTU(),
-	}, nil
-}
-
-func (vb *VXLANBackend) Run(ctx context.Context) {
-	log.Info("Watching for L3 misses")
-	misses := make(chan *netlink.Neigh, 100)
-	// Unfrtunately MonitorMisses does not take a cancel channel
-	// as there's no wait to interrupt netlink socket recv
-	go vb.dev.MonitorMisses(misses)
-
-	wg := sync.WaitGroup{}
-
-	log.Info("Watching for new subnet leases")
-	evts := make(chan []subnet.Event)
-	wg.Add(1)
-	go func() {
-		subnet.WatchLeases(ctx, vb.sm, vb.network, vb.lease, evts)
-		log.Info("WatchLeases exited")
-		wg.Done()
-	}()
-
-	defer wg.Wait()
-	initialEvtsBatch := <-evts
-	for {
-		err := vb.handleInitialSubnetEvents(initialEvtsBatch)
-		if err == nil {
-			break
-		}
-		log.Error(err, " About to retry")
-		time.Sleep(time.Second)
-	}
-
-	for {
-		select {
-		case miss := <-misses:
-			vb.handleMiss(miss)
-
-		case evtBatch := <-evts:
-			vb.handleSubnetEvents(evtBatch)
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (vb *VXLANBackend) UnregisterNetwork(ctx context.Context, name string) {
+	return newNetwork(network, be.sm, be.extIface, dev, vxlanNet, l)
 }
 
 // So we can make it JSON (un)marshalable
@@ -200,139 +144,3 @@ func (hw *hardwareAddr) UnmarshalJSON(b []byte) error {
 	*hw = hardwareAddr(mac)
 	return nil
 }
-
-type vxlanLeaseAttrs struct {
-	VtepMAC hardwareAddr
-}
-
-func (vb *VXLANBackend) handleSubnetEvents(batch []subnet.Event) {
-	for _, evt := range batch {
-		switch evt.Type {
-		case subnet.EventAdded:
-			log.Info("Subnet added: ", evt.Lease.Subnet)
-
-			if evt.Lease.Attrs.BackendType != "vxlan" {
-				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
-				continue
-			}
-
-			var attrs vxlanLeaseAttrs
-			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
-				log.Error("Error decoding subnet lease JSON: ", err)
-				continue
-			}
-			vb.rts.set(evt.Lease.Subnet, net.HardwareAddr(attrs.VtepMAC))
-			vb.dev.AddL2(neigh{IP: evt.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(attrs.VtepMAC)})
-
-		case subnet.EventRemoved:
-			log.Info("Subnet removed: ", evt.Lease.Subnet)
-
-			if evt.Lease.Attrs.BackendType != "vxlan" {
-				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
-				continue
-			}
-
-			var attrs vxlanLeaseAttrs
-			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
-				log.Error("Error decoding subnet lease JSON: ", err)
-				continue
-			}
-
-			if len(attrs.VtepMAC) > 0 {
-				vb.dev.DelL2(neigh{IP: evt.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(attrs.VtepMAC)})
-			}
-			vb.rts.remove(evt.Lease.Subnet)
-
-		default:
-			log.Error("Internal error: unknown event type: ", int(evt.Type))
-		}
-	}
-}
-
-func (vb *VXLANBackend) handleInitialSubnetEvents(batch []subnet.Event) error {
-	log.Infof("Handling initial subnet events")
-	fdbTable, err := vb.dev.GetL2List()
-	if err != nil {
-		return fmt.Errorf("Error fetching L2 table: %v", err)
-	}
-
-	for _, fdbEntry := range fdbTable {
-		log.Infof("fdb already populated with: %s %s ", fdbEntry.IP, fdbEntry.HardwareAddr)
-	}
-
-	evtMarker := make([]bool, len(batch))
-	leaseAttrsList := make([]vxlanLeaseAttrs, len(batch))
-	fdbEntryMarker := make([]bool, len(fdbTable))
-
-	for i, evt := range batch {
-		if evt.Lease.Attrs.BackendType != "vxlan" {
-			log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
-			evtMarker[i] = true
-			continue
-		}
-
-		if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &leaseAttrsList[i]); err != nil {
-			log.Error("Error decoding subnet lease JSON: ", err)
-			evtMarker[i] = true
-			continue
-		}
-
-		for j, fdbEntry := range fdbTable {
-			if evt.Lease.Attrs.PublicIP.ToIP().Equal(fdbEntry.IP) && bytes.Equal([]byte(leaseAttrsList[i].VtepMAC), []byte(fdbEntry.HardwareAddr)) {
-				evtMarker[i] = true
-				fdbEntryMarker[j] = true
-				break
-			}
-		}
-		vb.rts.set(evt.Lease.Subnet, net.HardwareAddr(leaseAttrsList[i].VtepMAC))
-	}
-
-	for j, marker := range fdbEntryMarker {
-		if !marker && fdbTable[j].IP != nil {
-			err := vb.dev.DelL2(neigh{IP: ip.FromIP(fdbTable[j].IP), MAC: fdbTable[j].HardwareAddr})
-			if err != nil {
-				log.Error("Delete L2 failed: ", err)
-			}
-		}
-	}
-
-	for i, marker := range evtMarker {
-		if !marker {
-			err := vb.dev.AddL2(neigh{IP: batch[i].Lease.Attrs.PublicIP, MAC: net.HardwareAddr(leaseAttrsList[i].VtepMAC)})
-			if err != nil {
-				log.Error("Add L2 failed: ", err)
-			}
-
-		}
-	}
-	return nil
-}
-
-func (vb *VXLANBackend) handleMiss(miss *netlink.Neigh) {
-	switch {
-	case len(miss.IP) == 0 && len(miss.HardwareAddr) == 0:
-		log.Info("Ignoring nil miss")
-
-	case len(miss.HardwareAddr) == 0:
-		vb.handleL3Miss(miss)
-
-	default:
-		log.Infof("Ignoring not a miss: %v, %v", miss.HardwareAddr, miss.IP)
-	}
-}
-
-func (vb *VXLANBackend) handleL3Miss(miss *netlink.Neigh) {
-	log.Infof("L3 miss: %v", miss.IP)
-
-	rt := vb.rts.findByNetwork(ip.FromIP(miss.IP))
-	if rt == nil {
-		log.Infof("Route for %v not found", miss.IP)
-		return
-	}
-
-	if err := vb.dev.AddL3(neigh{IP: ip.FromIP(miss.IP), MAC: rt.vtepMAC}); err != nil {
-		log.Errorf("AddL3 failed: %v", err)
-	} else {
-		log.Info("AddL3 succeeded")
-	}
-}

main.go

@@ -30,6 +30,14 @@ import (
 	"github.com/coreos/flannel/network"
 	"github.com/coreos/flannel/remote"
 	"github.com/coreos/flannel/subnet"
+
+	// Backends need to be imported for their init() to get executed and them to register
+	_ "github.com/coreos/flannel/backend/alloc"
+	_ "github.com/coreos/flannel/backend/awsvpc"
+	_ "github.com/coreos/flannel/backend/gce"
+	_ "github.com/coreos/flannel/backend/hostgw"
+	_ "github.com/coreos/flannel/backend/udp"
+	_ "github.com/coreos/flannel/backend/vxlan"
 )
 
 type CmdLineOpts struct {

network/backend.go

@@ -1,36 +0,0 @@
-package network
-
-import (
-	"fmt"
-	"net"
-	"strings"
-
-	"github.com/coreos/flannel/backend"
-	"github.com/coreos/flannel/backend/alloc"
-	"github.com/coreos/flannel/backend/awsvpc"
-	"github.com/coreos/flannel/backend/gce"
-	"github.com/coreos/flannel/backend/hostgw"
-	"github.com/coreos/flannel/backend/udp"
-	"github.com/coreos/flannel/backend/vxlan"
-	"github.com/coreos/flannel/subnet"
-)
-
-type beNewFunc func(sm subnet.Manager, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error)
-
-var backendMap = map[string]beNewFunc{
-	"udp":     udp.New,
-	"alloc":   alloc.New,
-	"host-gw": hostgw.New,
-	"vxlan":   vxlan.New,
-	"aws-vpc": awsvpc.New,
-	"gce":     gce.New,
-}
-
-func newBackend(sm subnet.Manager, backendType string, extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (backend.Backend, error) {
-	betype := strings.ToLower(backendType)
-	befunc, ok := backendMap[betype]
-	if !ok {
-		return nil, fmt.Errorf("unknown backend type")
-	}
-	return befunc(sm, extIface, extIaddr, extEaddr)
-}

network/ipmasq.go

@@ -24,36 +24,47 @@ import (
 	"github.com/coreos/flannel/pkg/ip"
 )
 
+func rules(ipn ip.IP4Net) [][]string {
+	n := ipn.String()
+
+	return [][]string{
+		// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0)
+		{"-s", n, "-d", n, "-j", "ACCEPT"},
+		// NAT if it's not multicast traffic
+		{"-s", n, "!", "-d", "224.0.0.0/4", "-j", "MASQUERADE"},
+		// Masquerade anything headed towards flannel from the host
+		{"!", "-s", n, "-d", n, "-j", "MASQUERADE"},
+	}
+}
+
 func setupIPMasq(ipn ip.IP4Net) error {
 	ipt, err := iptables.New()
 	if err != nil {
-		return fmt.Errorf("failed to setup IP Masquerade. iptables was not found")
+		return fmt.Errorf("failed to set up IP Masquerade. iptables was not found")
 	}
 
-	err = ipt.ClearChain("nat", "FLANNEL")
-	if err != nil {
-		return fmt.Errorf("Failed to create/clear FLANNEL chain in NAT table: %v", err)
+	for _, rule := range rules(ipn) {
+		log.Info("Adding iptables rule: ", strings.Join(rule, " "))
+		err = ipt.AppendUnique("nat", "POSTROUTING", rule...)
+		if err != nil {
+			return fmt.Errorf("failed to insert IP masquerade rule: %v", err)
+		}
 	}
 
-	rules := [][]string{
-		// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0)
-		{"FLANNEL", "-d", ipn.String(), "-j", "ACCEPT"},
-		// NAT if it's not multicast traffic
-		{"FLANNEL", "!", "-d", "224.0.0.0/4", "-j", "MASQUERADE"},
-		// This rule will take everything coming from overlay and send it to FLANNEL chain
-		{"POSTROUTING", "-s", ipn.String(), "-j", "FLANNEL"},
-		// Masquerade anything headed towards flannel from the host
-		{"POSTROUTING", "!", "-s", ipn.String(), "-d", ipn.String(), "-j", "MASQUERADE"},
-	}
+	return nil
+}
 
-	for _, rule := range rules {
-		log.Info("Adding iptables rule: ", strings.Join(rule, " "))
-		chain := rule[0]
-		args := rule[1:len(rule)]
+func teardownIPMasq(ipn ip.IP4Net) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return fmt.Errorf("failed to teardown IP Masquerade. iptables was not found")
+	}
 
-		err = ipt.AppendUnique("nat", chain, args...)
+	for _, rule := range rules(ipn) {
+		log.Info("Deleting iptables rule: ", strings.Join(rule, " "))
+		err = ipt.Delete("nat", "POSTROUTING", rule...)
 		if err != nil {
-			return fmt.Errorf("Failed to insert IP masquerade rule: %v", err)
+			return fmt.Errorf("failed to delete IP masquerade rule: %v", err)
 		}
 	}
 

network/manager.go

@@ -16,6 +16,7 @@
 package network
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"net"
@@ -43,6 +44,8 @@ type CmdLineOpts struct {
 	watchNetworks bool
 }
 
+var errAlreadyExists = errors.New("already exists")
+
 var opts CmdLineOpts
 
 func init() {
@@ -58,13 +61,13 @@ func init() {
 type Manager struct {
 	ctx             context.Context
 	sm              subnet.Manager
+	bm              backend.Manager
 	allowedNetworks map[string]bool
+	mux             sync.Mutex
 	networks        map[string]*Network
 	watch           bool
 	ipMasq          bool
-	extIface        *net.Interface
-	iaddr           net.IP
-	eaddr           net.IP
+	extIface        *backend.ExternalInterface
 }
 
 func (m *Manager) isNetAllowed(name string) bool {
@@ -81,41 +84,22 @@ func (m *Manager) isMultiNetwork() bool {
 }
 
 func NewNetworkManager(ctx context.Context, sm subnet.Manager) (*Manager, error) {
-	iface, iaddr, err := lookupExtIface(opts.iface)
+	extIface, err := lookupExtIface(opts.iface)
 	if err != nil {
 		return nil, err
 	}
 
-	if iface.MTU == 0 {
-		return nil, fmt.Errorf("Failed to determine MTU for %s interface", iaddr)
-	}
-
-	var eaddr net.IP
-
-	if len(opts.publicIP) > 0 {
-		eaddr = net.ParseIP(opts.publicIP)
-		if eaddr == nil {
-			return nil, fmt.Errorf("Invalid public IP address", opts.publicIP)
-		}
-	}
-
-	if eaddr == nil {
-		eaddr = iaddr
-	}
-
-	log.Infof("Using %s as external interface", iaddr)
-	log.Infof("Using %s as external endpoint", eaddr)
+	bm := backend.NewManager(ctx, sm, extIface)
 
 	manager := &Manager{
 		ctx:             ctx,
 		sm:              sm,
+		bm:              bm,
 		allowedNetworks: make(map[string]bool),
 		networks:        make(map[string]*Network),
 		watch:           opts.watchNetworks,
 		ipMasq:          opts.ipMasq,
-		extIface:        iface,
-		iaddr:           iaddr,
-		eaddr:           eaddr,
+		extIface:        extIface,
 	}
 
 	for _, name := range strings.Split(opts.networks, ",") {
@@ -133,17 +117,17 @@ func NewNetworkManager(ctx context.Context, sm subnet.Manager) (*Manager, error)
 
 		for _, n := range result.Snapshot {
 			if manager.isNetAllowed(n) {
-				manager.networks[n] = NewNetwork(ctx, sm, n, manager.ipMasq)
+				manager.networks[n] = NewNetwork(ctx, sm, bm, n, manager.ipMasq)
 			}
 		}
 	} else {
-		manager.networks[""] = NewNetwork(ctx, sm, "", manager.ipMasq)
+		manager.networks[""] = NewNetwork(ctx, sm, bm, "", manager.ipMasq)
 	}
 
 	return manager, nil
 }
 
-func lookupExtIface(ifname string) (*net.Interface, net.IP, error) {
+func lookupExtIface(ifname string) (*backend.ExternalInterface, error) {
 	var iface *net.Interface
 	var iaddr net.IP
 	var err error
@@ -152,32 +136,56 @@ func lookupExtIface(ifname string) (*net.Interface, net.IP, error) {
 		if iaddr = net.ParseIP(ifname); iaddr != nil {
 			iface, err = ip.GetInterfaceByIP(iaddr)
 			if err != nil {
-				return nil, nil, fmt.Errorf("Error looking up interface %s: %s", ifname, err)
+				return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
 			}
 		} else {
 			iface, err = net.InterfaceByName(ifname)
 			if err != nil {
-				return nil, nil, fmt.Errorf("Error looking up interface %s: %s", ifname, err)
+				return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
 			}
 		}
 	} else {
 		log.Info("Determining IP address of default interface")
 		if iface, err = ip.GetDefaultGatewayIface(); err != nil {
-			return nil, nil, fmt.Errorf("Failed to get default interface: %s", err)
+			return nil, fmt.Errorf("failed to get default interface: %s", err)
 		}
 	}
 
 	if iaddr == nil {
 		iaddr, err = ip.GetIfaceIP4Addr(iface)
 		if err != nil {
-			return nil, nil, fmt.Errorf("Failed to find IPv4 address for interface %s", iface.Name)
+			return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
+		}
+	}
+
+	if iface.MTU == 0 {
+		return nil, fmt.Errorf("failed to determine MTU for %s interface", iaddr)
+	}
+
+	var eaddr net.IP
+
+	if len(opts.publicIP) > 0 {
+		eaddr = net.ParseIP(opts.publicIP)
+		if eaddr == nil {
+			return nil, fmt.Errorf("invalid public IP address", opts.publicIP)
 		}
 	}
 
-	return iface, iaddr, nil
+	if eaddr == nil {
+		eaddr = iaddr
+	}
+
+	log.Infof("Using %s as external interface", iaddr)
+	log.Infof("Using %s as external endpoint", eaddr)
+
+	return &backend.ExternalInterface{
+		Iface:     iface,
+		IfaceAddr: iaddr,
+		ExtAddr:   eaddr,
+	}, nil
 }
 
-func writeSubnetFile(path string, nw ip.IP4Net, ipMasq bool, sd *backend.SubnetDef) error {
+func writeSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network) error {
 	dir, name := filepath.Split(path)
 	os.MkdirAll(dir, 0755)
 
@@ -189,12 +197,12 @@ func writeSubnetFile(path string, nw ip.IP4Net, ipMasq bool, sd *backend.SubnetD
 
 	// Write out the first usable IP by incrementing
 	// sn.IP by one
-	sn := sd.Lease.Subnet
+	sn := bn.Lease().Subnet
 	sn.IP += 1
 
 	fmt.Fprintf(f, "FLANNEL_NETWORK=%s\n", nw)
 	fmt.Fprintf(f, "FLANNEL_SUBNET=%s\n", sn)
-	fmt.Fprintf(f, "FLANNEL_MTU=%d\n", sd.MTU)
+	fmt.Fprintf(f, "FLANNEL_MTU=%d\n", bn.MTU())
 	_, err = fmt.Fprintf(f, "FLANNEL_IPMASQ=%v\n", ipMasq)
 	f.Close()
 	if err != nil {
@@ -206,31 +214,62 @@ func writeSubnetFile(path string, nw ip.IP4Net, ipMasq bool, sd *backend.SubnetD
 	return os.Rename(tempFile, path)
 }
 
-func (m *Manager) RunNetwork(net *Network) {
-	sn := net.Init(m.extIface, m.iaddr, m.eaddr)
-	if sn != nil {
+func (m *Manager) addNetwork(n *Network) error {
+	m.mux.Lock()
+	defer m.mux.Unlock()
+
+	if _, ok := m.networks[n.Name]; ok {
+		return errAlreadyExists
+	}
+	m.networks[n.Name] = n
+	return nil
+}
+
+func (m *Manager) delNetwork(n *Network) {
+	m.mux.Lock()
+	delete(m.networks, n.Name)
+	m.mux.Unlock()
+}
+
+func (m *Manager) getNetwork(netname string) (*Network, bool) {
+	m.mux.Lock()
+	n, ok := m.networks[netname]
+	m.mux.Unlock()
+
+	return n, ok
+}
+
+func (m *Manager) forEachNetwork(f func(n *Network)) {
+	m.mux.Lock()
+	for _, n := range m.networks {
+		f(n)
+	}
+	m.mux.Unlock()
+}
+
+func (m *Manager) runNetwork(n *Network) {
+	n.Run(m.extIface, func(bn backend.Network) {
 		if m.isMultiNetwork() {
-			path := filepath.Join(opts.subnetDir, net.Name) + ".env"
-			if err := writeSubnetFile(path, net.Config.Network, m.ipMasq, sn); err != nil {
-				log.Warningf("%v failed to write subnet file: %s", net.Name, err)
+			path := filepath.Join(opts.subnetDir, n.Name) + ".env"
+			if err := writeSubnetFile(path, n.Config.Network, m.ipMasq, bn); err != nil {
+				log.Warningf("%v failed to write subnet file: %s", n.Name, err)
 				return
 			}
 		} else {
-			if err := writeSubnetFile(opts.subnetFile, net.Config.Network, m.ipMasq, sn); err != nil {
-				log.Warningf("%v failed to write subnet file: %s", net.Name, err)
+			if err := writeSubnetFile(opts.subnetFile, n.Config.Network, m.ipMasq, bn); err != nil {
+				log.Warningf("%v failed to write subnet file: %s", n.Name, err)
 				return
 			}
 			daemon.SdNotify("READY=1")
 		}
+	})
 
-		log.Infof("Running network %v", net.Name)
-		net.Run()
-		log.Infof("%v exited", net.Name)
-	}
+	m.delNetwork(n)
 }
 
 func (m *Manager) watchNetworks() {
 	wg := sync.WaitGroup{}
+	defer wg.Wait()
 
 	events := make(chan []subnet.Event)
 	wg.Add(1)
@@ -244,65 +283,63 @@ func (m *Manager) watchNetworks() {
 	for {
 		select {
 		case <-m.ctx.Done():
-			break
+			return
 
 		case evtBatch := <-events:
 			for _, e := range evtBatch {
 				netname := e.Network
 				if !m.isNetAllowed(netname) {
+					log.Infof("Network %q is not allowed", netname)
 					continue
 				}
 
 				switch e.Type {
 				case subnet.EventAdded:
-					if _, ok := m.networks[netname]; ok {
+					n := NewNetwork(m.ctx, m.sm, m.bm, netname, m.ipMasq)
+					if err := m.addNetwork(n); err != nil {
+						log.Infof("Network %q: %v", netname, err)
 						continue
 					}
-					net := NewNetwork(m.ctx, m.sm, netname, m.ipMasq)
-					m.networks[netname] = net
+
+					log.Infof("Network added: %v", netname)
+
 					wg.Add(1)
 					go func() {
-						m.RunNetwork(net)
+						m.runNetwork(n)
 						wg.Done()
 					}()
 
 				case subnet.EventRemoved:
-					net, ok := m.networks[netname]
+					log.Infof("Network removed: %v", netname)
+
+					n, ok := m.getNetwork(netname)
 					if !ok {
 						log.Warningf("Network %v unknown; ignoring EventRemoved", netname)
 						continue
 					}
-					net.Cancel()
-					delete(m.networks, netname)
+					n.Cancel()
 				}
 			}
 		}
 	}
-
-	wg.Wait()
 }
 
 func (m *Manager) Run() {
 	wg := sync.WaitGroup{}
 
 	// Run existing networks
-	for _, net := range m.networks {
+	m.forEachNetwork(func(n *Network) {
 		wg.Add(1)
 		go func(n *Network) {
-			m.RunNetwork(n)
+			m.runNetwork(n)
 			wg.Done()
-		}(net)
-	}
+		}(n)
+	})
 
 	if opts.watchNetworks {
-		wg.Add(1)
-		go func() {
-			m.watchNetworks()
-			wg.Done()
-		}()
-	} else {
-		<-m.ctx.Done()
+		m.watchNetworks()
 	}
 
 	wg.Wait()
+	m.bm.Wait()
 }

network/network.go

@@ -15,7 +15,7 @@
 package network
 
 import (
-	"net"
+	"fmt"
 	"sync"
 	"time"
 
@@ -25,108 +25,114 @@ import (
 	"github.com/coreos/flannel/subnet"
 )
 
+const (
+	renewMargin = time.Hour
+)
+
+var backends map[string]backend.Backend
+
 type Network struct {
-	Name       string
-	Config     *subnet.Config
+	Name   string
+	Config *subnet.Config
+
 	ctx        context.Context
 	cancelFunc context.CancelFunc
-
-	sm     subnet.Manager
-	ipMasq bool
-	be     backend.Backend
-	lease  *subnet.Lease
+	sm         subnet.Manager
+	bm         backend.Manager
+	ipMasq     bool
+	bn         backend.Network
 }
 
-func NewNetwork(ctx context.Context, sm subnet.Manager, name string, ipMasq bool) *Network {
-	ctx, cancel := context.WithCancel(ctx)
+func NewNetwork(ctx context.Context, sm subnet.Manager, bm backend.Manager, name string, ipMasq bool) *Network {
+	ctx, cf := context.WithCancel(ctx)
 
 	return &Network{
 		Name:       name,
-		ctx:        ctx,
-		cancelFunc: cancel,
 		sm:         sm,
+		bm:         bm,
 		ipMasq:     ipMasq,
+		ctx:        ctx,
+		cancelFunc: cf,
 	}
 }
 
-func (n *Network) Init(iface *net.Interface, iaddr net.IP, eaddr net.IP) *backend.SubnetDef {
-	var be backend.Backend
-	var sn *backend.SubnetDef
+func wrapError(desc string, err error) error {
+	if err == context.Canceled {
+		return err
+	}
+	return fmt.Errorf("failed to %v: %v", desc, err)
+}
 
-	steps := []func() error{
-		func() (err error) {
-			n.Config, err = n.sm.GetNetworkConfig(n.ctx, n.Name)
-			if err != nil {
-				log.Error("Failed to retrieve network config: ", err)
-			}
-			return
-		},
-
-		func() (err error) {
-			be, err = newBackend(n.sm, n.Config.BackendType, iface, iaddr, eaddr)
-			if err != nil {
-				log.Errorf("Failed to create and initialize network %v (type %v): %v", n.Name, n.Config.BackendType, err)
-			} else {
-				n.be = be
-			}
-			return
-		},
-
-		func() (err error) {
-			sn, err = be.RegisterNetwork(n.ctx, n.Name, n.Config)
-			if err != nil {
-				log.Errorf("Failed register network %v (type %v): %v", n.Name, n.Config.BackendType, err)
-			} else {
-				n.lease = sn.Lease
-			}
-			return
-		},
-
-		func() (err error) {
-			if n.ipMasq {
-				flannelNet := n.Config.Network
-				if err = setupIPMasq(flannelNet); err != nil {
-					log.Errorf("Failed to set up IP Masquerade for network %v: %v", n.Name, err)
-				}
-			}
-			return
-		},
+func (n *Network) init() error {
+	var err error
+
+	n.Config, err = n.sm.GetNetworkConfig(n.ctx, n.Name)
+	if err != nil {
+		return wrapError("retrieve network config", err)
 	}
 
-	for _, s := range steps {
-		for {
-			if err := s(); err == nil {
-				break
-			}
+	be, err := n.bm.GetBackend(n.Config.BackendType)
+	if err != nil {
+		return wrapError("create and initialize network", err)
+	}
 
-			select {
-			case <-time.After(time.Second):
+	n.bn, err = be.RegisterNetwork(n.ctx, n.Name, n.Config)
+	if err != nil {
+		return wrapError("register network", err)
+	}
 
-			case <-n.ctx.Done():
-				return nil
-			}
+	if n.ipMasq {
+		err = setupIPMasq(n.Config.Network)
+		if err != nil {
+			return wrapError("set up IP Masquerade", err)
 		}
 	}
 
-	return sn
+	return nil
 }
 
-func (n *Network) Run() {
+func (n *Network) Run(extIface *backend.ExternalInterface, inited func(bn backend.Network)) {
 	wg := sync.WaitGroup{}
+
+For:
+	for {
+		err := n.init()
+		switch err {
+		case nil:
+			break For
+		case context.Canceled:
+			return
+		default:
+			log.Error(err)
+			select {
+			case <-n.ctx.Done():
+				return
+			case <-time.After(time.Second):
+			}
+		}
+	}
+
+	inited(n.bn)
+
 	wg.Add(1)
 	go func() {
-		n.be.Run(n.ctx)
+		n.bn.Run(n.ctx)
 		wg.Done()
 	}()
 
 	wg.Add(1)
 	go func() {
-		subnet.LeaseRenewer(n.ctx, n.sm, n.Name, n.lease)
+		subnet.LeaseRenewer(n.ctx, n.sm, n.Name, n.bn.Lease())
 		wg.Done()
 	}()
 
-	<-n.ctx.Done()
-	n.be.UnregisterNetwork(n.ctx, n.Name)
+	defer func() {
+		if n.ipMasq {
+			if err := teardownIPMasq(n.Config.Network); err != nil {
+				log.Errorf("Failed to tear down IP Masquerade for network %v: %v", n.Name, err)
+			}
+		}
+	}()
 
 	wg.Wait()
 }

subnet/etcd.go

@@ -59,14 +59,14 @@ func NewEtcdManager(config *EtcdConfig) (Manager, error) {
 	}
 	return &EtcdManager{
 		registry:     r,
-		networkRegex: regexp.MustCompile(config.Prefix + `/([^/]*)/config`),
+		networkRegex: regexp.MustCompile(config.Prefix + `/([^/]*)(/|/config)?$`),
 	}, nil
 }
 
 func newEtcdManager(r Registry) Manager {
 	return &EtcdManager{
 		registry:     r,
-		networkRegex: regexp.MustCompile(`/coreos.com/network/([^/]*)/config`),
+		networkRegex: regexp.MustCompile(`/coreos.com/network/([^/]*)(/|/config)?$`),
 	}
 }
 
@@ -196,16 +196,16 @@ func (m *EtcdManager) acquireLeaseOnce(ctx context.Context, network string, conf
 	return nil, errors.New("Max retries reached trying to acquire a subnet")
 }
 
-func parseSubnetKey(s string) (ip.IP4Net, error) {
+func parseSubnetKey(s string) *ip.IP4Net {
 	if parts := subnetRegex.FindStringSubmatch(s); len(parts) == 3 {
 		snIp := net.ParseIP(parts[1]).To4()
 		prefixLen, err := strconv.ParseUint(parts[2], 10, 5)
 		if snIp != nil && err == nil {
-			return ip.IP4Net{IP: ip.FromIP(snIp), PrefixLen: uint(prefixLen)}, nil
+			return &ip.IP4Net{IP: ip.FromIP(snIp), PrefixLen: uint(prefixLen)}
 		}
 	}
 
-	return ip.IP4Net{}, errors.New("Error parsing IP Subnet")
+	return nil
 }
 
 func (m *EtcdManager) allocateSubnet(config *Config, leases []Lease) (ip.IP4Net, error) {
@@ -242,8 +242,7 @@ func (m *EtcdManager) getLeases(ctx context.Context, network string) ([]Lease, u
 
 	if err == nil {
 		for _, node := range resp.Node.Nodes {
-			sn, err := parseSubnetKey(node.Key)
-			if err == nil {
+			if sn := parseSubnetKey(node.Key); sn != nil {
 				attrs := &LeaseAttrs{}
 				if err = json.Unmarshal([]byte(node.Value), attrs); err == nil {
 					exp := time.Time{}
@@ -252,7 +251,7 @@ func (m *EtcdManager) getLeases(ctx context.Context, network string) ([]Lease, u
 					}
 
 					lease := Lease{
-						Subnet:     sn,
+						Subnet:     *sn,
 						Attrs:      attrs,
 						Expiration: exp,
 					}
@@ -351,6 +350,7 @@ DoWatch:
 			nextIndex = resp.Node.ModifiedIndex
 			goto DoWatch
 		}
+
 		return result, err
 
 	case isIndexTooSmall(err):
@@ -368,9 +368,9 @@ func isIndexTooSmall(err error) bool {
 }
 
 func parseSubnetWatchResponse(resp *etcd.Response) (LeaseWatchResult, error) {
-	sn, err := parseSubnetKey(resp.Node.Key)
-	if err != nil {
-		return LeaseWatchResult{}, fmt.Errorf("error parsing subnet IP: %s", resp.Node.Key)
+	sn := parseSubnetKey(resp.Node.Key)
+	if sn == nil {
+		return LeaseWatchResult{}, fmt.Errorf("%v %q: not a subnet, skipping", resp.Action, resp.Node.Key)
 	}
 
 	evt := Event{}
@@ -379,7 +379,7 @@ func parseSubnetWatchResponse(resp *etcd.Response) (LeaseWatchResult, error) {
 	case "delete", "expire":
 		evt = Event{
 			EventRemoved,
-			Lease{Subnet: sn},
+			Lease{Subnet: *sn},
 			"",
 		}
 
@@ -398,7 +398,7 @@ func parseSubnetWatchResponse(resp *etcd.Response) (LeaseWatchResult, error) {
 		evt = Event{
 			EventAdded,
 			Lease{
-				Subnet:     sn,
+				Subnet:     *sn,
 				Attrs:      attrs,
 				Expiration: exp,
 			},
@@ -414,18 +414,17 @@ func parseSubnetWatchResponse(resp *etcd.Response) (LeaseWatchResult, error) {
 
 // Returns network name from config key (eg, /coreos.com/network/foobar/config),
 // if the 'config' key isn't present we don't consider the network valid
-func (m *EtcdManager) parseNetworkKey(s string) (string, error) {
-	if parts := m.networkRegex.FindStringSubmatch(s); len(parts) == 2 {
-		return parts[1], nil
+func (m *EtcdManager) parseNetworkKey(s string) (string, bool) {
+	if parts := m.networkRegex.FindStringSubmatch(s); len(parts) == 3 {
+		return parts[1], parts[2] != ""
 	}
 
-	return "", errors.New("Error parsing Network key")
+	return "", false
 }
 
 func (m *EtcdManager) parseNetworkWatchResponse(resp *etcd.Response) (NetworkWatchResult, error, bool) {
-	netname, err := m.parseNetworkKey(resp.Node.Key)
-	if err != nil {
-		// Ignore non .../<netname>/config keys; tell caller to try again
+	netname, isConfig := m.parseNetworkKey(resp.Node.Key)
+	if netname == "" {
 		return NetworkWatchResult{}, nil, true
 	}
 
@@ -440,6 +439,11 @@ func (m *EtcdManager) parseNetworkWatchResponse(resp *etcd.Response) (NetworkWat
 		}
 
 	default:
+		if !isConfig {
+			// Ignore non .../<netname>/config keys; tell caller to try again
+			return NetworkWatchResult{}, nil, true
+		}
+
 		_, err := ParseConfig(resp.Node.Value)
 		if err != nil {
 			return NetworkWatchResult{}, err, false
@@ -470,8 +474,8 @@ func (m *EtcdManager) getNetworks(ctx context.Context) ([]string, uint64, error)
 		for _, node := range resp.Node.Nodes {
 			// Look for '/config' on the child nodes
 			for _, child := range node.Nodes {
-				netname, err := m.parseNetworkKey(child.Key)
-				if err == nil {
+				netname, isConfig := m.parseNetworkKey(child.Key)
+				if isConfig {
 					networks = append(networks, netname)
 				}
 			}