Vendoring in bronze1man goStrongswanVici repo.

This commit adds the IPSEC backend.

IPSEC Mode: Tunnel
This means that the entire IP packet is encrypted. We encapsulate using
the host IP addresses therefore we can have a separate IPSEC backend
instead of integrating it with existing backends. Running in tunnel mode
has the added advantage of not leaking the container IPs.

Authentication Mechanism: Pre-shared key (PSK)
For authentication, we use a PSK, generated by flannel and distributed
through etcd which in turn uses client-side certificates for
authentication.

Internet key exchange (IKE): Charon
We use charon daemon from the strongSwan project to negotiate Security
Associations (SAs) between the endpoints. SAs define the security
attributes, used to enable secure communication between endpoints [1].
IKE has two phases, the first establishes a secure channel using the
Diffie-Hellman key exchange and the second phase uses this secure
channel to negotiate the security attributes like, for example the keys
used to encrypt traffic. We use, 4096 bit Diffie-Hellman, AES-256-CBC
for encryption and SHA-256 for integrity. Note IKEv1 is used and phase 1
is run in Main mode.

Configuration
The only configuration parameter required is whether UDP Encapsulation
should be enabled.

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/.travis.yml

@@ -0,0 +1,6 @@
+language: go
+go:
+ - release
+
+script:
+ - go test -v

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 bronze1man
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/README.md

@@ -0,0 +1,146 @@
+strongswan vici golang client
+=============================
+[![Build Status](https://travis-ci.org/bronze1man/goStrongswanVici.svg)](https://travis-ci.org/bronze1man/goStrongswanVici)
+[![GoDoc](https://godoc.org/github.com/bronze1man/goStrongswanVici?status.svg)](https://godoc.org/github.com/bronze1man/goStrongswanVici)
+[![docs examples](https://sourcegraph.com/api/repos/github.com/bronze1man/goStrongswanVici/badges/docs-examples.png)](https://sourcegraph.com/github.com/bronze1man/goStrongswanVici)
+[![Total views](https://sourcegraph.com/api/repos/github.com/bronze1man/goStrongswanVici/counters/views.png)](https://sourcegraph.com/github.com/bronze1man/goStrongswanVici)
+[![GitHub issues](https://img.shields.io/github/issues/bronze1man/goStrongswanVici.svg)](https://github.com/bronze1man/goStrongswanVici/issues)
+[![GitHub stars](https://img.shields.io/github/stars/bronze1man/goStrongswanVici.svg)](https://github.com/bronze1man/goStrongswanVici/stargazers)
+[![GitHub forks](https://img.shields.io/github/forks/bronze1man/goStrongswanVici.svg)](https://github.com/bronze1man/goStrongswanVici/network)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](https://github.com/bronze1man/goStrongswanVici/blob/master/LICENSE)
+
+a golang implement of strongswan vici plugin client.
+
+### document
+* http://godoc.org/github.com/bronze1man/goStrongswanVici
+* https://github.com/strongswan/strongswan/tree/master/src/libcharon/plugins/vici
+
+### Implemented command list
+* version()
+* list-sas()
+* terminate()
+* load-conn()
+* list-conns()
+* unload-conn()
+* load-shared()
+
+If you need some commands, but it is not here .you can implement yourself, and send a pull request to this project.
+
+### example
+```go
+package main
+
+import (
+	"fmt"
+	"github.com/bronze1man/goStrongswanVici"
+)
+
+func main(){
+    // create a client.
+	client, err := goStrongswanVici.NewClientConnFromDefaultSocket()
+	if err != nil {
+		panic(err)
+	}
+	defer client.Close()
+
+	// get strongswan version
+	v, err := client.Version()
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("%#v\n", v)
+
+	childConfMap := make(map[string]goStrongswanVici.ChildSAConf)
+        childSAConf := goStrongswanVici.ChildSAConf{
+                Local_ts:      []string{"10.10.59.0/24"},
+                Remote_ts:     []string{"10.10.40.0/24"},
+                ESPProposals:  []string{"aes256-sha256-modp2048"},
+                StartAction:   "trap",
+		CloseAction:   "restart",
+                Mode:          "tunnel",
+                ReqID:         "10",
+                RekeyTime:     "10m",
+                InstallPolicy: "no",
+        }
+        childConfMap["test-child-conn"] = childSAConf
+
+        localAuthConf := goStrongswanVici.AuthConf{
+                AuthMethod: "psk",
+        }
+        remoteAuthConf := goStrongswanVici.AuthConf{
+                AuthMethod: "psk",
+        }
+
+	ikeConfMap := make(map[string] goStrongswanVici.IKEConf)
+
+        ikeConf := goStrongswanVici.IKEConf{
+                LocalAddrs:  []string{"192.168.198.10"},
+                RemoteAddrs: []string{"192.168.198.11"},
+                Proposals:   []string{"aes256-sha256-modp2048"},
+                Version:     "1",
+                LocalAuth:   localAuthConf,
+                RemoteAuth:  remoteAuthConf,
+                Children:    childConfMap,
+                Encap:       "no",
+        }
+
+	ikeConfMap["test-connection"] = ikeConf
+
+	//load connenction information into strongswan
+        err = client.LoadConn(&ikeConfMap)
+        if err != nil {
+                fmt.Printf("error loading connection: %v")
+                panic(err)
+        }
+
+	sharedKey := &goStrongswanVici.Key{
+                Typ:    "IKE",
+                Data:   "this is the key",
+                Owners: []string{"192.168.198.10"}, //IP of the remote host
+        }
+
+	//load shared key into strongswan
+        err = client.LoadShared(sharedKey)
+        if err != nil {
+                fmt.Printf("error returned from loadsharedkey \n")
+                panic(err)
+        }
+
+	//list-conns 
+	connList, err := client.ListConns("")
+	if err != nil {
+		fmt.Printf("error list-conns: %v \n", err)
+	}
+
+	for _, connection := range connList {
+		fmt.Printf("connection map: %v", connection)
+	}	
+
+	// get all conns info from strongswan
+	connInfo, err := client.ListAllVpnConnInfo()
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("found %d connections. \n", len(connInfo))
+
+	//unload connection from strongswan
+	unloadConnReq := &goStrongswanVici.UnloadConnRequest{
+			Name: "test-connection",
+			}
+	err = client.UnloadConn(unloadConnReq)
+	if err != nil {
+		panic(error)
+	}
+
+	// kill all conns in strongswan
+	for _, info := range connInfo {
+		fmt.Printf("kill connection id %s\n", info.Uniqueid)
+		err = client.Terminate(&goStrongswanVici.TerminateRequest{
+			Ike_id: info.Uniqueid,
+		})
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+```

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/client.go

@@ -0,0 +1,79 @@
+package goStrongswanVici
+
+import (
+	"net"
+)
+
+type ClientOptions struct {
+	Network string
+	Addr    string
+	// Dialer creates new network connection and has priority over
+	// Network and Addr options.
+	Dialer func() (net.Conn, error)
+}
+
+type Client struct {
+	o ClientOptions
+}
+
+func NewClient(options ClientOptions) (client *Client) {
+	if options.Dialer == nil {
+		options.Dialer = func() (net.Conn, error) {
+			return net.Dial(options.Network, options.Addr)
+		}
+	}
+	return &Client{
+		o: options,
+	}
+}
+
+func NewClientFromDefaultSocket() (client *Client) {
+	return NewClient(ClientOptions{
+		Network: "unix",
+		Addr:    "/var/run/charon.vici",
+	})
+}
+
+func (c *Client) NewConn() (conn *ClientConn, err error) {
+	conn1, err := c.o.Dialer()
+	if err != nil {
+		return nil, err
+	}
+	return NewClientConn(conn1), nil
+}
+
+func (c *Client) ListSas(ike string, ike_id string) (sas []map[string]IkeSa, err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	return conn.ListSas(ike, ike_id)
+}
+
+func (c *Client) ListAllVpnConnInfo() (list []VpnConnInfo, err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	return conn.ListAllVpnConnInfo()
+}
+
+func (c *Client) Version() (out *Version, err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	return conn.Version()
+}
+
+func (c *Client) Terminate(r *TerminateRequest) (err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	return conn.Terminate(r)
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/clientConn.go

@@ -0,0 +1,132 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+	"io"
+	"net"
+)
+
+// This object is not thread safe.
+// if you want concurrent, you need create more clients.
+type ClientConn struct {
+	conn          net.Conn
+	responseChan  chan segment
+	eventHandlers map[string]func(response map[string]interface{})
+	lastError     error
+}
+
+func (c *ClientConn) Close() error {
+	close(c.responseChan)
+	c.lastError = io.ErrClosedPipe
+	return c.conn.Close()
+}
+
+func NewClientConn(conn net.Conn) (client *ClientConn) {
+	client = &ClientConn{
+		conn:          conn,
+		responseChan:  make(chan segment, 2),
+		eventHandlers: map[string]func(response map[string]interface{}){},
+	}
+	go client.readThread()
+	return client
+}
+
+// it dial from unix:///var/run/charon.vici
+func NewClientConnFromDefaultSocket() (client *ClientConn, err error) {
+	conn, err := net.Dial("unix", "/var/run/charon.vici")
+	if err != nil {
+		return
+	}
+	return NewClientConn(conn), nil
+}
+
+func (c *ClientConn) Request(apiname string, request map[string]interface{}) (response map[string]interface{}, err error) {
+	err = writeSegment(c.conn, segment{
+		typ:  stCMD_REQUEST,
+		name: apiname,
+		msg:  request,
+	})
+	if err != nil {
+		fmt.Printf("error writing segment \n")
+		return
+	}
+	outMsg := <-c.responseChan
+
+	if c.lastError != nil {
+		return nil, c.lastError
+	}
+	if outMsg.typ != stCMD_RESPONSE {
+		return nil, fmt.Errorf("[%s] response error %d", apiname, outMsg.typ)
+	}
+	return outMsg.msg, nil
+}
+
+func (c *ClientConn) RegisterEvent(name string, handler func(response map[string]interface{})) (err error) {
+	if c.eventHandlers[name] != nil {
+		return fmt.Errorf("[event %s] register a event twice.", name)
+	}
+	c.eventHandlers[name] = handler
+	err = writeSegment(c.conn, segment{
+		typ:  stEVENT_REGISTER,
+		name: name,
+	})
+	if err != nil {
+		delete(c.eventHandlers, name)
+		return
+	}
+	outMsg := <-c.responseChan
+	//fmt.Printf("registerEvent %#v\n", outMsg)
+	if c.lastError != nil {
+		delete(c.eventHandlers, name)
+		return c.lastError
+	}
+
+	if outMsg.typ != stEVENT_CONFIRM {
+		delete(c.eventHandlers, name)
+		return fmt.Errorf("[event %s] response error %d", name, outMsg.typ)
+	}
+	return nil
+}
+
+func (c *ClientConn) UnregisterEvent(name string) (err error) {
+	err = writeSegment(c.conn, segment{
+		typ:  stEVENT_UNREGISTER,
+		name: name,
+	})
+	if err != nil {
+		return
+	}
+	outMsg := <-c.responseChan
+	//fmt.Printf("UnregisterEvent %#v\n", outMsg)
+	if c.lastError != nil {
+		return c.lastError
+	}
+
+	if outMsg.typ != stEVENT_CONFIRM {
+		return fmt.Errorf("[event %s] response error %d", name, outMsg.typ)
+	}
+	delete(c.eventHandlers, name)
+	return nil
+}
+
+func (c *ClientConn) readThread() {
+	for {
+		outMsg, err := readSegment(c.conn)
+		if err != nil {
+			c.lastError = err
+			return
+		}
+		switch outMsg.typ {
+		case stCMD_RESPONSE, stEVENT_CONFIRM:
+			c.responseChan <- outMsg
+		case stEVENT:
+			handler := c.eventHandlers[outMsg.name]
+			if handler != nil {
+				handler(outMsg.msg)
+			}
+		default:
+			c.lastError = fmt.Errorf("[Client.readThread] unknow msg type %d", outMsg.typ)
+			return
+		}
+	}
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/doc.go

@@ -0,0 +1,5 @@
+/*
+a golang implement of strongswan vici plugin client.
+https://github.com/strongswan/strongswan/tree/master/src/libcharon/plugins/vici
+*/
+package goStrongswanVici

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/err.go

@@ -0,0 +1,24 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+func handlePanic(f func() error) (err error) {
+	defer func() {
+		r := recover()
+		//no panic
+		if r == nil {
+			return
+		}
+		//panic a error
+		if e, ok := r.(error); ok {
+			err = e
+			return
+		}
+		//panic another stuff
+		err = fmt.Errorf("%s", r)
+	}()
+	err = f()
+	return
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/err_test.go

@@ -0,0 +1,28 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestHandlePanic(ot *testing.T) {
+	err := handlePanic(func() error {
+		panic("1")
+	})
+	if err == nil {
+		panic("err==nil")
+	}
+	if err.Error() != "1" {
+		panic(`err.Error()!="1"`)
+	}
+
+	err = handlePanic(func() error {
+		return fmt.Errorf("%d", 2)
+	})
+	if err == nil {
+		panic("err==nil")
+	}
+	if err.Error() != "2" {
+		panic(`err.Error()!="2" ` + err.Error())
+	}
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/listConns.go

@@ -0,0 +1,51 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+func (c *ClientConn) ListConns(ike string) ([]map[string]IKEConf, error) {
+	conns := []map[string]IKEConf{}
+	var eventErr error
+	var err error
+
+	err = c.RegisterEvent("list-conn", func(response map[string]interface{}) {
+		fmt.Printf("Entered the response function \n")
+		conn := &map[string]IKEConf{}
+		err = ConvertFromGeneral(response, conn)
+		if err != nil {
+			fmt.Printf("error from convert from general\n")
+			eventErr = fmt.Errorf("list-conn event error: %v", err)
+			return
+		}
+		fmt.Printf("Converted from general \n")
+		conns = append(conns, *conn)
+		fmt.Printf("Appended to conn\n")
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error registering list-conn event: %v", err)
+	}
+
+	if eventErr != nil {
+		return nil, eventErr
+	}
+
+	reqMap := map[string]interface{}{}
+
+	if ike != "" {
+		reqMap["ike"] = ike
+	}
+
+	_, err = c.Request("list-conns", reqMap)
+	if err != nil {
+		return nil, fmt.Errorf("error requesting list-conns: %v", err)
+	}
+
+	err = c.UnregisterEvent("list-conn")
+	if err != nil {
+		return nil, fmt.Errorf("error unregistering list-conns event: %v", err)
+	}
+
+	return conns, nil
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/listSas.go

@@ -0,0 +1,171 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+	"strconv"
+)
+
+//from list-sa event
+type IkeSa struct {
+	Uniqueid        string               `json:"uniqueid"` //called ike_id in terminate() argument.
+	Version         string               `json:"version"`
+	State           string               `json:"state"` //had saw: ESTABLISHED
+	Local_host      string               `json:"local-host"`
+	Local_id        string               `json:"local-id"`
+	Remote_host     string               `json:"remote-host"`
+	Remote_id       string               `json:"remote-id"`
+	Remote_xauth_id string               `json:"remote-xauth-id"` //client username
+	Initiator       string               `json:"initiator"`
+	Initiator_spi   string               `json:"initiator-spi"`
+	Responder_spi   string               `json:"responder-spi"`
+	Encr_alg        string               `json:"encr-alg"`
+	Encr_keysize    string               `json:"encr-keysize"`
+	Integ_alg       string               `json:"integ-alg"`
+	Integ_keysize   string               `json:"integ-keysize"`
+	Prf_alg         string               `json:"prf-alg"`
+	Dh_group        string               `json:"dh-group"`
+	Established     string               `json:"established"`
+	Rekey_time      string               `json:"rekey-time"`
+	Reauth_time     string               `json:"reauth-time"`
+	Child_sas       map[string]Child_sas `json:"child-sas"` //key means child-sa-name(conn name in ipsec.conf)
+}
+
+type Child_sas struct {
+	Reqid         string `json:"reqid"`
+	State         string `json:"state"` //had saw: INSTALLED
+	Mode          string `json:"mode"`  //had saw: TUNNEL
+	Protocol      string `json:"protocol"`
+	Encap         string `json:"encap"`
+	Spi_in        string `json:"spi-in"`
+	Spi_out       string `json:"spi-out"`
+	Cpi_in        string `json:"cpi-in"`
+	Cpi_out       string `json:"cpi-out"`
+	Encr_alg      string `json:"encr-alg"`
+	Encr_keysize  string `json:"encr-keysize"`
+	Integ_alg     string `json:"integ-alg"`
+	Integ_keysize string `json:"integ-keysize"`
+	Prf_alg       string `json:"prf-alg"`
+	Dh_group      string `json:"dh-group"`
+	Esn           string `json:"esn"`
+	Bytes_in      string `json:"bytes-in"` //bytes into this machine
+	Packets_in    string `json:"packets-in"`
+	Use_in        string `json:"use-in"`
+	Bytes_out     string `json:"bytes-out"` // bytes out of this machine
+	Packets_out   string `json:"packets-out"`
+	Use_out       string `json:"use-out"`
+	Rekey_time    string `json:"rekey-time"`
+	Life_time     string `json:"life-time"`
+	Install_time  string `json:"install-time"`
+}
+
+func (s *Child_sas) GetBytesIn() uint64 {
+	num, err := strconv.ParseUint(s.Bytes_in, 10, 64)
+	if err != nil {
+		return 0
+	}
+	return num
+}
+
+func (s *Child_sas) GetBytesOut() uint64 {
+	num, err := strconv.ParseUint(s.Bytes_out, 10, 64)
+	if err != nil {
+		return 0
+	}
+	return num
+}
+
+// To be simple, list all clients that are connecting to this server .
+// A client is a sa.
+// Lists currently active IKE_SAs
+func (c *ClientConn) ListSas(ike string, ike_id string) (sas []map[string]IkeSa, err error) {
+	sas = []map[string]IkeSa{}
+	var eventErr error
+	//register event
+	err = c.RegisterEvent("list-sa", func(response map[string]interface{}) {
+		sa := &map[string]IkeSa{}
+		err = ConvertFromGeneral(response, sa)
+		if err != nil {
+			fmt.Printf("list-sa event error: %s\n", err)
+			eventErr = err
+			return
+		}
+		sas = append(sas, *sa)
+		//fmt.Printf("event %#v\n", response)
+	})
+	if err != nil {
+		return
+	}
+	if eventErr != nil {
+		return
+	}
+
+	inMap := map[string]interface{}{}
+	if ike != "" {
+		inMap["ike"] = ike
+	}
+	if ike_id != "" {
+		inMap["ike_id"] = ike_id
+	}
+	_, err = c.Request("list-sas", inMap)
+	if err != nil {
+		return
+	}
+	//fmt.Printf("request finish %#v\n", sas)
+	err = c.UnregisterEvent("list-sa")
+	if err != nil {
+		return
+	}
+	return
+}
+
+//a vpn conn in the strongswan server
+type VpnConnInfo struct {
+	IkeSa
+	Child_sas
+	IkeSaName   string //looks like conn name in ipsec.conf, content is same as ChildSaName
+	ChildSaName string //looks like conn name in ipsec.conf
+}
+
+func (c *VpnConnInfo) GuessUserName() string {
+	if c.Remote_xauth_id != "" {
+		return c.Remote_xauth_id
+	}
+	if c.Remote_id != "" {
+		return c.Remote_id
+	}
+	return ""
+}
+
+// a helper method to avoid complex data struct in ListSas
+// if it only have one child_sas ,it will put it into info.Child_sas
+func (c *ClientConn) ListAllVpnConnInfo() (list []VpnConnInfo, err error) {
+	sasList, err := c.ListSas("", "")
+	if err != nil {
+		return
+	}
+	list = make([]VpnConnInfo, len(sasList))
+	for i, sa := range sasList {
+		info := VpnConnInfo{}
+		if len(sa) != 1 {
+			fmt.Printf("[vici.ListAllVpnConnInfo] warning: len(sa)[%d]!=1\n", len(sa))
+		}
+		for ikeSaName, ikeSa := range sa {
+			info.IkeSaName = ikeSaName
+			info.IkeSa = ikeSa
+			//if len(ikeSa.Child_sas) != 1 {
+			//	fmt.Println("[vici.ListAllVpnConnInfo] warning: len(ikeSa.Child_sas)[%d]!=1", len(ikeSa.Child_sas))
+			//}
+			for childSaName, childSa := range ikeSa.Child_sas {
+				info.ChildSaName = childSaName
+				info.Child_sas = childSa
+				break
+			}
+			break
+		}
+		if len(info.IkeSa.Child_sas) == 1 {
+			info.IkeSa.Child_sas = nil
+		}
+		list[i] = info
+	}
+	return
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/loadConn.go

@@ -0,0 +1,55 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type Connection struct {
+	ConnConf map[string]IKEConf `json:"connections"`
+}
+
+type IKEConf struct {
+	LocalAddrs  []string `json:"local_addrs"`
+	RemoteAddrs []string `json:"remote_addrs"`
+	Proposals   []string `json:"proposals"` //aes128-sha256-modp1024
+	Version     string   `json:"version"`   //1 for ikev1, 0 for ikev1 & ikev2
+	Encap       string   `json:"encap"`     //yes,no
+	KeyingTries string   `json:"keyingtries"`
+	//	RekyTime   string                 `json:"rekey_time"`
+	LocalAuth  AuthConf               `json:"local"`
+	RemoteAuth AuthConf               `json:"remote"`
+	Children   map[string]ChildSAConf `json:"children"`
+}
+
+type AuthConf struct {
+	AuthMethod string `json:"auth"` //psk
+}
+
+type ChildSAConf struct {
+	Local_ts      []string `json:"local_ts"`
+	Remote_ts     []string `json:"remote_ts"`
+	ESPProposals  []string `json:"esp_proposals"` //aes128-sha1_modp1024
+	StartAction   string   `json:"start_action"`  //none,trap,start
+	CloseAction   string   `json:"close_action"`
+	ReqID         string   `json:"reqid"`
+	RekeyTime     string   `json:"rekey_time"`
+	Mode          string   `json:"mode"`
+	InstallPolicy string   `json:"policies"`
+}
+
+func (c *ClientConn) LoadConn(conn *map[string]IKEConf) error {
+	requestMap := &map[string]interface{}{}
+
+	err := ConvertToGeneral(conn, requestMap)
+
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+
+	}
+	msg, err := c.Request("load-conn", *requestMap)
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful LoadConn: %v", msg["success"])
+	}
+
+	return nil
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/loadShared.go

@@ -0,0 +1,28 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type Key struct {
+	Typ    string   `json:"type"`
+	Data   string   `json:"data"`
+	Owners []string `json:"owners"`
+}
+
+func (c *ClientConn) LoadShared(key *Key) error {
+	requestMap := &map[string]interface{}{}
+
+	err := ConvertToGeneral(key, requestMap)
+
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	msg, err := c.Request("load-shared", *requestMap)
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful loadSharedKey: %v", msg["success"])
+	}
+
+	return nil
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/marshal.go

@@ -0,0 +1,29 @@
+package goStrongswanVici
+
+import (
+	"encoding/json"
+)
+
+//concrete data type to general data type
+// concrete data type like *Version
+// general data type include map[string]interface{} []string string
+// TODO make it faster
+func ConvertToGeneral(concrete interface{}, general interface{}) (err error) {
+	b, err := json.Marshal(concrete)
+	if err != nil {
+		return
+	}
+	return json.Unmarshal(b, general)
+}
+
+// general data type to concrete data type
+// concrete data type like *Version
+// general data type include map[string]interface{} []string string
+// TODO make it faster
+func ConvertFromGeneral(general interface{}, concrete interface{}) (err error) {
+	b, err := json.Marshal(general)
+	if err != nil {
+		return
+	}
+	return json.Unmarshal(b, concrete)
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/msg.go

@@ -0,0 +1,342 @@
+package goStrongswanVici
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+)
+
+type segmentType byte
+
+const (
+	stCMD_REQUEST      segmentType = 0
+	stCMD_RESPONSE                 = 1
+	stCMD_UNKNOWN                  = 2
+	stEVENT_REGISTER               = 3
+	stEVENT_UNREGISTER             = 4
+	stEVENT_CONFIRM                = 5
+	stEVENT_UNKNOWN                = 6
+	stEVENT                        = 7
+)
+
+func (t segmentType) hasName() bool {
+	switch t {
+	case stCMD_REQUEST, stEVENT_REGISTER, stEVENT_UNREGISTER, stEVENT:
+		return true
+	}
+	return false
+}
+func (t segmentType) isValid() bool {
+	switch t {
+	case stCMD_REQUEST, stCMD_RESPONSE, stCMD_UNKNOWN, stEVENT_REGISTER,
+		stEVENT_UNREGISTER, stEVENT_CONFIRM, stEVENT_UNKNOWN, stEVENT:
+		return true
+	}
+	return false
+}
+
+func (t segmentType) hasMsg() bool {
+	switch t {
+	case stCMD_REQUEST, stCMD_RESPONSE, stEVENT:
+		return true
+	}
+	return false
+}
+
+type elementType byte
+
+const (
+	etSECTION_START elementType = 1
+	etSECTION_END               = 2
+	etKEY_VALUE                 = 3
+	etLIST_START                = 4
+	etLIST_ITEM                 = 5
+	etLIST_END                  = 6
+)
+
+type segment struct {
+	typ  segmentType
+	name string
+	msg  map[string]interface{}
+}
+
+//msg 在内部以下列3种类型表示(降低复杂度)
+// string
+// map[string]interface{}
+// []string
+func writeSegment(w io.Writer, msg segment) (err error) {
+	if !msg.typ.isValid() {
+		return fmt.Errorf("[writeSegment] msg.typ %d not defined", msg.typ)
+	}
+	buf := &bytes.Buffer{}
+	buf.WriteByte(byte(msg.typ))
+	//name
+	if msg.typ.hasName() {
+		err = writeString1(buf, msg.name)
+		if err != nil {
+			fmt.Printf("error returned from writeString1i \n")
+			return
+		}
+	}
+
+	if msg.typ.hasMsg() {
+		err = writeMap(buf, msg.msg)
+		if err != nil {
+			fmt.Printf("error retruned from writeMap \n")
+			return
+		}
+	}
+
+	//写长度
+	err = binary.Write(w, binary.BigEndian, uint32(buf.Len()))
+	if err != nil {
+		fmt.Printf("[writeSegment] error writing to binary \n")
+		return
+	}
+
+	_, err = buf.WriteTo(w)
+	if err != nil {
+		fmt.Printf("[writeSegment] error writing to buffer \n")
+		return
+	}
+
+	return nil
+}
+
+func readSegment(inR io.Reader) (msg segment, err error) {
+	//长度
+	var length uint32
+	err = binary.Read(inR, binary.BigEndian, &length)
+	if err != nil {
+		return
+	}
+	r := bufio.NewReader(&io.LimitedReader{
+		R: inR,
+		N: int64(length),
+	})
+	//类型
+	c, err := r.ReadByte()
+	if err != nil {
+		return
+	}
+	msg.typ = segmentType(c)
+	if !msg.typ.isValid() {
+		return msg, fmt.Errorf("[readSegment] msg.typ %d not defined", msg.typ)
+	}
+	if msg.typ.hasName() {
+		msg.name, err = readString1(r)
+		if err != nil {
+			return
+		}
+	}
+	if msg.typ.hasMsg() {
+		msg.msg, err = readMap(r, true)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+//一个字节长度的字符串
+func writeString1(w *bytes.Buffer, s string) (err error) {
+	length := len(s)
+	if length > 255 {
+		return fmt.Errorf("[writeString1] length>255")
+	}
+	w.WriteByte(byte(length))
+	w.WriteString(s)
+	return
+}
+
+func readString1(r *bufio.Reader) (s string, err error) {
+	length, err := r.ReadByte()
+	if err != nil {
+		return
+	}
+	buf := make([]byte, length)
+	_, err = io.ReadFull(r, buf)
+	if err != nil {
+		return
+	}
+	return string(buf), nil
+}
+
+//两个字节长度的字符串
+func writeString2(w *bytes.Buffer, s string) (err error) {
+	length := len(s)
+	if length > 65535 {
+		return fmt.Errorf("[writeString2] length>65535")
+	}
+	binary.Write(w, binary.BigEndian, uint16(length))
+	w.WriteString(s)
+	return
+}
+
+func readString2(r io.Reader) (s string, err error) {
+	var length uint16
+	err = binary.Read(r, binary.BigEndian, &length)
+	if err != nil {
+		return
+	}
+	buf := make([]byte, length)
+	_, err = io.ReadFull(r, buf)
+	if err != nil {
+		return
+	}
+	return string(buf), nil
+}
+
+func writeKeyMap(w *bytes.Buffer, name string, msg map[string]interface{}) (err error) {
+	w.WriteByte(byte(etSECTION_START))
+	err = writeString1(w, name)
+	if err != nil {
+		return
+	}
+	writeMap(w, msg)
+	w.WriteByte(byte(etSECTION_END))
+	return nil
+}
+
+func writeKeyList(w *bytes.Buffer, name string, msg []string) (err error) {
+	w.WriteByte(byte(etLIST_START))
+	err = writeString1(w, name)
+	if err != nil {
+		return
+	}
+	for _, s := range msg {
+		w.WriteByte(byte(etLIST_ITEM))
+		err = writeString2(w, s)
+		if err != nil {
+			return
+		}
+	}
+	w.WriteByte(byte(etLIST_END))
+	return nil
+}
+
+func writeKeyString(w *bytes.Buffer, name string, msg string) (err error) {
+	w.WriteByte(byte(etKEY_VALUE))
+	err = writeString1(w, name)
+	if err != nil {
+		return
+	}
+	err = writeString2(w, msg)
+	return
+}
+
+func writeMap(w *bytes.Buffer, msg map[string]interface{}) (err error) {
+	for k, v := range msg {
+		switch t := v.(type) {
+		case map[string]interface{}:
+			writeKeyMap(w, k, t)
+		case []string:
+			writeKeyList(w, k, t)
+		case string:
+			writeKeyString(w, k, t)
+		case []interface{}:
+			str := make([]string, len(t))
+			for i := range t {
+				str[i] = t[i].(string)
+			}
+			writeKeyList(w, k, str)
+		default:
+			return fmt.Errorf("[writeMap] can not write type %T right now", msg)
+		}
+	}
+	return nil
+}
+
+//SECTION_START has been read already.
+func readKeyMap(r *bufio.Reader) (key string, msg map[string]interface{}, err error) {
+	key, err = readString1(r)
+	if err != nil {
+		return
+	}
+	msg, err = readMap(r, false)
+	return
+}
+
+//LIST_START has been read already.
+func readKeyList(r *bufio.Reader) (key string, msg []string, err error) {
+	key, err = readString1(r)
+	if err != nil {
+		return
+	}
+	msg = []string{}
+	for {
+		var c byte
+		c, err = r.ReadByte()
+		if err != nil {
+			return
+		}
+		switch elementType(c) {
+		case etLIST_ITEM:
+			value, err := readString2(r)
+			if err != nil {
+				return "", nil, err
+			}
+			msg = append(msg, value)
+		case etLIST_END: //end of outer list
+			return key, msg, nil
+		default:
+			return "", nil, fmt.Errorf("[readKeyList] protocol error 2")
+		}
+	}
+	return
+}
+
+//KEY_VALUE has been read already.
+func readKeyString(r *bufio.Reader) (key string, msg string, err error) {
+	key, err = readString1(r)
+	if err != nil {
+		return
+	}
+	msg, err = readString2(r)
+	if err != nil {
+		return
+	}
+	return
+}
+
+//SECTION_START has been read already.
+func readMap(r *bufio.Reader, isRoot bool) (msg map[string]interface{}, err error) {
+	msg = map[string]interface{}{}
+	for {
+		c, err := r.ReadByte()
+		if err == io.EOF && isRoot { //may be root section
+			return msg, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		switch elementType(c) {
+		case etSECTION_START:
+			key, value, err := readKeyMap(r)
+			if err != nil {
+				return nil, err
+			}
+			msg[key] = value
+		case etLIST_START:
+			key, value, err := readKeyList(r)
+			if err != nil {
+				return nil, err
+			}
+			msg[key] = value
+		case etKEY_VALUE:
+			key, value, err := readKeyString(r)
+			if err != nil {
+				return nil, err
+			}
+			msg[key] = value
+		case etSECTION_END: //end of outer section
+			return msg, nil
+		default:
+			panic(fmt.Errorf("[readMap] protocol error 1, %d %#v", c, msg))
+			//return nil, fmt.Errorf("[readMap] protocol error 1, %d",c)
+		}
+	}
+	return
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/msg_test.go

@@ -0,0 +1,110 @@
+package goStrongswanVici
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestMsg(ot *testing.T) {
+	for _, msg := range []map[string]interface{}{
+		map[string]interface{}{
+			"a": "1",
+		},
+		map[string]interface{}{
+			"a": []string{
+				"1", "2",
+			},
+		},
+		map[string]interface{}{
+			"a": map[string]interface{}{
+				"d": "e",
+				"e": []string{
+					"1", "2",
+				},
+			},
+		},
+		map[string]interface{}{
+			"a": []string{
+				"1", "2",
+			},
+			"b": "a",
+			"c": map[string]interface{}{
+				"d": "e",
+				"e": []string{
+					"1", "2",
+				},
+			},
+		},
+		map[string]interface{}{
+			"key1": "value1",
+			"section1": map[string]interface{}{
+				"sub-section": map[string]interface{}{
+					"key2": "value2",
+				},
+				"list1": []string{"item1", "item2"},
+			},
+		},
+	} {
+		buf := &bytes.Buffer{}
+		in := segment{
+			typ:  stCMD_REQUEST,
+			name: "good",
+			msg:  msg,
+		}
+		err := writeSegment(buf, in)
+		mustNotError(err)
+		content := buf.Bytes()
+		out, err := readSegment(buf)
+		mustNotError(err)
+		//fmt.Println(content)
+		if !reflect.DeepEqual(in, out) {
+			in1, err := json.Marshal(in.msg)
+			mustNotError(err)
+			out1, err := json.Marshal(out.msg)
+			mustNotError(err)
+			fmt.Println(content)
+			panic("!reflect.DeepEqual(in,out)\n" + string(in1) + "\n" + string(out1))
+		}
+	}
+
+	content := []byte{
+		0x0, 0x0, 0x0, 0x5e, //length 94
+		0x1,                                     //CMD_RESPONSE
+		0x3,                                     //KEY_VALUE
+		0x6, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, //daemon
+		0x0, 0x6, 0x63, 0x68, 0x61, 0x72, 0x6f, 0x6e, //charon
+		0x3, 0x7, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x0, 0x5, 0x35, 0x2e, 0x32, 0x2e, 0x32,
+		0x3, 0x7, 0x73, 0x79, 0x73, 0x6e, 0x61, 0x6d, 0x65, 0x0, 0x5, 0x4c, 0x69, 0x6e, 0x75, 0x78, 0x3, 0x7, 0x72,
+		0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x0, 0x11, 0x33, 0x2e, 0x31, 0x33, 0x2e, 0x30, 0x2d, 0x34, 0x34, 0x2d,
+		0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x3, 0x7, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x0, 0x6, 0x78,
+		0x38, 0x36, 0x5f, 0x36, 0x34}
+	buf := bytes.NewBuffer(content)
+	out, err := readSegment(buf)
+	mustNotError(err)
+
+	in := segment{
+		typ: stCMD_RESPONSE,
+		msg: map[string]interface{}{
+			"daemon":  "charon",
+			"machine": "x86_64",
+			"release": "3.13.0-44-generic",
+			"sysname": "Linux",
+			"version": "5.2.2",
+		},
+	}
+	if !reflect.DeepEqual(in, out) {
+		in1, err := json.Marshal(in.msg)
+		mustNotError(err)
+		out1, err := json.Marshal(out.msg)
+		mustNotError(err)
+		panic("!reflect.DeepEqual(in,out)\n" + string(in1) + "\n" + string(out1))
+	}
+}
+func mustNotError(err error) {
+	if err != nil {
+		panic(err)
+	}
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/terminate.go

@@ -0,0 +1,32 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type TerminateRequest struct {
+	Child    string `json:"child,omitempty"`
+	Ike      string `json:"ike,omitempty"`
+	Child_id string `json:"child-id,omitempty"`
+	Ike_id   string `json:"ike-id,omitempty"`
+	Timeout  string `json:"timeout,omitempty"`
+	Loglevel string `json:"loglevel,omitempty"`
+}
+
+// To be simple, kill a client that is connecting to this server. A client is a sa.
+//Terminates an SA while streaming control-log events.
+func (c *ClientConn) Terminate(r *TerminateRequest) (err error) {
+	err = handlePanic(func() (err error) {
+		reqMap := &map[string]interface{}{}
+		ConvertToGeneral(r, reqMap)
+		msg, err := c.Request("terminate", *reqMap)
+		if err != nil {
+			return
+		}
+		if msg["success"] != "yes" {
+			return fmt.Errorf("[Terminate] %s", msg["errmsg"])
+		}
+		return
+	})
+	return
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/unloadConn.go

@@ -0,0 +1,24 @@
+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type UnloadConnRequest struct {
+	Name string `json:"name"`
+}
+
+func (c *ClientConn) UnloadConn(r *UnloadConnRequest) error {
+	reqMap := &map[string]interface{}{}
+	ConvertToGeneral(r, reqMap)
+	msg, err := c.Request("unload-conn", *reqMap)
+	if err != nil {
+		return err
+	}
+
+	if msg["success"] != "yes" {
+		return fmt.Errorf("[Unload-Connection] %s", msg["errmsg"])
+	}
+
+	return nil
+}

Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici/version.go

@@ -0,0 +1,19 @@
+package goStrongswanVici
+
+type Version struct {
+	Daemon  string `json:"daemon"`
+	Version string `json:"version"`
+	Sysname string `json:"sysname"`
+	Release string `json:"release"`
+	Machine string `json:"machine"`
+}
+
+func (c *ClientConn) Version() (out *Version, err error) {
+	msg, err := c.Request("version", nil)
+	if err != nil {
+		return
+	}
+	out = &Version{}
+	err = ConvertFromGeneral(msg, out)
+	return
+}

README.md

@@ -25,7 +25,7 @@ Flannel is focused on networking. For network policy, other projects such as [Ca
 
 The easiest way to deploy flannel with Kubernetes is to use one of several deployment tools and distributions that network clusters with flannel by default. For example, CoreOS's [Tectonic][tectonic] sets up flannel in the Kubernetes clusters it creates using the open source [Tectonic Installer][tectonic-installer] to drive the setup process.
 
-Though not required, it's recommended that flannel uses the Kubernetes API as its backing store which avoids the need to deploy a discrete `etcd` cluster for `flannel`. This `flannel` mode is known as the *kube subnet manager*.
+Though not required , it's recommended that flannel uses the Kubernetes API as its backing store which avoids the need to deploy a discrete `etcd` cluster for `flannel `. This `flannel`mode is known as the *kube subnet manager*.
 
 ### Deploying flannel manually
 
@@ -65,9 +65,10 @@ See [CONTRIBUTING][contributing] for details on submitting patches and the contr
 
 See [reporting bugs][reporting] for details about reporting any issues.
 
-## License
-
-Flannel is under the Apache 2.0 license. See the [LICENSE][license] file for details.
+  ## License
+   
+ 
+  Flannel is under the Apache 2.0 license. See the [LICENSE][license] file for details. 
 
 [calico]: http://www.projectcalico.org
 [pod-cidr]: https://kubernetes.io/docs/admin/kubelet/

backend/ipsec/handle_charon.go

@@ -0,0 +1,183 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipsec
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"syscall"
+	"time"
+
+	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/bronze1man/goStrongswanVici"
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/subnet"
+)
+
+type CharonIKEDaemon struct {
+	path string
+}
+
+func NewCharonIKEDaemon(charonPath string) (*CharonIKEDaemon, error) {
+	path, err := exec.LookPath(charonPath)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info("Launching IKE charon path: ", path)
+
+	return &CharonIKEDaemon{
+		path: path,
+	}, nil
+}
+
+func (charon *CharonIKEDaemon) Run() error {
+	cmd := exec.Cmd{
+		Path: charon.path,
+		SysProcAttr: &syscall.SysProcAttr{
+			Pdeathsig: syscall.SIGTERM,
+		},
+	}
+
+	cmd.Stderr = os.Stderr
+
+	return cmd.Run()
+}
+
+func (charon *CharonIKEDaemon) LoadSharedKey(remotePublicIP, password string) error {
+	var err error
+	var client *goStrongswanVici.ClientConn
+
+	for {
+		client, err = goStrongswanVici.NewClientConnFromDefaultSocket()
+		if err == nil {
+			break
+		} else {
+			log.Error("ClientConnection failed: ", err)
+			log.Infof("Retrying in 1 second ...")
+			time.Sleep(1 * time.Second)
+		}
+	}
+
+	defer client.Close()
+
+	sharedKey := &goStrongswanVici.Key{
+		Typ:    "IKE",
+		Data:   password,
+		Owners: []string{remotePublicIP},
+	}
+
+	err = client.LoadShared(sharedKey)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Loaded shared key for: %v", remotePublicIP)
+	return nil
+}
+
+func (charon *CharonIKEDaemon) LoadConnection(localLease, remoteLease *subnet.Lease, reqID, encap string) error {
+	var err error
+	var client *goStrongswanVici.ClientConn
+
+	for {
+		client, err = goStrongswanVici.NewClientConnFromDefaultSocket()
+		if err == nil {
+			break
+		} else {
+			log.Info("ClientConnection failed: ", err)
+			log.Infof("Retying in 1 second ...")
+			time.Sleep(1 * time.Second)
+		}
+	}
+	defer client.Close()
+
+	childConfMap := make(map[string]goStrongswanVici.ChildSAConf)
+	childSAConf := goStrongswanVici.ChildSAConf{
+		Local_ts:     []string{localLease.Subnet.String()},
+		Remote_ts:    []string{remoteLease.Subnet.String()},
+		ESPProposals: []string{"aes256-sha256-modp4096"},
+		StartAction:  "start",
+		CloseAction:  "trap",
+		Mode:         "tunnel",
+		ReqID:        reqID,
+		//		RekeyTime:     rekeyTime,
+		InstallPolicy: "no",
+	}
+
+	childSAConfName := formatChildSAConfName(localLease, remoteLease)
+
+	childConfMap[childSAConfName] = childSAConf
+
+	localAuthConf := goStrongswanVici.AuthConf{
+		AuthMethod: "psk",
+	}
+	remoteAuthConf := goStrongswanVici.AuthConf{
+		AuthMethod: "psk",
+	}
+
+	ikeConf := goStrongswanVici.IKEConf{
+		LocalAddrs:  []string{localLease.Attrs.PublicIP.String()},
+		RemoteAddrs: []string{remoteLease.Attrs.PublicIP.String()},
+		Proposals:   []string{"aes256-sha256-modp4096"},
+		Version:     "1",
+		KeyingTries: "0", //continues to retry
+		LocalAuth:   localAuthConf,
+		RemoteAuth:  remoteAuthConf,
+		Children:    childConfMap,
+		Encap:       encap,
+	}
+	ikeConfMap := make(map[string]goStrongswanVici.IKEConf)
+
+	connectionName := formatConnectionName(localLease, remoteLease)
+	ikeConfMap[connectionName] = ikeConf
+
+	err = client.LoadConn(&ikeConfMap)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Loaded connection: %v", connectionName)
+	return nil
+}
+
+func (charon *CharonIKEDaemon) UnloadCharonConnection(localLease, remoteLease *subnet.Lease) error {
+	client, err := goStrongswanVici.NewClientConnFromDefaultSocket()
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	connectionName := formatConnectionName(localLease, remoteLease)
+	unloadConnRequest := &goStrongswanVici.UnloadConnRequest{
+		Name: connectionName,
+	}
+
+	err = client.UnloadConn(unloadConnRequest)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Unloaded connection: %v", connectionName)
+	return nil
+}
+
+func formatConnectionName(localLease, remoteLease *subnet.Lease) string {
+	return fmt.Sprintf("%s-%s-%s-%s", localLease.Attrs.PublicIP, localLease.Subnet, remoteLease.Subnet, remoteLease.Attrs.PublicIP)
+}
+
+func formatChildSAConfName(localLease, remoteLease *subnet.Lease) string {
+	return fmt.Sprintf("%s-%s", localLease.Subnet, remoteLease.Subnet)
+}

backend/ipsec/handle_xfrm.go

@@ -0,0 +1,95 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipsec
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
+
+	"github.com/coreos/flannel/subnet"
+)
+
+func AddXFRMPolicy(myLease, remoteLease *subnet.Lease, dir netlink.Dir, reqID int) error {
+	src := myLease.Subnet.ToIPNet()
+
+	dst := remoteLease.Subnet.ToIPNet()
+
+	policy := netlink.XfrmPolicy{
+		Src: src,
+		Dst: dst,
+		Dir: dir,
+	}
+
+	tunnelLeft := myLease.Attrs.PublicIP.ToIP()
+	tunnelRight := remoteLease.Attrs.PublicIP.ToIP()
+
+	tmpl := netlink.XfrmPolicyTmpl{
+		Src:   tunnelLeft,
+		Dst:   tunnelRight,
+		Proto: netlink.XFRM_PROTO_ESP,
+		Mode:  netlink.XFRM_MODE_TUNNEL,
+		Reqid: reqID,
+	}
+
+	log.Infof("Adding ipsec policy: %+v", tmpl)
+
+	policy.Tmpls = append(policy.Tmpls, tmpl)
+
+	if err := netlink.XfrmPolicyAdd(&policy); err != nil {
+		return fmt.Errorf("error adding policy: %+v err: %v", policy, err)
+	}
+
+	return nil
+}
+
+func DeleteXFRMPolicy(localSubnet, remoteSubnet *net.IPNet, localPublicIP, remotePublicIP net.IP, dir netlink.Dir, reqID int) error {
+	src := localSubnet
+	dst := remoteSubnet
+
+	policy := netlink.XfrmPolicy{
+		Src: src,
+		Dst: dst,
+		Dir: dir,
+	}
+
+	tunnelLeft := localPublicIP
+	tunnelRight := remotePublicIP
+
+	tmpl := netlink.XfrmPolicyTmpl{
+		Src:   tunnelLeft,
+		Dst:   tunnelRight,
+		Proto: netlink.XFRM_PROTO_ESP,
+		Mode:  netlink.XFRM_MODE_TUNNEL,
+		Reqid: reqID,
+	}
+
+	log.Infof("Deleting ipsec policy: %+v", tmpl)
+
+	policy.Tmpls = append(policy.Tmpls, tmpl)
+
+	if err := netlink.XfrmPolicyDel(&policy); err != nil {
+		return fmt.Errorf("error deleting policy: %+v err: %v", policy, err)
+	}
+
+	return nil
+}
+
+func GetIPSECPolicies() ([]netlink.XfrmPolicy, error) {
+	return netlink.XfrmPolicyList(syscall.AF_INET)
+}

backend/ipsec/ipsec.go

@@ -0,0 +1,137 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipsec
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+)
+
+var CharonPath string
+
+const (
+	defaultCharonPath = "/opt/flannel/libexec/ipsec/charon"
+	passwordLength    = 40
+)
+
+func init() {
+	backend.Register("ipsec", New)
+}
+
+type IPSECBackend struct {
+	sm       subnet.Manager
+	extIface *backend.ExternalInterface
+}
+
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
+	be := &IPSECBackend{
+		sm:       sm,
+		extIface: extIface,
+	}
+
+	return be, nil
+}
+
+func (be *IPSECBackend) RegisterNetwork(ctx context.Context, netname string, config *subnet.Config) (backend.Network, error) {
+	cfg := struct {
+		UDPEncap bool
+	}{}
+
+	if len(config.Backend) > 0 {
+		log.Info("i.config.backend length > 0")
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
+			return nil, fmt.Errorf("error decoding IPSEC backend config: %v", err)
+		}
+	}
+
+	attrs := subnet.LeaseAttrs{
+		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
+		BackendType: "ipsec",
+	}
+
+	l, err := be.sm.AcquireLease(ctx, netname, &attrs)
+
+	switch err {
+	case nil:
+
+	case context.Canceled, context.DeadlineExceeded:
+		return nil, err
+
+	default:
+		return nil, fmt.Errorf("failed to acquire lease: %v", err)
+	}
+
+	if CharonPath == "" {
+		CharonPath = defaultCharonPath
+	}
+
+	ikeDaemon, err := NewCharonIKEDaemon(CharonPath)
+	if err != nil {
+		return nil, fmt.Errorf("error creating CharonIKEDaemon struct: %v", err)
+	}
+
+	log.Info("UDPEncap: ", cfg.UDPEncap)
+
+	password, err := GenerateRandomString(passwordLength)
+	if err != nil {
+		return nil, fmt.Errorf("error generating random string: %v", err)
+	}
+
+	err = be.sm.CreateBackendData(ctx, netname, password)
+	if err != nil {
+		return nil, fmt.Errorf("error creating password: %v", err)
+	}
+
+	password, err = be.sm.GetBackendData(ctx, netname)
+	if err != nil {
+		return nil, fmt.Errorf("error getting password: %v", err)
+	}
+
+	return newNetwork(netname, be.sm, be.extIface, cfg.UDPEncap, password, ikeDaemon, l)
+}
+
+func (be *IPSECBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func generateRandomBytes(n int) ([]byte, error) {
+	b := make([]byte, n)
+	_, err := rand.Read(b)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func GenerateRandomString(s int) (string, error) {
+	b, err := generateRandomBytes(s)
+
+	if err != nil {
+		return "", err
+	}
+
+	return base64.StdEncoding.EncodeToString(b), nil
+}

backend/ipsec/network.go

@@ -0,0 +1,284 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipsec
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
+	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
+	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/subnet"
+)
+
+const (
+	/*
+	   New IP header (Tunnel Mode)   : 20
+	   SPI (ESP Header)              : 4
+	   Sequence (ESP Header)         : 4
+	   ESP-AES IV                    : 16
+	   ESP-AES Pad                   : 0-15
+	   Pad length (ESP Trailer)      : 1
+	   Next Header (ESP Trailer)     : 1
+	   ESP-SHA-256 ICV               : 16
+	*/
+	ipsecOverhead    = 77
+	udpEncapOverhead = 8
+	defaultReqID     = 11
+)
+
+type network struct {
+	backend.SimpleNetwork
+	name     string
+	password string
+	UDPEncap bool
+	sm       subnet.Manager
+	iked     *CharonIKEDaemon
+}
+
+func newNetwork(name string, sm subnet.Manager, extIface *backend.ExternalInterface, UDPEncap bool, password string, ikeDaemon *CharonIKEDaemon, l *subnet.Lease) (*network, error) {
+	n := &network{
+		SimpleNetwork: backend.SimpleNetwork{
+			SubnetLease: l,
+			ExtIface:    extIface,
+		},
+		name:     name,
+		sm:       sm,
+		iked:     ikeDaemon,
+		password: password,
+		UDPEncap: UDPEncap,
+	}
+
+	return n, nil
+}
+
+func (n *network) Run(ctx context.Context) {
+	wg := sync.WaitGroup{}
+	defer wg.Wait()
+
+	wg.Add(1)
+	go func() {
+		log.Info("Starting charon \n")
+		n.startIKEDaemon()
+		log.Info("Charon daemon exited")
+		wg.Done()
+	}()
+
+	log.Info("Watching for new subnet leases")
+
+	evts := make(chan []subnet.Event)
+
+	wg.Add(1)
+	go func() {
+		subnet.WatchLeases(ctx, n.sm, n.name, n.SubnetLease, evts)
+		log.Info("WatchLeases exited")
+		wg.Done()
+	}()
+
+	initialEvtsBatch := <-evts
+	for {
+		err := n.handleInitialSubnetEvents(initialEvtsBatch)
+		if err == nil {
+			break
+		}
+
+		log.Error(err, " Retrying")
+		time.Sleep(time.Second)
+	}
+
+	for {
+		select {
+		case evtsBatch := <-evts:
+			n.handleSubnetEvents(evtsBatch)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (n *network) handleInitialSubnetEvents(batch []subnet.Event) error {
+	log.Infof("Handling initial subnet events \n")
+
+	installedPolicies, err := GetIPSECPolicies()
+	if err != nil {
+		return fmt.Errorf("error getting ipsec policies: %v", err)
+	}
+
+	evtMarker := make([]bool, len(batch))
+	policyMarker := make([]bool, len(installedPolicies))
+
+	for k, evt := range batch {
+		if evt.Lease.Attrs.BackendType != "ipsec" {
+			log.Warningf("Ignoring non-ipsec subnet event type:%v", evt.Lease.Attrs.BackendType)
+			evtMarker[k] = true
+			continue
+		}
+
+		for j, policy := range installedPolicies {
+			if (policy.Src.String() == n.SubnetLease.Subnet.ToIPNet().String()) && (policy.Dst.String() == evt.Lease.Subnet.ToIPNet().String()) {
+				if policy.Dir != netlink.XFRM_DIR_OUT {
+					continue
+				}
+
+				if (policy.Tmpls[0].Src.Equal(n.SubnetLease.Attrs.PublicIP.ToIP())) && (policy.Tmpls[0].Dst.Equal(evt.Lease.Attrs.PublicIP.ToIP())) {
+					evtMarker[k] = true
+					policyMarker[j] = true
+				}
+			}
+		}
+	}
+
+	for k, marker := range evtMarker {
+		if !marker {
+			if err := n.AddIPSECPolicies(&batch[k].Lease, defaultReqID); err != nil {
+				log.Errorf("error adding initial ipsec policy: %v", err)
+			}
+		}
+	}
+
+	for _, evt := range batch {
+		if err := n.iked.LoadSharedKey(evt.Lease.Attrs.PublicIP.String(), n.password); err != nil {
+			log.Errorf("error loading initial shared key: %v", err)
+		}
+
+		if err := n.iked.LoadConnection(n.SubnetLease, &evt.Lease, strconv.Itoa(defaultReqID), strconv.FormatBool(n.UDPEncap)); err != nil {
+			log.Errorf("error loading initial connection into IKE daemon: %v", err)
+		}
+	}
+
+	for j, marker := range policyMarker {
+		if !marker {
+			if installedPolicies[j].Dir != netlink.XFRM_DIR_OUT {
+				continue
+			}
+
+			if err := n.DeleteIPSECPolicies(installedPolicies[j].Src, installedPolicies[j].Dst, installedPolicies[j].Tmpls[0].Src, installedPolicies[j].Tmpls[0].Dst, installedPolicies[j].Tmpls[0].Reqid); err != nil {
+				log.Errorf("error deleting installed policy")
+			}
+		}
+	}
+
+	return nil
+}
+
+func (n *network) handleSubnetEvents(batch []subnet.Event) {
+	for _, evt := range batch {
+		switch evt.Type {
+		case subnet.EventAdded:
+			log.Info("Subnet added: ", evt.Lease.Subnet)
+
+			if evt.Lease.Attrs.BackendType != "ipsec" {
+				log.Warningf("Ignoring non-ipsec event: type: %v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if evt.Lease.Subnet.Equal(n.SubnetLease.Subnet) {
+				log.Warningf("Ignoring own lease add event: %+v", evt.Lease)
+				continue
+			}
+
+			if err := n.AddIPSECPolicies(&evt.Lease, defaultReqID); err != nil {
+				log.Errorf("error adding ipsec policy: %v", err)
+			}
+
+			if err := n.iked.LoadSharedKey(evt.Lease.Attrs.PublicIP.String(), n.password); err != nil {
+				log.Errorf("error loading shared key into IKE daemon: %v", err)
+			}
+
+			if err := n.iked.LoadConnection(n.SubnetLease, &evt.Lease, strconv.Itoa(defaultReqID), strconv.FormatBool(n.UDPEncap)); err != nil {
+				log.Errorf("error loading connection into IKE daemon: %v", err)
+			}
+
+		case subnet.EventRemoved:
+			log.Info("Subnet removed: ", evt.Lease.Subnet)
+			if evt.Lease.Attrs.BackendType != "ipsec" {
+				log.Warningf("Ignoring non-ipsec event: type: %v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if evt.Lease.Subnet.Equal(n.SubnetLease.Subnet) {
+				log.Warningf("Ignoring own lease remove event: %+v", evt.Lease)
+				continue
+			}
+
+			if err := n.iked.UnloadCharonConnection(n.SubnetLease, &evt.Lease); err != nil {
+				log.Errorf("error unloading charon connections: %v", err)
+			}
+
+			if err := n.DeleteIPSECPolicies(n.SubnetLease.Subnet.ToIPNet(), evt.Lease.Subnet.ToIPNet(), n.SubnetLease.Attrs.PublicIP.ToIP(), evt.Lease.Attrs.PublicIP.ToIP(), defaultReqID); err != nil {
+				log.Errorf("error deleting ipsec policies: %v", err)
+			}
+		}
+	}
+}
+
+func (n *network) startIKEDaemon() {
+	if err := n.iked.Run(); err != nil {
+		log.Info("error starting IKE daemon: ", err)
+	}
+}
+
+func (n *network) MTU() int {
+	mtu := n.ExtIface.Iface.MTU - ipsecOverhead
+	if n.UDPEncap {
+		mtu -= udpEncapOverhead
+	}
+
+	return mtu
+}
+
+func (n *network) AddIPSECPolicies(remoteLease *subnet.Lease, reqID int) error {
+	err := AddXFRMPolicy(n.SubnetLease, remoteLease, netlink.XFRM_DIR_OUT, reqID)
+	if err != nil {
+		return fmt.Errorf("error adding ipsec out policy: %v", err)
+	}
+
+	err = AddXFRMPolicy(remoteLease, n.SubnetLease, netlink.XFRM_DIR_IN, reqID)
+	if err != nil {
+		return fmt.Errorf("error adding ipsec in policy: %v", err)
+	}
+
+	err = AddXFRMPolicy(remoteLease, n.SubnetLease, netlink.XFRM_DIR_FWD, reqID)
+	if err != nil {
+		return fmt.Errorf("error adding ipsec fwd policy: %v", err)
+	}
+
+	return nil
+}
+
+func (n *network) DeleteIPSECPolicies(localSubnet, remoteSubnet *net.IPNet, localPublicIP, remotePublicIP net.IP, reqID int) error {
+	err := DeleteXFRMPolicy(localSubnet, remoteSubnet, localPublicIP, remotePublicIP, netlink.XFRM_DIR_OUT, reqID)
+	if err != nil {
+		return fmt.Errorf("error deleting ipsec out policy: %v", err)
+	}
+
+	err = DeleteXFRMPolicy(remoteSubnet, localSubnet, remotePublicIP, localPublicIP, netlink.XFRM_DIR_IN, reqID)
+	if err != nil {
+		return fmt.Errorf("error deleting ipsec in policy: %v", err)
+	}
+
+	err = DeleteXFRMPolicy(remoteSubnet, localSubnet, remotePublicIP, localPublicIP, netlink.XFRM_DIR_FWD, reqID)
+	if err != nil {
+		return fmt.Errorf("error deleting ipsec fwd policy: %v", err)
+	}
+
+	return nil
+}

main.go

@@ -32,6 +32,7 @@ import (
 	log "github.com/golang/glog"
 	"golang.org/x/net/context"
 
+	"github.com/coreos/flannel/backend/ipsec"
 	"github.com/coreos/flannel/network"
 	"github.com/coreos/flannel/pkg/ip"
 	"github.com/coreos/flannel/subnet"
@@ -92,6 +93,7 @@ type CmdLineOpts struct {
 	subnetLeaseRenewMargin int
 	healthzIP              string
 	healthzPort            int
+	charonPath     string
 }
 
 var (
@@ -121,6 +123,7 @@ func init() {
 	flannelFlags.BoolVar(&opts.version, "version", false, "print version and exit")
 	flannelFlags.StringVar(&opts.healthzIP, "healthz-ip", "0.0.0.0", "the IP address for healthz server to listen")
 	flannelFlags.IntVar(&opts.healthzPort, "healthz-port", 0, "the port for healthz server to listen(0 to disable)")
+	flag.StringVar(&opts.charonPath, "charon-path", "", "path to charon executable")
 
 	// glog will log to tmp files by default. override so all entries
 	// can flow into journald (if running under systemd)
@@ -283,6 +286,10 @@ func main() {
 		os.Exit(1)
 	}
 
+	if opts.charonPath != "" {
+		ipsec.CharonPath = opts.charonPath
+	}
+
 	// Set up ipMasq if needed
 	if opts.ipMasq {
 		go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()))

remote/client.go

@@ -0,0 +1,408 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remote
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"path"
+	"time"
+
+	"github.com/coreos/etcd/pkg/transport"
+	"golang.org/x/net/context"
+
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+)
+
+// implements subnet.Manager by sending requests to the server
+type RemoteManager struct {
+	base      string // includes scheme, host, and port, and version
+	transport *Transport
+}
+
+func NewTransport(info transport.TLSInfo) (*Transport, error) {
+	cfg, err := info.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	t := &Transport{
+		// timeouts taken from http.DefaultTransport
+		Dial: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout: 10 * time.Second,
+		TLSClientConfig:     cfg,
+	}
+
+	return t, nil
+}
+
+func NewRemoteManager(listenAddr, cafile, certfile, keyfile string) (subnet.Manager, error) {
+	tls := transport.TLSInfo{
+		CAFile:   cafile,
+		CertFile: certfile,
+		KeyFile:  keyfile,
+	}
+
+	t, err := NewTransport(tls)
+	if err != nil {
+		return nil, err
+	}
+
+	var scheme string
+	if tls.Empty() && tls.CAFile == "" {
+		scheme = "http://"
+	} else {
+		scheme = "https://"
+	}
+
+	return &RemoteManager{
+		base:      scheme + listenAddr + "/v1",
+		transport: t,
+	}, nil
+}
+
+func (m *RemoteManager) mkurl(network string, parts ...string) string {
+	if network == "" {
+		network = "/_"
+	}
+	if network[0] != '/' {
+		network = "/" + network
+	}
+	return m.base + path.Join(append([]string{network}, parts...)...)
+}
+
+func (m *RemoteManager) GetNetworkConfig(ctx context.Context, network string) (*subnet.Config, error) {
+	url := m.mkurl(network, "config")
+
+	resp, err := m.httpVerb(ctx, "GET", url, "", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpError(resp)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	config, err := subnet.ParseConfig(string(body))
+	if err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+func (m *RemoteManager) AcquireLease(ctx context.Context, network string, attrs *subnet.LeaseAttrs) (*subnet.Lease, error) {
+	url := m.mkurl(network, "leases/")
+
+	body, err := json.Marshal(attrs)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := m.httpVerb(ctx, "POST", url, "application/json", body)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpError(resp)
+	}
+
+	newLease := &subnet.Lease{}
+	if err := json.NewDecoder(resp.Body).Decode(newLease); err != nil {
+		return nil, err
+	}
+
+	return newLease, nil
+}
+
+func (m *RemoteManager) RenewLease(ctx context.Context, network string, lease *subnet.Lease) error {
+	url := m.mkurl(network, "leases", lease.Key())
+
+	body, err := json.Marshal(lease)
+	if err != nil {
+		return err
+	}
+
+	resp, err := m.httpVerb(ctx, "PUT", url, "application/json", body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return httpError(resp)
+	}
+
+	newLease := &subnet.Lease{}
+	if err := json.NewDecoder(resp.Body).Decode(newLease); err != nil {
+		return err
+	}
+
+	*lease = *newLease
+	return nil
+}
+
+func (m *RemoteManager) RevokeLease(ctx context.Context, network string, sn ip.IP4Net) error {
+	url := m.mkurl(network, "leases", subnet.MakeSubnetKey(sn))
+
+	resp, err := m.httpVerb(ctx, "DELETE", url, "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return httpError(resp)
+	}
+
+	return nil
+}
+
+func (m *RemoteManager) watch(ctx context.Context, url string, cursor interface{}, wr interface{}) error {
+	if cursor != nil {
+		c, ok := cursor.(string)
+		if !ok {
+			return fmt.Errorf("internal error: RemoteManager.watch received non-string cursor")
+		}
+
+		url = fmt.Sprintf("%v?next=%v", url, c)
+	}
+
+	resp, err := m.httpVerb(ctx, "GET", url, "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return httpError(resp)
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(wr); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *RemoteManager) WatchLease(ctx context.Context, network string, sn ip.IP4Net, cursor interface{}) (subnet.LeaseWatchResult, error) {
+	url := m.mkurl(network, "leases", subnet.MakeSubnetKey(sn))
+
+	wr := subnet.LeaseWatchResult{}
+	err := m.watch(ctx, url, cursor, &wr)
+	if err != nil {
+		return subnet.LeaseWatchResult{}, err
+	}
+	if _, ok := wr.Cursor.(string); !ok {
+		return subnet.LeaseWatchResult{}, fmt.Errorf("watch returned non-string cursor")
+	}
+
+	return wr, nil
+}
+
+func (m *RemoteManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (subnet.LeaseWatchResult, error) {
+	url := m.mkurl(network, "leases")
+
+	wr := subnet.LeaseWatchResult{}
+	err := m.watch(ctx, url, cursor, &wr)
+	if err != nil {
+		return subnet.LeaseWatchResult{}, err
+	}
+	if _, ok := wr.Cursor.(string); !ok {
+		return subnet.LeaseWatchResult{}, fmt.Errorf("watch returned non-string cursor")
+	}
+
+	return wr, nil
+}
+
+func (m *RemoteManager) WatchNetworks(ctx context.Context, cursor interface{}) (subnet.NetworkWatchResult, error) {
+	wr := subnet.NetworkWatchResult{}
+	err := m.watch(ctx, m.base+"/", cursor, &wr)
+	if err != nil {
+		return subnet.NetworkWatchResult{}, err
+	}
+
+	if _, ok := wr.Cursor.(string); !ok {
+		return subnet.NetworkWatchResult{}, fmt.Errorf("watch returned non-string cursor")
+	}
+
+	return wr, nil
+}
+
+func (m *RemoteManager) AddReservation(ctx context.Context, network string, r *subnet.Reservation) error {
+	url := m.mkurl(network, "reservations")
+
+	body, err := json.Marshal(r)
+	if err != nil {
+		return err
+	}
+
+	resp, err := m.httpVerb(ctx, "POST", url, "application/json", body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return httpError(resp)
+	}
+
+	return nil
+}
+
+func (m *RemoteManager) CreateBackendData(ctx context.Context, network, data string) error {
+	url := m.mkurl(network, "backend-data")
+
+	body, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+
+	resp, err := m.httpVerb(ctx, "POST", url, "application/json", body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return httpError(resp)
+	}
+
+	return nil
+}
+
+func (m *RemoteManager) RemoveReservation(ctx context.Context, network string, sn ip.IP4Net) error {
+	url := m.mkurl(network, "reservations", subnet.MakeSubnetKey(sn))
+
+	resp, err := m.httpVerb(ctx, "DELETE", url, "", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return httpError(resp)
+	}
+
+	return nil
+}
+
+func (m *RemoteManager) ListReservations(ctx context.Context, network string) ([]subnet.Reservation, error) {
+	url := m.mkurl(network, "reservations")
+
+	resp, err := m.httpVerb(ctx, "GET", url, "", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpError(resp)
+	}
+	rs := []subnet.Reservation{}
+	if err := json.NewDecoder(resp.Body).Decode(&rs); err != nil {
+		return nil, err
+	}
+
+	return rs, nil
+}
+
+func (m *RemoteManager) GetBackendData(ctx context.Context, network string) (string, error) {
+	url := m.mkurl(network, "backend-data")
+
+	resp, err := m.httpVerb(ctx, "GET", url, "", nil)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", httpError(resp)
+	}
+
+	data := ""
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return "", err
+	}
+
+	return data, nil
+}
+
+func httpError(resp *http.Response) error {
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	return fmt.Errorf("%v: %v", resp.Status, string(b))
+}
+
+type httpRespErr struct {
+	resp *http.Response
+	err  error
+}
+
+func (m *RemoteManager) httpDo(ctx context.Context, req *http.Request) (*http.Response, error) {
+	// Run the HTTP request in a goroutine (so it can be canceled) and pass
+	// the result via the channel c
+	client := &http.Client{Transport: m.transport}
+	c := make(chan httpRespErr, 1)
+	go func() {
+		resp, err := client.Do(req)
+		c <- httpRespErr{resp, err}
+	}()
+
+	select {
+	case <-ctx.Done():
+		m.transport.CancelRequest(req)
+		<-c // Wait for f to return.
+		return nil, ctx.Err()
+	case r := <-c:
+		return r.resp, r.err
+	}
+}
+
+func (m *RemoteManager) httpVerb(ctx context.Context, method, url, contentType string, body []byte) (*http.Response, error) {
+	var r io.Reader
+	if body != nil {
+		r = bytes.NewBuffer(body)
+	}
+
+	req, err := http.NewRequest(method, url, r)
+	if err != nil {
+		return nil, err
+	}
+
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+	return m.httpDo(ctx, req)
+}

remote/server.go

@@ -0,0 +1,447 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remote
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+
+	"github.com/coreos/etcd/pkg/transport"
+	"github.com/coreos/go-systemd/activation"
+	"github.com/coreos/go-systemd/daemon"
+	log "github.com/golang/glog"
+	"github.com/gorilla/mux"
+	"golang.org/x/net/context"
+
+	"github.com/coreos/flannel/subnet"
+)
+
+type handler func(context.Context, subnet.Manager, http.ResponseWriter, *http.Request)
+
+func jsonResponse(w http.ResponseWriter, code int, v interface{}) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(code)
+	if err := json.NewEncoder(w).Encode(v); err != nil {
+		log.Error("Error JSON encoding response: %v", err)
+	}
+}
+
+// GET /{network}/config
+func handleGetNetworkConfig(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	c, err := sm.GetNetworkConfig(ctx, network)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, c)
+}
+
+func handleAcquireLease(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	attrs := subnet.LeaseAttrs{}
+	if err := json.NewDecoder(r.Body).Decode(&attrs); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, "JSON decoding error: ", err)
+		return
+	}
+
+	lease, err := sm.AcquireLease(ctx, network, &attrs)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, lease)
+}
+
+// PUT /{network}/{lease.network}
+func handleRenewLease(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	lease := subnet.Lease{}
+	if err := json.NewDecoder(r.Body).Decode(&lease); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, "JSON decoding error: ", err)
+		return
+	}
+
+	if err := sm.RenewLease(ctx, network, &lease); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, lease)
+}
+
+func handleRevokeLease(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	sn := subnet.ParseSubnetKey(mux.Vars(r)["subnet"])
+	if sn == nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, "failed to parse subnet")
+		return
+	}
+
+	if err := sm.RevokeLease(ctx, network, *sn); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+}
+
+func handleGetBackendData(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	p, err := sm.GetBackendData(ctx, network)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, p)
+}
+
+func handleCreateBackendData(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	data := ""
+	if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, "JSON decoding error: ", err)
+		return
+	}
+
+	if err := sm.CreateBackendData(ctx, network, data); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
+
+func getCursor(u *url.URL) interface{} {
+	vals, ok := u.Query()["next"]
+	if !ok {
+		return nil
+	}
+	return vals[0]
+}
+
+// GET /{network}/leases/subnet?next=cursor
+func handleWatchLease(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	sn := subnet.ParseSubnetKey(mux.Vars(r)["subnet"])
+	if sn == nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, "bad subnet")
+		return
+	}
+
+	cursor := getCursor(r.URL)
+
+	wr, err := sm.WatchLease(ctx, network, *sn, cursor)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	switch wr.Cursor.(type) {
+	case string:
+	case fmt.Stringer:
+		wr.Cursor = wr.Cursor.(fmt.Stringer).String()
+	default:
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, fmt.Errorf("internal error: watch cursor is of unknown type"))
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, wr)
+}
+
+// GET /{network}/leases?next=cursor
+func handleWatchLeases(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	cursor := getCursor(r.URL)
+
+	wr, err := sm.WatchLeases(ctx, network, cursor)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	switch wr.Cursor.(type) {
+	case string:
+	case fmt.Stringer:
+		wr.Cursor = wr.Cursor.(fmt.Stringer).String()
+	default:
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, fmt.Errorf("internal error: watch cursor is of unknown type"))
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, wr)
+}
+
+// GET /?next=cursor watches
+// GET / retrieves all networks
+func handleNetworks(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	cursor := getCursor(r.URL)
+	wr, err := sm.WatchNetworks(ctx, cursor)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	switch wr.Cursor.(type) {
+	case string:
+	case fmt.Stringer:
+		wr.Cursor = wr.Cursor.(fmt.Stringer).String()
+	default:
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, fmt.Errorf("internal error: watch cursor is of unknown type"))
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, wr)
+}
+
+// POST /{network}/reservations
+func handleAddReservation(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	rsv := &subnet.Reservation{}
+	if err := json.NewDecoder(r.Body).Decode(rsv); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, "JSON decoding error: ", err)
+		return
+	}
+
+	if err := sm.AddReservation(ctx, network, rsv); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, fmt.Errorf("internal error: %v", err))
+		return
+	}
+}
+
+// DELETE /{network}/reservations/{subnet}
+func handleRemoveReservation(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	sn := subnet.ParseSubnetKey(mux.Vars(r)["subnet"])
+	if sn == nil {
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, "bad subnet")
+		return
+	}
+
+	if err := sm.RemoveReservation(ctx, network, *sn); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
+
+// GET /{network}/reservations
+func handleListReservations(ctx context.Context, sm subnet.Manager, w http.ResponseWriter, r *http.Request) {
+	network := mux.Vars(r)["network"]
+	if network == "_" {
+		network = ""
+	}
+
+	leases, err := sm.ListReservations(ctx, network)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, err)
+		return
+	}
+
+	jsonResponse(w, http.StatusOK, leases)
+}
+
+func bindHandler(h handler, ctx context.Context, sm subnet.Manager) http.HandlerFunc {
+	return func(resp http.ResponseWriter, req *http.Request) {
+		h(ctx, sm, resp, req)
+	}
+}
+
+func fdListener(addr string) (net.Listener, error) {
+	fdOffset := 0
+	if addr != "" {
+		fd, err := strconv.Atoi(addr)
+		if err != nil {
+			return nil, fmt.Errorf("fd index is not a number")
+		}
+		fdOffset = fd - 3
+	}
+
+	listeners, err := activation.Listeners(false)
+	if err != nil {
+		return nil, err
+	}
+
+	if fdOffset >= len(listeners) {
+		return nil, fmt.Errorf("fd %v is out of range (%v)", addr, len(listeners)+3)
+	}
+
+	if listeners[fdOffset] == nil {
+		return nil, fmt.Errorf("fd %v was not socket activated", addr)
+	}
+
+	return listeners[fdOffset], nil
+}
+
+func listener(addr, cafile, certfile, keyfile string) (net.Listener, error) {
+	rex := regexp.MustCompile("(?:([a-z]+)://)?(.*)")
+	groups := rex.FindStringSubmatch(addr)
+
+	var l net.Listener
+	var err error
+
+	switch {
+	case groups == nil:
+		return nil, fmt.Errorf("bad listener address")
+
+	case groups[1] == "", groups[1] == "tcp":
+		if l, err = net.Listen("tcp", groups[2]); err != nil {
+			return nil, err
+		}
+
+	case groups[1] == "fd":
+		if l, err = fdListener(groups[2]); err != nil {
+			return nil, err
+		}
+
+	default:
+		return nil, fmt.Errorf("bad listener scheme")
+	}
+
+	tlsinfo := transport.TLSInfo{
+		CAFile:   cafile,
+		CertFile: certfile,
+		KeyFile:  keyfile,
+	}
+
+	if !tlsinfo.Empty() {
+		cfg, err := tlsinfo.ServerConfig()
+		if err != nil {
+			return nil, err
+		}
+
+		l = tls.NewListener(l, cfg)
+	}
+
+	return l, nil
+}
+
+func RunServer(ctx context.Context, sm subnet.Manager, listenAddr, cafile, certfile, keyfile string) {
+	// {network} is always required a the API level but to
+	// keep backward compat, special "_" network is allowed
+	// that means "no network"
+
+	r := mux.NewRouter()
+	r.HandleFunc("/v1/{network}/config", bindHandler(handleGetNetworkConfig, ctx, sm)).Methods("GET")
+
+	r.HandleFunc("/v1/{network}/leases", bindHandler(handleAcquireLease, ctx, sm)).Methods("POST")
+	r.HandleFunc("/v1/{network}/leases/{subnet}", bindHandler(handleWatchLease, ctx, sm)).Methods("GET")
+	r.HandleFunc("/v1/{network}/leases/{subnet}", bindHandler(handleRenewLease, ctx, sm)).Methods("PUT")
+	r.HandleFunc("/v1/{network}/leases/{subnet}", bindHandler(handleRevokeLease, ctx, sm)).Methods("DELETE")
+	r.HandleFunc("/v1/{network}/leases", bindHandler(handleWatchLeases, ctx, sm)).Methods("GET")
+	r.HandleFunc("/v1/", bindHandler(handleNetworks, ctx, sm)).Methods("GET")
+	r.HandleFunc("/v1/{network}/backend-data", bindHandler(handleGetBackendData, ctx, sm)).Methods("GET")
+	r.HandleFunc("/v1/{network}/backend-data", bindHandler(handleCreateBackendData, ctx, sm)).Methods("POST")
+
+	r.HandleFunc("/v1/{network}/reservations", bindHandler(handleListReservations, ctx, sm)).Methods("GET")
+	r.HandleFunc("/v1/{network}/reservations", bindHandler(handleAddReservation, ctx, sm)).Methods("POST")
+	r.HandleFunc("/v1/{network}/reservations/{subnet}", bindHandler(handleRemoveReservation, ctx, sm)).Methods("DELETE")
+
+	l, err := listener(listenAddr, cafile, certfile, keyfile)
+	if err != nil {
+		log.Errorf("Error listening on %v: %v", listenAddr, err)
+		return
+	}
+
+	c := make(chan error, 1)
+	go func() {
+		c <- http.Serve(l, httpLogger(r))
+	}()
+
+	daemon.SdNotify("READY=1")
+
+	select {
+	case <-ctx.Done():
+		l.Close()
+		<-c
+
+	case err := <-c:
+		log.Errorf("Error serving on %v: %v", listenAddr, err)
+	}
+}

subnet/local_manager.go

@@ -0,0 +1,516 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package subnet
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	etcd "github.com/coreos/etcd/client"
+	"github.com/coreos/flannel/pkg/ip"
+	log "github.com/golang/glog"
+	"golang.org/x/net/context"
+)
+
+const (
+	raceRetries = 10
+	subnetTTL   = 24 * time.Hour
+)
+
+type LocalManager struct {
+	registry Registry
+}
+
+type watchCursor struct {
+	index uint64
+}
+
+func isErrEtcdTestFailed(e error) bool {
+	if e == nil {
+		return false
+	}
+	etcdErr, ok := e.(etcd.Error)
+	return ok && etcdErr.Code == etcd.ErrorCodeTestFailed
+}
+
+func isErrEtcdNodeExist(e error) bool {
+	if e == nil {
+		return false
+	}
+	etcdErr, ok := e.(etcd.Error)
+	return ok || etcdErr.Code == etcd.ErrorCodeNodeExist
+}
+
+func isErrEtcdKeyNotFound(e error) bool {
+	if e == nil {
+		return false
+	}
+	etcdErr, ok := e.(etcd.Error)
+	return ok || etcdErr.Code == etcd.ErrorCodeKeyNotFound
+}
+
+func (c watchCursor) String() string {
+	return strconv.FormatUint(c.index, 10)
+}
+
+func NewLocalManager(config *EtcdConfig) (Manager, error) {
+	r, err := newEtcdSubnetRegistry(config, nil)
+	if err != nil {
+		return nil, err
+	}
+	return newLocalManager(r), nil
+}
+
+func newLocalManager(r Registry) Manager {
+	return &LocalManager{
+		registry: r,
+	}
+}
+
+func (m *LocalManager) GetNetworkConfig(ctx context.Context, network string) (*Config, error) {
+	cfg, err := m.registry.getNetworkConfig(ctx, network)
+	if err != nil {
+		return nil, err
+	}
+
+	return ParseConfig(cfg)
+}
+
+func (m *LocalManager) AcquireLease(ctx context.Context, network string, attrs *LeaseAttrs) (*Lease, error) {
+	config, err := m.GetNetworkConfig(ctx, network)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < raceRetries; i++ {
+		l, err := m.tryAcquireLease(ctx, network, config, attrs.PublicIP, attrs)
+		switch err {
+		case nil:
+			return l, nil
+		case errTryAgain:
+			continue
+		default:
+			return nil, err
+		}
+	}
+
+	return nil, errors.New("Max retries reached trying to acquire a subnet")
+}
+
+func findLeaseByIP(leases []Lease, pubIP ip.IP4) *Lease {
+	for _, l := range leases {
+		if pubIP == l.Attrs.PublicIP {
+			return &l
+		}
+	}
+
+	return nil
+}
+
+func (m *LocalManager) tryAcquireLease(ctx context.Context, network string, config *Config, extIaddr ip.IP4, attrs *LeaseAttrs) (*Lease, error) {
+	leases, _, err := m.registry.getSubnets(ctx, network)
+	if err != nil {
+		return nil, err
+	}
+
+	// try to reuse a subnet if there's one that matches our IP
+	if l := findLeaseByIP(leases, extIaddr); l != nil {
+		// make sure the existing subnet is still within the configured network
+		if isSubnetConfigCompat(config, l.Subnet) {
+			log.Infof("Found lease (%v) for current IP (%v), reusing", l.Subnet, extIaddr)
+
+			ttl := time.Duration(0)
+			if !l.Expiration.IsZero() {
+				// Not a reservation
+				ttl = subnetTTL
+			}
+			exp, err := m.registry.updateSubnet(ctx, network, l.Subnet, attrs, ttl, 0)
+			if err != nil {
+				return nil, err
+			}
+
+			l.Attrs = *attrs
+			l.Expiration = exp
+			return l, nil
+		} else {
+			log.Infof("Found lease (%v) for current IP (%v) but not compatible with current config, deleting", l.Subnet, extIaddr)
+			if err := m.registry.deleteSubnet(ctx, network, l.Subnet); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// no existing match, grab a new one
+	sn, err := m.allocateSubnet(config, leases)
+	if err != nil {
+		return nil, err
+	}
+
+	exp, err := m.registry.createSubnet(ctx, network, sn, attrs, subnetTTL)
+	switch {
+	case err == nil:
+		return &Lease{
+			Subnet:     sn,
+			Attrs:      *attrs,
+			Expiration: exp,
+		}, nil
+	case isErrEtcdNodeExist(err):
+		return nil, errTryAgain
+	default:
+		return nil, err
+	}
+}
+
+func (m *LocalManager) allocateSubnet(config *Config, leases []Lease) (ip.IP4Net, error) {
+	log.Infof("Picking subnet in range %s ... %s", config.SubnetMin, config.SubnetMax)
+
+	var bag []ip.IP4
+	sn := ip.IP4Net{IP: config.SubnetMin, PrefixLen: config.SubnetLen}
+
+OuterLoop:
+	for ; sn.IP <= config.SubnetMax && len(bag) < 100; sn = sn.Next() {
+		for _, l := range leases {
+			if sn.Overlaps(l.Subnet) {
+				continue OuterLoop
+			}
+		}
+		bag = append(bag, sn.IP)
+	}
+
+	if len(bag) == 0 {
+		return ip.IP4Net{}, errors.New("out of subnets")
+	} else {
+		i := randInt(0, len(bag))
+		return ip.IP4Net{IP: bag[i], PrefixLen: config.SubnetLen}, nil
+	}
+}
+
+func (m *LocalManager) RevokeLease(ctx context.Context, network string, sn ip.IP4Net) error {
+	return m.registry.deleteSubnet(ctx, network, sn)
+}
+
+func (m *LocalManager) RenewLease(ctx context.Context, network string, lease *Lease) error {
+	exp, err := m.registry.updateSubnet(ctx, network, lease.Subnet, &lease.Attrs, subnetTTL, 0)
+	if err != nil {
+		return err
+	}
+
+	lease.Expiration = exp
+	return nil
+}
+
+func getNextIndex(cursor interface{}) (uint64, error) {
+	nextIndex := uint64(0)
+
+	if wc, ok := cursor.(watchCursor); ok {
+		nextIndex = wc.index
+	} else if s, ok := cursor.(string); ok {
+		var err error
+		nextIndex, err = strconv.ParseUint(s, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("failed to parse cursor: %v", err)
+		}
+	} else {
+		return 0, fmt.Errorf("internal error: watch cursor is of unknown type")
+	}
+
+	return nextIndex, nil
+}
+
+func (m *LocalManager) leaseWatchReset(ctx context.Context, network string, sn ip.IP4Net) (LeaseWatchResult, error) {
+	l, index, err := m.registry.getSubnet(ctx, network, sn)
+	if err != nil {
+		return LeaseWatchResult{}, err
+	}
+
+	return LeaseWatchResult{
+		Snapshot: []Lease{*l},
+		Cursor:   watchCursor{index},
+	}, nil
+}
+
+func (m *LocalManager) WatchLease(ctx context.Context, network string, sn ip.IP4Net, cursor interface{}) (LeaseWatchResult, error) {
+	if cursor == nil {
+		return m.leaseWatchReset(ctx, network, sn)
+	}
+
+	nextIndex, err := getNextIndex(cursor)
+	if err != nil {
+		return LeaseWatchResult{}, err
+	}
+
+	evt, index, err := m.registry.watchSubnet(ctx, network, nextIndex, sn)
+
+	switch {
+	case err == nil:
+		return LeaseWatchResult{
+			Events: []Event{evt},
+			Cursor: watchCursor{index},
+		}, nil
+
+	case isIndexTooSmall(err):
+		log.Warning("Watch of subnet leases failed because etcd index outside history window")
+		return m.leaseWatchReset(ctx, network, sn)
+
+	default:
+		return LeaseWatchResult{}, err
+	}
+}
+
+func (m *LocalManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (LeaseWatchResult, error) {
+	if cursor == nil {
+		return m.leasesWatchReset(ctx, network)
+	}
+
+	nextIndex, err := getNextIndex(cursor)
+	if err != nil {
+		return LeaseWatchResult{}, err
+	}
+
+	evt, index, err := m.registry.watchSubnets(ctx, network, nextIndex)
+
+	switch {
+	case err == nil:
+		return LeaseWatchResult{
+			Events: []Event{evt},
+			Cursor: watchCursor{index},
+		}, nil
+
+	case isIndexTooSmall(err):
+		log.Warning("Watch of subnet leases failed because etcd index outside history window")
+		return m.leasesWatchReset(ctx, network)
+
+	default:
+		return LeaseWatchResult{}, err
+	}
+}
+
+func (m *LocalManager) WatchNetworks(ctx context.Context, cursor interface{}) (NetworkWatchResult, error) {
+	if cursor == nil {
+		return m.networkWatchReset(ctx)
+	}
+
+	nextIndex, err := getNextIndex(cursor)
+	if err != nil {
+		return NetworkWatchResult{}, err
+	}
+
+	for {
+		evt, index, err := m.registry.watchNetworks(ctx, nextIndex)
+
+		switch {
+		case err == nil:
+			return NetworkWatchResult{
+				Events: []Event{evt},
+				Cursor: watchCursor{index},
+			}, nil
+
+		case err == errTryAgain:
+			nextIndex = index
+
+		case isIndexTooSmall(err):
+			log.Warning("Watch of networks failed because etcd index outside history window")
+			return m.networkWatchReset(ctx)
+
+		default:
+			return NetworkWatchResult{}, err
+		}
+	}
+}
+
+func isIndexTooSmall(err error) bool {
+	etcdErr, ok := err.(etcd.Error)
+	return ok && etcdErr.Code == etcd.ErrorCodeEventIndexCleared
+}
+
+// leasesWatchReset is called when incremental lease watch failed and we need to grab a snapshot
+func (m *LocalManager) leasesWatchReset(ctx context.Context, network string) (LeaseWatchResult, error) {
+	wr := LeaseWatchResult{}
+
+	leases, index, err := m.registry.getSubnets(ctx, network)
+	if err != nil {
+		return wr, fmt.Errorf("failed to retrieve subnet leases: %v", err)
+	}
+
+	wr.Cursor = watchCursor{index}
+	wr.Snapshot = leases
+	return wr, nil
+}
+
+// networkWatchReset is called when incremental network watch failed and we need to grab a snapshot
+func (m *LocalManager) networkWatchReset(ctx context.Context) (NetworkWatchResult, error) {
+	wr := NetworkWatchResult{}
+
+	networks, index, err := m.registry.getNetworks(ctx)
+	if err != nil {
+		return wr, fmt.Errorf("failed to retrieve networks: %v", err)
+	}
+
+	wr.Cursor = watchCursor{index}
+	wr.Snapshot = networks
+	return wr, nil
+}
+
+func isSubnetConfigCompat(config *Config, sn ip.IP4Net) bool {
+	if sn.IP < config.SubnetMin || sn.IP > config.SubnetMax {
+		return false
+	}
+
+	return sn.PrefixLen == config.SubnetLen
+}
+
+func (m *LocalManager) tryAddReservation(ctx context.Context, network string, r *Reservation) error {
+	attrs := &LeaseAttrs{
+		PublicIP: r.PublicIP,
+	}
+
+	_, err := m.registry.createSubnet(ctx, network, r.Subnet, attrs, 0)
+	switch {
+	case err == nil:
+		return nil
+
+	case !isErrEtcdNodeExist(err):
+		return err
+	}
+
+	// This subnet or its reservation already exists.
+	// Get what's there and
+	// - if PublicIP matches, remove the TTL make it a reservation
+	// - otherwise, error out
+	sub, asof, err := m.registry.getSubnet(ctx, network, r.Subnet)
+	switch {
+	case err == nil:
+	case isErrEtcdKeyNotFound(err):
+		// Subnet just got expired or was deleted
+		return errTryAgain
+	default:
+		return err
+	}
+
+	if sub.Attrs.PublicIP != r.PublicIP {
+		// Subnet already taken
+		return ErrLeaseTaken
+	}
+
+	// remove TTL
+	_, err = m.registry.updateSubnet(ctx, network, r.Subnet, &sub.Attrs, 0, asof)
+	if isErrEtcdTestFailed(err) {
+		return errTryAgain
+	}
+	return err
+}
+
+func (m *LocalManager) AddReservation(ctx context.Context, network string, r *Reservation) error {
+	config, err := m.GetNetworkConfig(ctx, network)
+	if err != nil {
+		return err
+	}
+
+	if config.SubnetLen != r.Subnet.PrefixLen {
+		return fmt.Errorf("reservation subnet has mask incompatible with network config")
+	}
+
+	if !config.Network.Overlaps(r.Subnet) {
+		return fmt.Errorf("reservation subnet is outside of flannel network")
+	}
+
+	for i := 0; i < raceRetries; i++ {
+		err := m.tryAddReservation(ctx, network, r)
+		switch {
+		case err == nil:
+			return nil
+		case err == errTryAgain:
+			continue
+		default:
+			return err
+		}
+	}
+
+	return ErrNoMoreTries
+}
+
+func (m *LocalManager) tryRemoveReservation(ctx context.Context, network string, subnet ip.IP4Net) error {
+	sub, asof, err := m.registry.getSubnet(ctx, network, subnet)
+	if err != nil {
+		return err
+	}
+
+	// add back the TTL
+	_, err = m.registry.updateSubnet(ctx, network, subnet, &sub.Attrs, subnetTTL, asof)
+	if isErrEtcdTestFailed(err) {
+		return errTryAgain
+	}
+	return err
+}
+
+//RemoveReservation removes the subnet by setting TTL back to subnetTTL (24hours)
+func (m *LocalManager) RemoveReservation(ctx context.Context, network string, subnet ip.IP4Net) error {
+	for i := 0; i < raceRetries; i++ {
+		err := m.tryRemoveReservation(ctx, network, subnet)
+		switch {
+		case err == nil:
+			return nil
+		case err == errTryAgain:
+			continue
+		default:
+			return err
+		}
+	}
+
+	return ErrNoMoreTries
+}
+
+func (m *LocalManager) ListReservations(ctx context.Context, network string) ([]Reservation, error) {
+	subnets, _, err := m.registry.getSubnets(ctx, network)
+	if err != nil {
+		return nil, err
+	}
+
+	rsvs := []Reservation{}
+	for _, sub := range subnets {
+		// Reservations don't have TTL and so no expiration
+		if !sub.Expiration.IsZero() {
+			continue
+		}
+
+		r := Reservation{
+			Subnet:   sub.Subnet,
+			PublicIP: sub.Attrs.PublicIP,
+		}
+		rsvs = append(rsvs, r)
+	}
+
+	return rsvs, nil
+}
+
+func (m *LocalManager) CreateBackendData(ctx context.Context, network, data string) error {
+	err := m.registry.createBackendData(ctx, network, data)
+
+	if err == nil {
+		return nil
+	}
+
+	if etcdErr, ok := err.(etcd.Error); ok && etcdErr.Code == etcd.ErrorCodeNodeExist {
+		return nil
+	}
+
+	return err
+}
+
+func (m *LocalManager) GetBackendData(ctx context.Context, network string) (string, error) {
+	return m.registry.getBackendData(ctx, network)
+}

subnet/mock_registry.go

@@ -0,0 +1,488 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package subnet
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	etcd "github.com/coreos/etcd/client"
+	"golang.org/x/net/context"
+
+	"github.com/coreos/flannel/pkg/ip"
+)
+
+type netwk struct {
+	config        string
+	backendData   string
+	subnets       []Lease
+	subnetsEvents chan event
+	mux           sync.Mutex
+	subnetEvents  map[ip.IP4Net]chan event
+}
+
+func (n *netwk) sendSubnetEvent(sn ip.IP4Net, e event) {
+	n.subnetsEvents <- e
+
+	n.mux.Lock()
+	c, ok := n.subnetEvents[sn]
+	if !ok {
+		c = make(chan event, 10)
+		n.subnetEvents[sn] = c
+	}
+	n.mux.Unlock()
+	c <- e
+}
+
+func (n *netwk) subnetEventsChan(sn ip.IP4Net) chan event {
+	n.mux.Lock()
+	c, ok := n.subnetEvents[sn]
+	if !ok {
+		c = make(chan event, 10)
+		n.subnetEvents[sn] = c
+	}
+	n.mux.Unlock()
+	return c
+}
+
+type event struct {
+	evt   Event
+	index uint64
+}
+
+type MockSubnetRegistry struct {
+	mux           sync.Mutex
+	networks      map[string]*netwk
+	networkEvents chan event
+	index         uint64
+}
+
+func NewMockRegistry(network, config string, initialSubnets []Lease) *MockSubnetRegistry {
+	msr := &MockSubnetRegistry{
+		networkEvents: make(chan event, 1000),
+		index:         1000,
+		networks:      make(map[string]*netwk),
+	}
+
+	msr.networks[network] = &netwk{
+		config:        config,
+		subnets:       initialSubnets,
+		subnetsEvents: make(chan event, 1000),
+		subnetEvents:  make(map[ip.IP4Net]chan event),
+	}
+	return msr
+}
+
+func (msr *MockSubnetRegistry) getNetworkConfig(ctx context.Context, network string) (string, error) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return "", fmt.Errorf("Network %s not found", network)
+	}
+	return n.config, nil
+}
+
+func (msr *MockSubnetRegistry) setConfig(network, config string) error {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return fmt.Errorf("Network %s not found", network)
+	}
+	n.config = config
+	return nil
+}
+
+func (msr *MockSubnetRegistry) getSubnets(ctx context.Context, network string) ([]Lease, uint64, error) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return nil, 0, fmt.Errorf("Network %s not found", network)
+	}
+
+	subs := make([]Lease, len(n.subnets))
+	copy(subs, n.subnets)
+	return subs, msr.index, nil
+}
+
+func (msr *MockSubnetRegistry) getSubnet(ctx context.Context, network string, sn ip.IP4Net) (*Lease, uint64, error) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return nil, 0, fmt.Errorf("Network %s not found", network)
+	}
+	for _, l := range n.subnets {
+		if l.Subnet.Equal(sn) {
+			return &l, msr.index, nil
+		}
+	}
+	return nil, msr.index, fmt.Errorf("subnet %s not found", sn)
+}
+
+func (msr *MockSubnetRegistry) createSubnet(ctx context.Context, network string, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration) (time.Time, error) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return time.Time{}, fmt.Errorf("Network %s not found", network)
+	}
+
+	// check for existing
+	if _, _, err := n.findSubnet(sn); err == nil {
+		return time.Time{}, etcd.Error{
+			Code:  etcd.ErrorCodeNodeExist,
+			Index: msr.index,
+		}
+	}
+
+	msr.index += 1
+
+	exp := time.Time{}
+	if ttl != 0 {
+		exp = clock.Now().Add(ttl)
+	}
+
+	l := Lease{
+		Subnet:     sn,
+		Attrs:      *attrs,
+		Expiration: exp,
+		asof:       msr.index,
+	}
+	n.subnets = append(n.subnets, l)
+
+	evt := Event{
+		Type:    EventAdded,
+		Lease:   l,
+		Network: network,
+	}
+
+	n.sendSubnetEvent(sn, event{evt, msr.index})
+
+	return exp, nil
+}
+
+func (msr *MockSubnetRegistry) updateSubnet(ctx context.Context, network string, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration, asof uint64) (time.Time, error) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return time.Time{}, fmt.Errorf("Network %s not found", network)
+	}
+
+	msr.index += 1
+
+	exp := time.Time{}
+	if ttl != 0 {
+		exp = clock.Now().Add(ttl)
+	}
+
+	sub, i, err := n.findSubnet(sn)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	sub.Attrs = *attrs
+	sub.asof = msr.index
+	sub.Expiration = exp
+	n.subnets[i] = sub
+	n.sendSubnetEvent(sn, event{
+		Event{
+			Type:    EventAdded,
+			Lease:   sub,
+			Network: network,
+		}, msr.index,
+	})
+
+	return sub.Expiration, nil
+}
+
+func (msr *MockSubnetRegistry) deleteSubnet(ctx context.Context, network string, sn ip.IP4Net) error {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return fmt.Errorf("Network %s not found", network)
+	}
+
+	msr.index += 1
+
+	sub, i, err := n.findSubnet(sn)
+	if err != nil {
+		return err
+	}
+
+	n.subnets[i] = n.subnets[len(n.subnets)-1]
+	n.subnets = n.subnets[:len(n.subnets)-1]
+	sub.asof = msr.index
+	n.sendSubnetEvent(sn, event{
+		Event{
+			Type:    EventRemoved,
+			Lease:   sub,
+			Network: network,
+		}, msr.index,
+	})
+
+	return nil
+}
+
+func (msr *MockSubnetRegistry) watchSubnets(ctx context.Context, network string, since uint64) (Event, uint64, error) {
+	msr.mux.Lock()
+	n, ok := msr.networks[network]
+	msr.mux.Unlock()
+
+	if !ok {
+		return Event{}, 0, fmt.Errorf("Network %s not found", network)
+	}
+
+	for {
+		msr.mux.Lock()
+		index := msr.index
+		msr.mux.Unlock()
+
+		if since < index {
+			return Event{}, 0, etcd.Error{
+				Code:    etcd.ErrorCodeEventIndexCleared,
+				Cause:   "out of date",
+				Message: "cursor is out of date",
+				Index:   index,
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return Event{}, 0, ctx.Err()
+
+		case e := <-n.subnetsEvents:
+			if e.index > since {
+				return e.evt, e.index, nil
+			}
+		}
+	}
+}
+
+func (msr *MockSubnetRegistry) watchSubnet(ctx context.Context, network string, since uint64, sn ip.IP4Net) (Event, uint64, error) {
+	msr.mux.Lock()
+	n, ok := msr.networks[network]
+	msr.mux.Unlock()
+
+	if !ok {
+		return Event{}, 0, fmt.Errorf("Network %s not found", network)
+	}
+
+	for {
+		msr.mux.Lock()
+		index := msr.index
+		msr.mux.Unlock()
+
+		if since < index {
+			return Event{}, msr.index, etcd.Error{
+				Code:    etcd.ErrorCodeEventIndexCleared,
+				Cause:   "out of date",
+				Message: "cursor is out of date",
+				Index:   index,
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return Event{}, index, ctx.Err()
+
+		case e := <-n.subnetEventsChan(sn):
+			if e.index > since {
+				return e.evt, index, nil
+			}
+		}
+	}
+}
+
+func (msr *MockSubnetRegistry) expireSubnet(network string, sn ip.IP4Net) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return
+	}
+
+	if sub, i, err := n.findSubnet(sn); err == nil {
+		msr.index += 1
+		n.subnets[i] = n.subnets[len(n.subnets)-1]
+		n.subnets = n.subnets[:len(n.subnets)-1]
+		sub.asof = msr.index
+		n.sendSubnetEvent(sn, event{
+			Event{
+				Type:  EventRemoved,
+				Lease: sub,
+			}, msr.index,
+		})
+	}
+}
+
+func configKeyToNetworkKey(configKey string) string {
+	if !strings.HasSuffix(configKey, "/config") {
+		return ""
+	}
+	return strings.TrimSuffix(configKey, "/config")
+}
+
+func (msr *MockSubnetRegistry) getNetworks(ctx context.Context) ([]string, uint64, error) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	ns := []string{}
+
+	for n, _ := range msr.networks {
+		ns = append(ns, n)
+	}
+
+	return ns, msr.index, nil
+}
+
+func (msr *MockSubnetRegistry) watchNetworks(ctx context.Context, since uint64) (Event, uint64, error) {
+	msr.mux.Lock()
+	index := msr.index
+	msr.mux.Unlock()
+
+	for {
+		if since < index {
+			return Event{}, 0, etcd.Error{
+				Code:    etcd.ErrorCodeEventIndexCleared,
+				Cause:   "out of date",
+				Message: "cursor is out of date",
+				Index:   index,
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return Event{}, 0, ctx.Err()
+
+		case e := <-msr.networkEvents:
+			if e.index > since {
+				return e.evt, e.index, nil
+			}
+		}
+	}
+}
+
+func (msr *MockSubnetRegistry) getNetwork(ctx context.Context, network string) (*netwk, error) {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	n, ok := msr.networks[network]
+	if !ok {
+		return nil, fmt.Errorf("Network %s not found", network)
+	}
+
+	return n, nil
+}
+
+func (msr *MockSubnetRegistry) CreateNetwork(ctx context.Context, network, config string) error {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	_, ok := msr.networks[network]
+	if ok {
+		return fmt.Errorf("Network %s already exists", network)
+	}
+
+	msr.index += 1
+
+	n := &netwk{
+		config:        network,
+		subnetsEvents: make(chan event, 1000),
+		subnetEvents:  make(map[ip.IP4Net]chan event),
+	}
+
+	msr.networks[network] = n
+	msr.networkEvents <- event{
+		Event{
+			Type:    EventAdded,
+			Network: network,
+		}, msr.index,
+	}
+
+	return nil
+}
+
+func (msr *MockSubnetRegistry) DeleteNetwork(ctx context.Context, network string) error {
+	msr.mux.Lock()
+	defer msr.mux.Unlock()
+
+	_, ok := msr.networks[network]
+	if !ok {
+		return fmt.Errorf("Network %s not found", network)
+
+	}
+	delete(msr.networks, network)
+
+	msr.index += 1
+
+	msr.networkEvents <- event{
+		Event{
+			Type:    EventRemoved,
+			Network: network,
+		}, msr.index,
+	}
+
+	return nil
+}
+
+func (n *netwk) findSubnet(sn ip.IP4Net) (Lease, int, error) {
+	for i, sub := range n.subnets {
+		if sub.Subnet.Equal(sn) {
+			return sub, i, nil
+		}
+	}
+	return Lease{}, 0, fmt.Errorf("subnet not found")
+}
+
+func (msr *MockSubnetRegistry) createBackendData(ctx context.Context, network, data string) error {
+	n, ok := msr.networks[network]
+	if !ok {
+		return fmt.Errorf("network %s not found", network)
+	}
+
+	if n.backendData == "" {
+		n.backendData = data
+	}
+
+	return nil
+}
+
+func (msr *MockSubnetRegistry) getBackendData(ctx context.Context, network string) (string, error) {
+	n, ok := msr.networks[network]
+	if !ok {
+		return "", fmt.Errorf("network %s not found", network)
+	}
+
+	if n.backendData == "" {
+		return "", fmt.Errorf("backendData not set for %s network", network)
+	}
+
+	return n.backendData, nil
+}

subnet/registry.go

@@ -0,0 +1,444 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package subnet
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path"
+	"regexp"
+	"sync"
+	"time"
+
+	etcd "github.com/coreos/etcd/client"
+	"github.com/coreos/etcd/pkg/transport"
+	log "github.com/golang/glog"
+	"golang.org/x/net/context"
+
+	"github.com/coreos/flannel/pkg/ip"
+)
+
+var (
+	subnetRegex *regexp.Regexp = regexp.MustCompile(`(\d+\.\d+.\d+.\d+)-(\d+)`)
+	errTryAgain                = errors.New("try again")
+)
+
+type Registry interface {
+	getNetworkConfig(ctx context.Context, network string) (string, error)
+	getSubnets(ctx context.Context, network string) ([]Lease, uint64, error)
+	getSubnet(ctx context.Context, network string, sn ip.IP4Net) (*Lease, uint64, error)
+	createSubnet(ctx context.Context, network string, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration) (time.Time, error)
+	updateSubnet(ctx context.Context, network string, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration, asof uint64) (time.Time, error)
+	deleteSubnet(ctx context.Context, network string, sn ip.IP4Net) error
+	watchSubnets(ctx context.Context, network string, since uint64) (Event, uint64, error)
+	watchSubnet(ctx context.Context, network string, since uint64, sn ip.IP4Net) (Event, uint64, error)
+	getNetworks(ctx context.Context) ([]string, uint64, error)
+	watchNetworks(ctx context.Context, since uint64) (Event, uint64, error)
+	createBackendData(ctx context.Context, network, data string) error
+	getBackendData(ctx context.Context, network string) (string, error)
+}
+
+type EtcdConfig struct {
+	Endpoints []string
+	Keyfile   string
+	Certfile  string
+	CAFile    string
+	Prefix    string
+	Username  string
+	Password  string
+}
+
+type etcdNewFunc func(c *EtcdConfig) (etcd.KeysAPI, error)
+
+type etcdSubnetRegistry struct {
+	cliNewFunc   etcdNewFunc
+	mux          sync.Mutex
+	cli          etcd.KeysAPI
+	etcdCfg      *EtcdConfig
+	networkRegex *regexp.Regexp
+}
+
+func newEtcdClient(c *EtcdConfig) (etcd.KeysAPI, error) {
+	tlsInfo := transport.TLSInfo{
+		CertFile: c.Certfile,
+		KeyFile:  c.Keyfile,
+		CAFile:   c.CAFile,
+	}
+
+	t, err := transport.NewTransport(tlsInfo, time.Second)
+	if err != nil {
+		return nil, err
+	}
+
+	cli, err := etcd.New(etcd.Config{
+		Endpoints: c.Endpoints,
+		Transport: t,
+		Username:  c.Username,
+		Password:  c.Password,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return etcd.NewKeysAPI(cli), nil
+}
+
+func newEtcdSubnetRegistry(config *EtcdConfig, cliNewFunc etcdNewFunc) (Registry, error) {
+	r := &etcdSubnetRegistry{
+		etcdCfg:      config,
+		networkRegex: regexp.MustCompile(config.Prefix + `/([^/]*)(/|/config)?$`),
+	}
+	if cliNewFunc != nil {
+		r.cliNewFunc = cliNewFunc
+	} else {
+		r.cliNewFunc = newEtcdClient
+	}
+
+	var err error
+	r.cli, err = r.cliNewFunc(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}
+
+func (esr *etcdSubnetRegistry) getNetworkConfig(ctx context.Context, network string) (string, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "config")
+	resp, err := esr.client().Get(ctx, key, &etcd.GetOptions{Quorum: true})
+	if err != nil {
+		return "", err
+	}
+	return resp.Node.Value, nil
+}
+
+// getSubnets queries etcd to get a list of currently allocated leases for a given network.
+// It returns the leases along with the "as-of" etcd-index that can be used as the starting
+// point for etcd watch.
+func (esr *etcdSubnetRegistry) getSubnets(ctx context.Context, network string) ([]Lease, uint64, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "subnets")
+	resp, err := esr.client().Get(ctx, key, &etcd.GetOptions{Recursive: true, Quorum: true})
+	if err != nil {
+		if etcdErr, ok := err.(etcd.Error); ok && etcdErr.Code == etcd.ErrorCodeKeyNotFound {
+			// key not found: treat it as empty set
+			return []Lease{}, etcdErr.Index, nil
+		}
+		return nil, 0, err
+	}
+
+	leases := []Lease{}
+	for _, node := range resp.Node.Nodes {
+		l, err := nodeToLease(node)
+		if err != nil {
+			log.Warningf("Ignoring bad subnet node: %v", err)
+			continue
+		}
+
+		leases = append(leases, *l)
+	}
+
+	return leases, resp.Index, nil
+}
+
+func (esr *etcdSubnetRegistry) getSubnet(ctx context.Context, network string, sn ip.IP4Net) (*Lease, uint64, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "subnets", MakeSubnetKey(sn))
+	resp, err := esr.client().Get(ctx, key, &etcd.GetOptions{Quorum: true})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	l, err := nodeToLease(resp.Node)
+	return l, resp.Index, err
+}
+
+func (esr *etcdSubnetRegistry) createSubnet(ctx context.Context, network string, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration) (time.Time, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "subnets", MakeSubnetKey(sn))
+	value, err := json.Marshal(attrs)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	opts := &etcd.SetOptions{
+		PrevExist: etcd.PrevNoExist,
+		TTL:       ttl,
+	}
+
+	resp, err := esr.client().Set(ctx, key, string(value), opts)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	exp := time.Time{}
+	if resp.Node.Expiration != nil {
+		exp = *resp.Node.Expiration
+	}
+
+	return exp, nil
+}
+
+func (esr *etcdSubnetRegistry) updateSubnet(ctx context.Context, network string, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration, asof uint64) (time.Time, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "subnets", MakeSubnetKey(sn))
+	value, err := json.Marshal(attrs)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	resp, err := esr.client().Set(ctx, key, string(value), &etcd.SetOptions{
+		PrevIndex: asof,
+		TTL:       ttl,
+	})
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	exp := time.Time{}
+	if resp.Node.Expiration != nil {
+		exp = *resp.Node.Expiration
+	}
+
+	return exp, nil
+}
+
+func (esr *etcdSubnetRegistry) deleteSubnet(ctx context.Context, network string, sn ip.IP4Net) error {
+	key := path.Join(esr.etcdCfg.Prefix, network, "subnets", MakeSubnetKey(sn))
+	_, err := esr.client().Delete(ctx, key, nil)
+	return err
+}
+
+func (esr *etcdSubnetRegistry) watchSubnets(ctx context.Context, network string, since uint64) (Event, uint64, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "subnets")
+	opts := &etcd.WatcherOptions{
+		AfterIndex: since,
+		Recursive:  true,
+	}
+	e, err := esr.client().Watcher(key, opts).Next(ctx)
+	if err != nil {
+		return Event{}, 0, err
+	}
+
+	evt, err := parseSubnetWatchResponse(e)
+	return evt, e.Node.ModifiedIndex, err
+}
+
+func (esr *etcdSubnetRegistry) watchSubnet(ctx context.Context, network string, since uint64, sn ip.IP4Net) (Event, uint64, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "subnets", MakeSubnetKey(sn))
+	opts := &etcd.WatcherOptions{
+		AfterIndex: since,
+	}
+
+	e, err := esr.client().Watcher(key, opts).Next(ctx)
+	if err != nil {
+		return Event{}, 0, err
+	}
+
+	evt, err := parseSubnetWatchResponse(e)
+	return evt, e.Node.ModifiedIndex, err
+}
+
+// getNetworks queries etcd to get a list of network names.  It returns the
+// networks along with the 'as-of' etcd-index that can be used as the starting
+// point for etcd watch.
+func (esr *etcdSubnetRegistry) getNetworks(ctx context.Context) ([]string, uint64, error) {
+	resp, err := esr.client().Get(ctx, esr.etcdCfg.Prefix, &etcd.GetOptions{Recursive: true, Quorum: true})
+
+	networks := []string{}
+
+	if err == nil {
+		for _, node := range resp.Node.Nodes {
+			// Look for '/config' on the child nodes
+			for _, child := range node.Nodes {
+				netname, isConfig := esr.parseNetworkKey(child.Key)
+				if isConfig {
+					networks = append(networks, netname)
+				}
+			}
+		}
+
+		return networks, resp.Index, nil
+	}
+
+	if etcdErr, ok := err.(etcd.Error); ok && etcdErr.Code == etcd.ErrorCodeKeyNotFound {
+		// key not found: treat it as empty set
+		return networks, etcdErr.Index, nil
+	}
+
+	return nil, 0, err
+}
+
+func (esr *etcdSubnetRegistry) watchNetworks(ctx context.Context, since uint64) (Event, uint64, error) {
+	key := esr.etcdCfg.Prefix
+	opts := &etcd.WatcherOptions{
+		AfterIndex: since,
+		Recursive:  true,
+	}
+	e, err := esr.client().Watcher(key, opts).Next(ctx)
+	if err != nil {
+		return Event{}, 0, err
+	}
+
+	return esr.parseNetworkWatchResponse(e)
+}
+
+func (esr *etcdSubnetRegistry) client() etcd.KeysAPI {
+	esr.mux.Lock()
+	defer esr.mux.Unlock()
+	return esr.cli
+}
+
+func (esr *etcdSubnetRegistry) resetClient() {
+	esr.mux.Lock()
+	defer esr.mux.Unlock()
+
+	var err error
+	esr.cli, err = newEtcdClient(esr.etcdCfg)
+	if err != nil {
+		panic(fmt.Errorf("resetClient: error recreating etcd client: %v", err))
+	}
+}
+
+func parseSubnetWatchResponse(resp *etcd.Response) (Event, error) {
+	sn := ParseSubnetKey(resp.Node.Key)
+	if sn == nil {
+		return Event{}, fmt.Errorf("%v %q: not a subnet, skipping", resp.Action, resp.Node.Key)
+	}
+
+	switch resp.Action {
+	case "delete", "expire":
+		return Event{
+			EventRemoved,
+			Lease{Subnet: *sn},
+			"",
+		}, nil
+
+	default:
+		attrs := &LeaseAttrs{}
+		err := json.Unmarshal([]byte(resp.Node.Value), attrs)
+		if err != nil {
+			return Event{}, err
+		}
+
+		exp := time.Time{}
+		if resp.Node.Expiration != nil {
+			exp = *resp.Node.Expiration
+		}
+
+		evt := Event{
+			EventAdded,
+			Lease{
+				Subnet:     *sn,
+				Attrs:      *attrs,
+				Expiration: exp,
+			},
+			"",
+		}
+		return evt, nil
+	}
+}
+
+func (esr *etcdSubnetRegistry) parseNetworkWatchResponse(resp *etcd.Response) (Event, uint64, error) {
+	index := resp.Node.ModifiedIndex
+	netname, isConfig := esr.parseNetworkKey(resp.Node.Key)
+	if netname == "" {
+		return Event{}, index, errTryAgain
+	}
+
+	evt := Event{}
+
+	switch resp.Action {
+	case "delete":
+		evt = Event{
+			EventRemoved,
+			Lease{},
+			netname,
+		}
+
+	default:
+		if !isConfig {
+			// Ignore non .../<netname>/config keys; tell caller to try again
+			return Event{}, index, errTryAgain
+		}
+
+		_, err := ParseConfig(resp.Node.Value)
+		if err != nil {
+			return Event{}, index, err
+		}
+
+		evt = Event{
+			EventAdded,
+			Lease{},
+			netname,
+		}
+	}
+
+	return evt, index, nil
+}
+
+// Returns network name from config key (eg, /coreos.com/network/foobar/config),
+// if the 'config' key isn't present we don't consider the network valid
+func (esr *etcdSubnetRegistry) parseNetworkKey(s string) (string, bool) {
+	if parts := esr.networkRegex.FindStringSubmatch(s); len(parts) == 3 {
+		return parts[1], parts[2] != ""
+	}
+
+	return "", false
+}
+
+func nodeToLease(node *etcd.Node) (*Lease, error) {
+	sn := ParseSubnetKey(node.Key)
+	if sn == nil {
+		return nil, fmt.Errorf("failed to parse subnet key %q", *sn)
+	}
+
+	attrs := &LeaseAttrs{}
+	if err := json.Unmarshal([]byte(node.Value), attrs); err != nil {
+		return nil, err
+	}
+
+	exp := time.Time{}
+	if node.Expiration != nil {
+		exp = *node.Expiration
+	}
+
+	lease := Lease{
+		Subnet:     *sn,
+		Attrs:      *attrs,
+		Expiration: exp,
+		asof:       node.ModifiedIndex,
+	}
+
+	return &lease, nil
+}
+
+func (esr *etcdSubnetRegistry) createBackendData(ctx context.Context, network, data string) error {
+	key := path.Join(esr.etcdCfg.Prefix, network, "backend-data")
+	_, err := esr.client().Create(ctx, key, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (esr *etcdSubnetRegistry) getBackendData(ctx context.Context, network string) (string, error) {
+	key := path.Join(esr.etcdCfg.Prefix, network, "backend-data")
+	etcdResponse, err := esr.client().Get(ctx, key, nil)
+
+	if err != nil {
+		return "", err
+	}
+
+	return etcdResponse.Node.Value, nil
+}

subnet/subnet.go

@@ -47,6 +47,11 @@ type Lease struct {
 	Asof uint64
 }
 
+type BackendAttrs struct {
+	Password    string
+	BackendType string
+}
+
 func (l *Lease) Key() string {
 	return MakeSubnetKey(l.Subnet)
 }
@@ -126,4 +131,7 @@ type Manager interface {
 	WatchLeases(ctx context.Context, cursor interface{}) (LeaseWatchResult, error)
 
 	Name() string
+
+	GetBackendData(ctx context.Context, network string) (string, error)
+	CreateBackendData(ctx context.Context, network, data string) error
 }