vxlan_network.go 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199
  1. // Copyright 2015 flannel authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // +build !windows
  15. package vxlan
  16. import (
  17. "encoding/json"
  18. "net"
  19. "sync"
  20. log "github.com/golang/glog"
  21. "github.com/vishvananda/netlink"
  22. "golang.org/x/net/context"
  23. "syscall"
  24. "github.com/coreos/flannel/backend"
  25. "github.com/coreos/flannel/pkg/ip"
  26. "github.com/coreos/flannel/subnet"
  27. )
  28. type network struct {
  29. backend.SimpleNetwork
  30. extIface *backend.ExternalInterface
  31. dev *vxlanDevice
  32. subnetMgr subnet.Manager
  33. }
  34. func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, _ ip.IP4Net, lease *subnet.Lease) (*network, error) {
  35. nw := &network{
  36. SimpleNetwork: backend.SimpleNetwork{
  37. SubnetLease: lease,
  38. ExtIface: extIface,
  39. },
  40. subnetMgr: subnetMgr,
  41. dev: dev,
  42. }
  43. return nw, nil
  44. }
  45. func (nw *network) Run(ctx context.Context) {
  46. wg := sync.WaitGroup{}
  47. log.V(0).Info("watching for new subnet leases")
  48. events := make(chan []subnet.Event)
  49. wg.Add(1)
  50. go func() {
  51. subnet.WatchLeases(ctx, nw.subnetMgr, nw.SubnetLease, events)
  52. log.V(1).Info("WatchLeases exited")
  53. wg.Done()
  54. }()
  55. defer wg.Wait()
  56. for {
  57. select {
  58. case evtBatch := <-events:
  59. nw.handleSubnetEvents(evtBatch)
  60. case <-ctx.Done():
  61. return
  62. }
  63. }
  64. }
  65. func (nw *network) MTU() int {
  66. return nw.dev.MTU()
  67. }
  68. type vxlanLeaseAttrs struct {
  69. VtepMAC hardwareAddr
  70. }
  71. func (nw *network) handleSubnetEvents(batch []subnet.Event) {
  72. for _, event := range batch {
  73. sn := event.Lease.Subnet
  74. attrs := event.Lease.Attrs
  75. if attrs.BackendType != "vxlan" {
  76. log.Warningf("ignoring non-vxlan subnet(%s): type=%v", sn, attrs.BackendType)
  77. continue
  78. }
  79. var vxlanAttrs vxlanLeaseAttrs
  80. if err := json.Unmarshal(attrs.BackendData, &vxlanAttrs); err != nil {
  81. log.Error("error decoding subnet lease JSON: ", err)
  82. continue
  83. }
  84. // This route is used when traffic should be vxlan encapsulated
  85. vxlanRoute := netlink.Route{
  86. LinkIndex: nw.dev.link.Attrs().Index,
  87. Scope: netlink.SCOPE_UNIVERSE,
  88. Dst: sn.ToIPNet(),
  89. Gw: sn.IP.ToIP(),
  90. }
  91. vxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)
  92. // directRouting is where the remote host is on the same subnet so vxlan isn't required.
  93. directRoute := netlink.Route{
  94. Dst: sn.ToIPNet(),
  95. Gw: attrs.PublicIP.ToIP(),
  96. }
  97. var directRoutingOK = false
  98. if nw.dev.directRouting {
  99. routes, err := netlink.RouteGet(attrs.PublicIP.ToIP())
  100. if err != nil {
  101. log.Errorf("Couldn't lookup route to %v: %v", attrs.PublicIP, err)
  102. continue
  103. }
  104. if len(routes) == 1 && routes[0].Gw == nil {
  105. // There is only a single route and there's no gateway (i.e. it's directly connected)
  106. directRoutingOK = true
  107. }
  108. }
  109. switch event.Type {
  110. case subnet.EventAdded:
  111. if directRoutingOK {
  112. log.V(2).Infof("Adding direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
  113. if err := netlink.RouteReplace(&directRoute); err != nil {
  114. log.Errorf("Error adding route to %v via %v: %v", sn, attrs.PublicIP, err)
  115. continue
  116. }
  117. } else {
  118. log.V(2).Infof("adding subnet: %s PublicIP: %s VtepMAC: %s", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))
  119. if err := nw.dev.AddARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
  120. log.Error("AddARP failed: ", err)
  121. continue
  122. }
  123. if err := nw.dev.AddFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
  124. log.Error("AddFDB failed: ", err)
  125. // Try to clean up the ARP entry then continue
  126. if err := nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
  127. log.Error("DelARP failed: ", err)
  128. }
  129. continue
  130. }
  131. // Set the route - the kernel would ARP for the Gw IP address if it hadn't already been set above so make sure
  132. // this is done last.
  133. if err := netlink.RouteReplace(&vxlanRoute); err != nil {
  134. log.Errorf("failed to add vxlanRoute (%s -> %s): %v", vxlanRoute.Dst, vxlanRoute.Gw, err)
  135. // Try to clean up both the ARP and FDB entries then continue
  136. if err := nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
  137. log.Error("DelARP failed: ", err)
  138. }
  139. if err := nw.dev.DelFDB(neighbor{IP: event.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
  140. log.Error("DelFDB failed: ", err)
  141. }
  142. continue
  143. }
  144. }
  145. case subnet.EventRemoved:
  146. if directRoutingOK {
  147. log.V(2).Infof("Removing direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
  148. if err := netlink.RouteDel(&directRoute); err != nil {
  149. log.Errorf("Error deleting route to %v via %v: %v", sn, attrs.PublicIP, err)
  150. }
  151. } else {
  152. log.V(2).Infof("removing subnet: %s PublicIP: %s VtepMAC: %s", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))
  153. // Try to remove all entries - don't bail out if one of them fails.
  154. if err := nw.dev.DelARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
  155. log.Error("DelARP failed: ", err)
  156. }
  157. if err := nw.dev.DelFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {
  158. log.Error("DelFDB failed: ", err)
  159. }
  160. if err := netlink.RouteDel(&vxlanRoute); err != nil {
  161. log.Errorf("failed to delete vxlanRoute (%s -> %s): %v", vxlanRoute.Dst, vxlanRoute.Gw, err)
  162. }
  163. }
  164. default:
  165. log.Error("internal error: unknown event type: ", int(event.Type))
  166. }
  167. }
  168. }