hostgw_windows.go 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272
  1. // Copyright 2018 flannel authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package hostgw
  15. import (
  16. "fmt"
  17. "strings"
  18. "sync"
  19. "time"
  20. "github.com/Microsoft/hcsshim"
  21. "github.com/flannel-io/flannel/backend"
  22. "github.com/flannel-io/flannel/pkg/ip"
  23. "github.com/flannel-io/flannel/pkg/routing"
  24. "github.com/flannel-io/flannel/subnet"
  25. "github.com/pkg/errors"
  26. "golang.org/x/net/context"
  27. "k8s.io/apimachinery/pkg/util/json"
  28. "k8s.io/apimachinery/pkg/util/wait"
  29. log "k8s.io/klog"
  30. )
  31. func init() {
  32. backend.Register("host-gw", New)
  33. }
  34. type HostgwBackend struct {
  35. sm subnet.Manager
  36. extIface *backend.ExternalInterface
  37. }
  38. func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
  39. if !extIface.ExtAddr.Equal(extIface.IfaceAddr) {
  40. return nil, fmt.Errorf("your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend")
  41. }
  42. be := &HostgwBackend{
  43. sm: sm,
  44. extIface: extIface,
  45. }
  46. return be, nil
  47. }
  48. func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
  49. // 1. Parse configuration
  50. cfg := struct {
  51. Name string
  52. DNSServerList string
  53. }{}
  54. if len(config.Backend) > 0 {
  55. if err := json.Unmarshal(config.Backend, &cfg); err != nil {
  56. return nil, errors.Wrap(err, "error decoding windows host-gw backend config")
  57. }
  58. }
  59. if len(cfg.Name) == 0 {
  60. cfg.Name = "cbr0"
  61. }
  62. log.Infof("HOST-GW config: %+v", cfg)
  63. n := &backend.RouteNetwork{
  64. SimpleNetwork: backend.SimpleNetwork{
  65. ExtIface: be.extIface,
  66. },
  67. SM: be.sm,
  68. BackendType: "host-gw",
  69. Mtu: be.extIface.Iface.MTU,
  70. LinkIndex: be.extIface.Iface.Index,
  71. }
  72. n.GetRoute = func(lease *subnet.Lease) *routing.Route {
  73. return &routing.Route{
  74. DestinationSubnet: lease.Subnet.ToIPNet(),
  75. GatewayAddress: lease.Attrs.PublicIP.ToIP(),
  76. InterfaceIndex: n.LinkIndex,
  77. }
  78. }
  79. // 2. Acquire the lease form subnet manager
  80. attrs := subnet.LeaseAttrs{
  81. PublicIP: ip.FromIP(be.extIface.ExtAddr),
  82. BackendType: "host-gw",
  83. }
  84. l, err := be.sm.AcquireLease(ctx, &attrs)
  85. switch err {
  86. case nil:
  87. n.SubnetLease = l
  88. case context.Canceled, context.DeadlineExceeded:
  89. return nil, err
  90. default:
  91. return nil, errors.Wrap(err, "failed to acquire lease")
  92. }
  93. // 3. Check if the network exists and has the expected settings
  94. createNewNetwork := true
  95. expectedSubnet := n.SubnetLease.Subnet
  96. expectedAddressPrefix := expectedSubnet.String()
  97. expectedGatewayAddress := (expectedSubnet.IP + 1).String()
  98. expectedPodGatewayAddress := expectedSubnet.IP + 2
  99. networkName := cfg.Name
  100. var waitErr, lastErr error
  101. existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
  102. if err == nil {
  103. for _, subnet := range existingNetwork.Subnets {
  104. if subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGatewayAddress {
  105. createNewNetwork = false
  106. log.Infof("Found existing HNSNetwork %s", networkName)
  107. break
  108. }
  109. }
  110. }
  111. // 4. Create a new HNSNetwork
  112. expectedNetwork := existingNetwork
  113. if createNewNetwork {
  114. if existingNetwork != nil {
  115. if _, err := existingNetwork.Delete(); err != nil {
  116. return nil, errors.Wrapf(err, "failed to delete existing HNSNetwork %s", networkName)
  117. }
  118. log.Infof("Deleted stale HNSNetwork %s", networkName)
  119. }
  120. expectedNetwork = &hcsshim.HNSNetwork{
  121. Name: networkName,
  122. Type: "L2Bridge",
  123. DNSServerList: cfg.DNSServerList,
  124. Subnets: []hcsshim.Subnet{
  125. {
  126. AddressPrefix: expectedAddressPrefix,
  127. GatewayAddress: expectedGatewayAddress,
  128. },
  129. },
  130. }
  131. jsonRequest, err := json.Marshal(expectedNetwork)
  132. if err != nil {
  133. return nil, errors.Wrapf(err, "failed to marshal %+v", expectedNetwork)
  134. }
  135. log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest))
  136. newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest))
  137. if err != nil {
  138. return nil, errors.Wrapf(err, "failed to create HNSNetwork %s", networkName)
  139. }
  140. // Wait for the network to populate Management IP
  141. log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName)
  142. var newNetworkID = newNetwork.Id
  143. waitErr = wait.Poll(500*time.Millisecond, 30*time.Second, func() (done bool, err error) {
  144. newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetworkID, "")
  145. return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil
  146. })
  147. if waitErr == wait.ErrWaitTimeout {
  148. // Do not swallow the root cause
  149. if lastErr != nil {
  150. waitErr = lastErr
  151. }
  152. return nil, errors.Wrapf(waitErr, "timeout, failed to get management IP from HNSNetwork %s", networkName)
  153. }
  154. // Wait for the interface with the management IP
  155. log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
  156. managementIP, err := ip.ParseIP4(newNetwork.ManagementIP)
  157. if err != nil {
  158. return nil, errors.Wrapf(err, "Failed to parse management ip (%s)", newNetwork.ManagementIP)
  159. }
  160. waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
  161. _, lastErr = ip.GetInterfaceByIP(managementIP.ToIP())
  162. return lastErr == nil, nil
  163. })
  164. if waitErr == wait.ErrWaitTimeout {
  165. return nil, errors.Wrapf(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
  166. }
  167. log.Infof("Created HNSNetwork %s", networkName)
  168. expectedNetwork = newNetwork
  169. }
  170. // 5. Ensure a 1.2 endpoint on this network in the host compartment
  171. createNewBridgeEndpoint := true
  172. bridgeEndpointName := networkName + "_ep"
  173. existingBridgeEndpoint, err := hcsshim.GetHNSEndpointByName(bridgeEndpointName)
  174. if err == nil && existingBridgeEndpoint.IPAddress.String() == expectedPodGatewayAddress.String() {
  175. log.Infof("Found existing bridge HNSEndpoint %s", bridgeEndpointName)
  176. createNewBridgeEndpoint = false
  177. }
  178. // 6. Create a bridge HNSEndpoint
  179. expectedBridgeEndpoint := existingBridgeEndpoint
  180. if createNewBridgeEndpoint {
  181. if existingBridgeEndpoint != nil {
  182. if _, err = existingBridgeEndpoint.Delete(); err != nil {
  183. return nil, errors.Wrapf(err, "failed to delete existing bridge HNSEndpoint %s", bridgeEndpointName)
  184. }
  185. log.Infof("Deleted stale bridge HNSEndpoint %s", bridgeEndpointName)
  186. }
  187. expectedBridgeEndpoint = &hcsshim.HNSEndpoint{
  188. Name: bridgeEndpointName,
  189. IPAddress: expectedPodGatewayAddress.ToIP(),
  190. VirtualNetwork: expectedNetwork.Id,
  191. }
  192. log.Infof("Attempting to create bridge HNSEndpoint %+v", expectedBridgeEndpoint)
  193. if expectedBridgeEndpoint, err = expectedBridgeEndpoint.Create(); err != nil {
  194. return nil, errors.Wrapf(err, "failed to create bridge HNSEndpoint %s", bridgeEndpointName)
  195. }
  196. log.Infof("Created bridge HNSEndpoint %s", bridgeEndpointName)
  197. }
  198. // Wait for the bridgeEndpoint to attach to the host
  199. log.Infof("Waiting to attach bridge endpoint %s to host", bridgeEndpointName)
  200. waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
  201. lastErr = expectedBridgeEndpoint.HostAttach(1)
  202. if lastErr == nil {
  203. return true, nil
  204. }
  205. // See https://github.com/flannel-io/flannel/issues/1391 and
  206. // hcsshim lacks some validations to detect the error, so we judge it by error message.
  207. if strings.Contains(lastErr.Error(), "This endpoint is already attached to the switch.") {
  208. return true, nil
  209. }
  210. return false, nil
  211. })
  212. if waitErr == wait.ErrWaitTimeout {
  213. return nil, errors.Wrapf(lastErr, "failed to hot attach bridge HNSEndpoint %s to host compartment", bridgeEndpointName)
  214. }
  215. log.Infof("Attached bridge endpoint %s to host successfully", bridgeEndpointName)
  216. // 7. Enable forwarding on the host interface and endpoint
  217. for _, interfaceIpAddress := range []string{expectedNetwork.ManagementIP, expectedBridgeEndpoint.IPAddress.String()} {
  218. ipv4, err := ip.ParseIP4(interfaceIpAddress)
  219. if err != nil {
  220. return nil, errors.Wrapf(err, "Failed to parse expected net interface ip (%s)", interfaceIpAddress)
  221. }
  222. netInterface, err := ip.GetInterfaceByIP(ipv4.ToIP())
  223. if err != nil {
  224. return nil, errors.Wrapf(err, "failed to find interface for IP Address %s", interfaceIpAddress)
  225. }
  226. log.Infof("Found %+v interface with IP %s", netInterface, interfaceIpAddress)
  227. // When a new hns network is created, the interface is modified, esp the name, index
  228. if expectedNetwork.ManagementIP == ipv4.String() {
  229. n.LinkIndex = netInterface.Index
  230. n.Name = netInterface.Name
  231. }
  232. if err := ip.EnableForwardingForInterface(netInterface); err != nil {
  233. return nil, errors.Wrapf(err, "failed to enable forwarding on %s index %d", netInterface.Name, netInterface.Index)
  234. }
  235. log.Infof("Enabled forwarding on %s index %d", netInterface.Name, netInterface.Index)
  236. }
  237. return n, nil
  238. }