kube.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359
  1. // Copyright 2016 flannel authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package kube
  15. import (
  16. "encoding/json"
  17. "errors"
  18. "fmt"
  19. "io/ioutil"
  20. "net"
  21. "os"
  22. "time"
  23. "github.com/flannel-io/flannel/pkg/ip"
  24. "github.com/flannel-io/flannel/subnet"
  25. "golang.org/x/net/context"
  26. v1 "k8s.io/api/core/v1"
  27. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  28. "k8s.io/apimachinery/pkg/runtime"
  29. "k8s.io/apimachinery/pkg/types"
  30. "k8s.io/apimachinery/pkg/util/strategicpatch"
  31. "k8s.io/apimachinery/pkg/util/wait"
  32. "k8s.io/apimachinery/pkg/watch"
  33. clientset "k8s.io/client-go/kubernetes"
  34. listers "k8s.io/client-go/listers/core/v1"
  35. "k8s.io/client-go/rest"
  36. "k8s.io/client-go/tools/cache"
  37. "k8s.io/client-go/tools/clientcmd"
  38. log "k8s.io/klog"
  39. )
  40. var (
  41. ErrUnimplemented = errors.New("unimplemented")
  42. )
  43. const (
  44. resyncPeriod = 5 * time.Minute
  45. nodeControllerSyncTimeout = 10 * time.Minute
  46. )
  47. type kubeSubnetManager struct {
  48. annotations annotations
  49. client clientset.Interface
  50. nodeName string
  51. nodeStore listers.NodeLister
  52. nodeController cache.Controller
  53. subnetConf *subnet.Config
  54. events chan subnet.Event
  55. setNodeNetworkUnavailable bool
  56. }
  57. func NewSubnetManager(ctx context.Context, apiUrl, kubeconfig, prefix, netConfPath string, setNodeNetworkUnavailable bool) (subnet.Manager, error) {
  58. var cfg *rest.Config
  59. var err error
  60. // Try to build kubernetes config from a master url or a kubeconfig filepath. If neither masterUrl
  61. // or kubeconfigPath are passed in we fall back to inClusterConfig. If inClusterConfig fails,
  62. // we fallback to the default config.
  63. cfg, err = clientcmd.BuildConfigFromFlags(apiUrl, kubeconfig)
  64. if err != nil {
  65. return nil, fmt.Errorf("fail to create kubernetes config: %v", err)
  66. }
  67. c, err := clientset.NewForConfig(cfg)
  68. if err != nil {
  69. return nil, fmt.Errorf("unable to initialize client: %v", err)
  70. }
  71. // The kube subnet mgr needs to know the k8s node name that it's running on so it can annotate it.
  72. // If we're running as a pod then the POD_NAME and POD_NAMESPACE will be populated and can be used to find the node
  73. // name. Otherwise, the environment variable NODE_NAME can be passed in.
  74. nodeName := os.Getenv("NODE_NAME")
  75. if nodeName == "" {
  76. podName := os.Getenv("POD_NAME")
  77. podNamespace := os.Getenv("POD_NAMESPACE")
  78. if podName == "" || podNamespace == "" {
  79. return nil, fmt.Errorf("env variables POD_NAME and POD_NAMESPACE must be set")
  80. }
  81. pod, err := c.CoreV1().Pods(podNamespace).Get(ctx, podName, metav1.GetOptions{})
  82. if err != nil {
  83. return nil, fmt.Errorf("error retrieving pod spec for '%s/%s': %v", podNamespace, podName, err)
  84. }
  85. nodeName = pod.Spec.NodeName
  86. if nodeName == "" {
  87. return nil, fmt.Errorf("node name not present in pod spec '%s/%s'", podNamespace, podName)
  88. }
  89. }
  90. netConf, err := ioutil.ReadFile(netConfPath)
  91. if err != nil {
  92. return nil, fmt.Errorf("failed to read net conf: %v", err)
  93. }
  94. sc, err := subnet.ParseConfig(string(netConf))
  95. if err != nil {
  96. return nil, fmt.Errorf("error parsing subnet config: %s", err)
  97. }
  98. sm, err := newKubeSubnetManager(ctx, c, sc, nodeName, prefix)
  99. if err != nil {
  100. return nil, fmt.Errorf("error creating network manager: %s", err)
  101. }
  102. sm.setNodeNetworkUnavailable = setNodeNetworkUnavailable
  103. go sm.Run(context.Background())
  104. log.Infof("Waiting %s for node controller to sync", nodeControllerSyncTimeout)
  105. err = wait.Poll(time.Second, nodeControllerSyncTimeout, func() (bool, error) {
  106. return sm.nodeController.HasSynced(), nil
  107. })
  108. if err != nil {
  109. return nil, fmt.Errorf("error waiting for nodeController to sync state: %v", err)
  110. }
  111. log.Infof("Node controller sync successful")
  112. return sm, nil
  113. }
  114. func newKubeSubnetManager(ctx context.Context, c clientset.Interface, sc *subnet.Config, nodeName, prefix string) (*kubeSubnetManager, error) {
  115. var err error
  116. var ksm kubeSubnetManager
  117. ksm.annotations, err = newAnnotations(prefix)
  118. if err != nil {
  119. return nil, err
  120. }
  121. ksm.client = c
  122. ksm.nodeName = nodeName
  123. ksm.subnetConf = sc
  124. ksm.events = make(chan subnet.Event, 5000)
  125. indexer, controller := cache.NewIndexerInformer(
  126. &cache.ListWatch{
  127. ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
  128. return ksm.client.CoreV1().Nodes().List(ctx, options)
  129. },
  130. WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
  131. return ksm.client.CoreV1().Nodes().Watch(ctx, options)
  132. },
  133. },
  134. &v1.Node{},
  135. resyncPeriod,
  136. cache.ResourceEventHandlerFuncs{
  137. AddFunc: func(obj interface{}) {
  138. ksm.handleAddLeaseEvent(subnet.EventAdded, obj)
  139. },
  140. UpdateFunc: ksm.handleUpdateLeaseEvent,
  141. DeleteFunc: func(obj interface{}) {
  142. node, isNode := obj.(*v1.Node)
  143. // We can get DeletedFinalStateUnknown instead of *api.Node here and we need to handle that correctly.
  144. if !isNode {
  145. deletedState, ok := obj.(cache.DeletedFinalStateUnknown)
  146. if !ok {
  147. log.Infof("Error received unexpected object: %v", obj)
  148. return
  149. }
  150. node, ok = deletedState.Obj.(*v1.Node)
  151. if !ok {
  152. log.Infof("Error deletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj)
  153. return
  154. }
  155. obj = node
  156. }
  157. ksm.handleAddLeaseEvent(subnet.EventRemoved, obj)
  158. },
  159. },
  160. cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
  161. )
  162. ksm.nodeController = controller
  163. ksm.nodeStore = listers.NewNodeLister(indexer)
  164. return &ksm, nil
  165. }
  166. func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj interface{}) {
  167. n := obj.(*v1.Node)
  168. if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
  169. return
  170. }
  171. l, err := ksm.nodeToLease(*n)
  172. if err != nil {
  173. log.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
  174. return
  175. }
  176. ksm.events <- subnet.Event{et, l}
  177. }
  178. func (ksm *kubeSubnetManager) handleUpdateLeaseEvent(oldObj, newObj interface{}) {
  179. o := oldObj.(*v1.Node)
  180. n := newObj.(*v1.Node)
  181. if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
  182. return
  183. }
  184. if o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&
  185. o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&
  186. o.Annotations[ksm.annotations.BackendPublicIP] == n.Annotations[ksm.annotations.BackendPublicIP] {
  187. return // No change to lease
  188. }
  189. l, err := ksm.nodeToLease(*n)
  190. if err != nil {
  191. log.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
  192. return
  193. }
  194. ksm.events <- subnet.Event{subnet.EventAdded, l}
  195. }
  196. func (ksm *kubeSubnetManager) GetNetworkConfig(ctx context.Context) (*subnet.Config, error) {
  197. return ksm.subnetConf, nil
  198. }
  199. func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.LeaseAttrs) (*subnet.Lease, error) {
  200. cachedNode, err := ksm.nodeStore.Get(ksm.nodeName)
  201. if err != nil {
  202. return nil, err
  203. }
  204. n := cachedNode.DeepCopy()
  205. if n.Spec.PodCIDR == "" {
  206. return nil, fmt.Errorf("node %q pod cidr not assigned", ksm.nodeName)
  207. }
  208. bd, err := attrs.BackendData.MarshalJSON()
  209. if err != nil {
  210. return nil, err
  211. }
  212. _, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
  213. if err != nil {
  214. return nil, err
  215. }
  216. if n.Annotations[ksm.annotations.BackendData] != string(bd) ||
  217. n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
  218. n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||
  219. n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
  220. (n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String()) {
  221. n.Annotations[ksm.annotations.BackendType] = attrs.BackendType
  222. n.Annotations[ksm.annotations.BackendData] = string(bd)
  223. if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
  224. if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {
  225. log.Infof("Overriding public ip with '%s' from node annotation '%s'",
  226. n.Annotations[ksm.annotations.BackendPublicIPOverwrite],
  227. ksm.annotations.BackendPublicIPOverwrite)
  228. n.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite]
  229. }
  230. } else {
  231. n.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String()
  232. }
  233. n.Annotations[ksm.annotations.SubnetKubeManaged] = "true"
  234. oldData, err := json.Marshal(cachedNode)
  235. if err != nil {
  236. return nil, err
  237. }
  238. newData, err := json.Marshal(n)
  239. if err != nil {
  240. return nil, err
  241. }
  242. patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
  243. if err != nil {
  244. return nil, fmt.Errorf("failed to create patch for node %q: %v", ksm.nodeName, err)
  245. }
  246. _, err = ksm.client.CoreV1().Nodes().Patch(ctx, ksm.nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
  247. if err != nil {
  248. return nil, err
  249. }
  250. }
  251. if ksm.setNodeNetworkUnavailable {
  252. log.Infoln("Setting NodeNetworkUnavailable")
  253. err = ksm.setNodeNetworkUnavailableFalse(ctx)
  254. if err != nil {
  255. log.Errorf("Unable to set NodeNetworkUnavailable to False for %q: %v", ksm.nodeName, err)
  256. }
  257. } else {
  258. log.Infoln("Skip setting NodeNetworkUnavailable")
  259. }
  260. return &subnet.Lease{
  261. Subnet: ip.FromIPNet(cidr),
  262. Attrs: *attrs,
  263. Expiration: time.Now().Add(24 * time.Hour),
  264. }, nil
  265. }
  266. func (ksm *kubeSubnetManager) WatchLeases(ctx context.Context, cursor interface{}) (subnet.LeaseWatchResult, error) {
  267. select {
  268. case event := <-ksm.events:
  269. return subnet.LeaseWatchResult{
  270. Events: []subnet.Event{event},
  271. }, nil
  272. case <-ctx.Done():
  273. return subnet.LeaseWatchResult{}, context.Canceled
  274. }
  275. }
  276. func (ksm *kubeSubnetManager) Run(ctx context.Context) {
  277. log.Infof("Starting kube subnet manager")
  278. ksm.nodeController.Run(ctx.Done())
  279. }
  280. func (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l subnet.Lease, err error) {
  281. l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP])
  282. if err != nil {
  283. return l, err
  284. }
  285. l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]
  286. l.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData])
  287. _, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
  288. if err != nil {
  289. return l, err
  290. }
  291. l.Subnet = ip.FromIPNet(cidr)
  292. return l, nil
  293. }
  294. // RenewLease: unimplemented
  295. func (ksm *kubeSubnetManager) RenewLease(ctx context.Context, lease *subnet.Lease) error {
  296. return ErrUnimplemented
  297. }
  298. func (ksm *kubeSubnetManager) WatchLease(ctx context.Context, sn ip.IP4Net, cursor interface{}) (subnet.LeaseWatchResult, error) {
  299. return subnet.LeaseWatchResult{}, ErrUnimplemented
  300. }
  301. func (ksm *kubeSubnetManager) Name() string {
  302. return fmt.Sprintf("Kubernetes Subnet Manager - %s", ksm.nodeName)
  303. }
  304. // Set Kubernetes NodeNetworkUnavailable to false when starting
  305. // https://kubernetes.io/docs/concepts/architecture/nodes/#condition
  306. func (ksm *kubeSubnetManager) setNodeNetworkUnavailableFalse(ctx context.Context) error {
  307. condition := v1.NodeCondition{
  308. Type: v1.NodeNetworkUnavailable,
  309. Status: v1.ConditionFalse,
  310. Reason: "FlannelIsUp",
  311. Message: "Flannel is running on this node",
  312. LastTransitionTime: metav1.Now(),
  313. LastHeartbeatTime: metav1.Now(),
  314. }
  315. raw, err := json.Marshal(&[]v1.NodeCondition{condition})
  316. if err != nil {
  317. return err
  318. }
  319. patch := []byte(fmt.Sprintf(`{"status":{"conditions":%s}}`, raw))
  320. _, err = ksm.client.CoreV1().Nodes().PatchStatus(ctx, ksm.nodeName, patch)
  321. return err
  322. }