main.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498
  1. // Copyright 2015 flannel authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "errors"
  17. "flag"
  18. "fmt"
  19. "net"
  20. "net/http"
  21. "os"
  22. "os/signal"
  23. "path/filepath"
  24. "regexp"
  25. "strconv"
  26. "strings"
  27. "syscall"
  28. "github.com/coreos/pkg/flagutil"
  29. log "github.com/golang/glog"
  30. "golang.org/x/net/context"
  31. "github.com/coreos/flannel/network"
  32. "github.com/coreos/flannel/pkg/ip"
  33. "github.com/coreos/flannel/subnet"
  34. "github.com/coreos/flannel/subnet/etcdv2"
  35. "github.com/coreos/flannel/subnet/kube"
  36. "github.com/coreos/flannel/version"
  37. "time"
  38. "github.com/joho/godotenv"
  39. // Backends need to be imported for their init() to get executed and them to register
  40. "github.com/coreos/flannel/backend"
  41. _ "github.com/coreos/flannel/backend/alivpc"
  42. _ "github.com/coreos/flannel/backend/alloc"
  43. _ "github.com/coreos/flannel/backend/awsvpc"
  44. _ "github.com/coreos/flannel/backend/extension"
  45. _ "github.com/coreos/flannel/backend/gce"
  46. _ "github.com/coreos/flannel/backend/hostgw"
  47. _ "github.com/coreos/flannel/backend/udp"
  48. _ "github.com/coreos/flannel/backend/vxlan"
  49. "github.com/coreos/go-systemd/daemon"
  50. )
  51. type CmdLineOpts struct {
  52. etcdEndpoints string
  53. etcdPrefix string
  54. etcdKeyfile string
  55. etcdCertfile string
  56. etcdCAFile string
  57. etcdUsername string
  58. etcdPassword string
  59. help bool
  60. version bool
  61. kubeSubnetMgr bool
  62. kubeApiUrl string
  63. kubeConfigFile string
  64. iface string
  65. ifaceRegex string
  66. ipMasq bool
  67. subnetFile string
  68. subnetDir string
  69. publicIP string
  70. subnetLeaseRenewMargin int
  71. healthzIP string
  72. healthzPort int
  73. }
  74. var (
  75. opts CmdLineOpts
  76. errInterrupted = errors.New("interrupted")
  77. errCanceled = errors.New("canceled")
  78. flannelFlags = flag.NewFlagSet("flannel", flag.ExitOnError)
  79. )
  80. func init() {
  81. flannelFlags.StringVar(&opts.etcdEndpoints, "etcd-endpoints", "http://127.0.0.1:4001,http://127.0.0.1:2379", "a comma-delimited list of etcd endpoints")
  82. flannelFlags.StringVar(&opts.etcdPrefix, "etcd-prefix", "/coreos.com/network", "etcd prefix")
  83. flannelFlags.StringVar(&opts.etcdKeyfile, "etcd-keyfile", "", "SSL key file used to secure etcd communication")
  84. flannelFlags.StringVar(&opts.etcdCertfile, "etcd-certfile", "", "SSL certification file used to secure etcd communication")
  85. flannelFlags.StringVar(&opts.etcdCAFile, "etcd-cafile", "", "SSL Certificate Authority file used to secure etcd communication")
  86. flannelFlags.StringVar(&opts.etcdUsername, "etcd-username", "", "username for BasicAuth to etcd")
  87. flannelFlags.StringVar(&opts.etcdPassword, "etcd-password", "", "password for BasicAuth to etcd")
  88. flannelFlags.StringVar(&opts.iface, "iface", "", "interface to use (IP or name) for inter-host communication")
  89. flannelFlags.StringVar(&opts.ifaceRegex, "iface-regex", "", "regex expression to match the first interface to use (IP or name) for inter-host communication. Skipped if the iface option is also specified")
  90. flannelFlags.StringVar(&opts.subnetFile, "subnet-file", "/run/flannel/subnet.env", "filename where env variables (subnet, MTU, ... ) will be written to")
  91. flannelFlags.StringVar(&opts.publicIP, "public-ip", "", "IP accessible by other nodes for inter-host communication")
  92. flannelFlags.IntVar(&opts.subnetLeaseRenewMargin, "subnet-lease-renew-margin", 60, "subnet lease renewal margin, in minutes.")
  93. flannelFlags.BoolVar(&opts.ipMasq, "ip-masq", false, "setup IP masquerade rule for traffic destined outside of overlay network")
  94. flannelFlags.BoolVar(&opts.kubeSubnetMgr, "kube-subnet-mgr", false, "contact the Kubernetes API for subnet assignment instead of etcd.")
  95. flannelFlags.StringVar(&opts.kubeApiUrl, "kube-api-url", "", "Kubernetes API server URL. Does not need to be specified if flannel is running in a pod.")
  96. flannelFlags.StringVar(&opts.kubeConfigFile, "kubeconfig-file", "", "kubeconfig file location. Does not need to be specified if flannel is running in a pod.")
  97. flannelFlags.BoolVar(&opts.version, "version", false, "print version and exit")
  98. flannelFlags.StringVar(&opts.healthzIP, "healthz-ip", "0.0.0.0", "the IP address for healthz server to listen")
  99. flannelFlags.IntVar(&opts.healthzPort, "healthz-port", 0, "the port for healthz server to listen(0 to disable)")
  100. // glog will log to tmp files by default. override so all entries
  101. // can flow into journald (if running under systemd)
  102. flag.Set("logtostderr", "true")
  103. // Only copy the non file logging options from glog
  104. copyFlag("v")
  105. copyFlag("vmodule")
  106. copyFlag("log_backtrace_at")
  107. // Define the usage function
  108. flannelFlags.Usage = usage
  109. // now parse command line args
  110. flannelFlags.Parse(os.Args[1:])
  111. }
  112. func copyFlag(name string) {
  113. flannelFlags.Var(flag.Lookup(name).Value, flag.Lookup(name).Name, flag.Lookup(name).Usage)
  114. }
  115. func usage() {
  116. fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]...\n", os.Args[0])
  117. flannelFlags.PrintDefaults()
  118. os.Exit(0)
  119. }
  120. func newSubnetManager() (subnet.Manager, error) {
  121. if opts.kubeSubnetMgr {
  122. return kube.NewSubnetManager(opts.kubeApiUrl, opts.kubeConfigFile)
  123. }
  124. cfg := &etcdv2.EtcdConfig{
  125. Endpoints: strings.Split(opts.etcdEndpoints, ","),
  126. Keyfile: opts.etcdKeyfile,
  127. Certfile: opts.etcdCertfile,
  128. CAFile: opts.etcdCAFile,
  129. Prefix: opts.etcdPrefix,
  130. Username: opts.etcdUsername,
  131. Password: opts.etcdPassword,
  132. }
  133. // Attempt to renew the lease for the subnet specified in the subnetFile
  134. prevSubnet := ReadSubnetFromSubnetFile(opts.subnetFile)
  135. return etcdv2.NewLocalManager(cfg, prevSubnet)
  136. }
  137. func main() {
  138. if opts.version {
  139. fmt.Fprintln(os.Stderr, version.Version)
  140. os.Exit(0)
  141. }
  142. flagutil.SetFlagsFromEnv(flannelFlags, "FLANNELD")
  143. // Work out which interface to use
  144. extIface, err := LookupExtIface(opts.iface, opts.ifaceRegex)
  145. if err != nil {
  146. log.Error("Failed to find interface to use: ", err)
  147. os.Exit(1)
  148. }
  149. sm, err := newSubnetManager()
  150. if err != nil {
  151. log.Error("Failed to create SubnetManager: ", err)
  152. os.Exit(1)
  153. }
  154. log.Infof("Created subnet manager: %+v", sm)
  155. // Register for SIGINT and SIGTERM
  156. log.Info("Installing signal handlers")
  157. sigs := make(chan os.Signal, 1)
  158. signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
  159. ctx, cancel := context.WithCancel(context.Background())
  160. go shutdown(sigs, cancel)
  161. if opts.healthzPort > 0 {
  162. go mustRunHealthz()
  163. }
  164. // Fetch the network config (i.e. what backend to use etc..).
  165. config, err := getConfig(ctx, sm)
  166. if err == errCanceled {
  167. exit()
  168. }
  169. // Create a backend manager then use it to create the backend and register the network with it.
  170. bm := backend.NewManager(ctx, sm, extIface)
  171. be, err := bm.GetBackend(config.BackendType)
  172. if err != nil {
  173. log.Errorf("Error fetching backend: %s", err)
  174. exit()
  175. }
  176. bn, err := be.RegisterNetwork(ctx, config)
  177. if err != nil {
  178. log.Errorf("Error registering network: %s", err)
  179. exit()
  180. }
  181. // Set up ipMasq if needed
  182. if opts.ipMasq {
  183. err = network.SetupIPMasq(config.Network, bn.Lease())
  184. if err != nil {
  185. // Continue, even though it failed.
  186. log.Errorf("Failed to set up IP Masquerade: %v", err)
  187. }
  188. defer func() {
  189. if err := network.TeardownIPMasq(config.Network, bn.Lease()); err != nil {
  190. log.Errorf("Failed to tear down IP Masquerade: %v", err)
  191. }
  192. }()
  193. }
  194. if err := WriteSubnetFile(opts.subnetFile, config.Network, opts.ipMasq, bn); err != nil {
  195. // Continue, even though it failed.
  196. log.Warningf("Failed to write subnet file: %s", err)
  197. } else {
  198. log.Infof("Wrote subnet file to %s", opts.subnetFile)
  199. }
  200. // Start "Running" the backend network. This will block until the context is done so run in another goroutine.
  201. go bn.Run(ctx)
  202. log.Infof("Finished starting backend.")
  203. daemon.SdNotify(false, "READY=1")
  204. // Kube subnet mgr doesn't lease the subnet for this node - it just uses the podCidr that's already assigned.
  205. if opts.kubeSubnetMgr {
  206. // Wait for the shutdown to be signalled
  207. <-ctx.Done()
  208. } else {
  209. // Block waiting to renew the lease
  210. _ = MonitorLease(ctx, sm, bn)
  211. }
  212. // To get to here, the Cancel signal must have been received or the lease has been revoked.
  213. exit()
  214. }
  215. func exit() {
  216. // Wait just a second for the cancel signal to propagate everywhere, then just exit cleanly.
  217. log.Info("Waiting for cancel to propagate...")
  218. time.Sleep(time.Second)
  219. log.Info("Exiting...")
  220. os.Exit(0)
  221. }
  222. func shutdown(sigs chan os.Signal, cancel context.CancelFunc) {
  223. // Wait for the shutdown signal.
  224. <-sigs
  225. // Unregister to get default OS nuke behaviour in case we don't exit cleanly
  226. signal.Stop(sigs)
  227. log.Info("Starting shutdown...")
  228. // Call cancel on the context to close everything down.
  229. cancel()
  230. log.Info("Sent cancel signal...")
  231. }
  232. func getConfig(ctx context.Context, sm subnet.Manager) (*subnet.Config, error) {
  233. // Retry every second until it succeeds
  234. for {
  235. config, err := sm.GetNetworkConfig(ctx)
  236. if err != nil {
  237. log.Errorf("Couldn't fetch network config: %s", err)
  238. } else if config == nil {
  239. log.Warningf("Couldn't find network config: %s", err)
  240. } else {
  241. log.Infof("Found network config - Backend type: %s", config.BackendType)
  242. return config, nil
  243. }
  244. select {
  245. case <-ctx.Done():
  246. return nil, errCanceled
  247. case <-time.After(1 * time.Second):
  248. fmt.Println("timed out")
  249. }
  250. }
  251. }
  252. func MonitorLease(ctx context.Context, sm subnet.Manager, bn backend.Network) error {
  253. // Use the subnet manager to start watching leases.
  254. evts := make(chan subnet.Event)
  255. go subnet.WatchLease(ctx, sm, bn.Lease().Subnet, evts)
  256. renewMargin := time.Duration(opts.subnetLeaseRenewMargin) * time.Minute
  257. dur := bn.Lease().Expiration.Sub(time.Now()) - renewMargin
  258. for {
  259. select {
  260. case <-time.After(dur):
  261. err := sm.RenewLease(ctx, bn.Lease())
  262. if err != nil {
  263. log.Error("Error renewing lease (trying again in 1 min): ", err)
  264. dur = time.Minute
  265. continue
  266. }
  267. log.Info("Lease renewed, new expiration: ", bn.Lease().Expiration)
  268. dur = bn.Lease().Expiration.Sub(time.Now()) - renewMargin
  269. case e := <-evts:
  270. switch e.Type {
  271. case subnet.EventAdded:
  272. bn.Lease().Expiration = e.Lease.Expiration
  273. dur = bn.Lease().Expiration.Sub(time.Now()) - renewMargin
  274. log.Infof("Waiting for %s to renew lease", dur)
  275. case subnet.EventRemoved:
  276. log.Error("Lease has been revoked. Shutting down daemon.")
  277. return errInterrupted
  278. }
  279. case <-ctx.Done():
  280. log.Infof("Stopped monitoring lease")
  281. return errCanceled
  282. }
  283. }
  284. }
  285. func LookupExtIface(ifname string, ifregex string) (*backend.ExternalInterface, error) {
  286. var iface *net.Interface
  287. var ifaceAddr net.IP
  288. var err error
  289. if len(ifname) > 0 {
  290. if ifaceAddr = net.ParseIP(ifname); ifaceAddr != nil {
  291. log.Infof("Searching for interface using %s", ifaceAddr)
  292. iface, err = ip.GetInterfaceByIP(ifaceAddr)
  293. if err != nil {
  294. return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
  295. }
  296. } else {
  297. iface, err = net.InterfaceByName(ifname)
  298. if err != nil {
  299. return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
  300. }
  301. }
  302. } else if len(ifregex) > 0 {
  303. // Use the regex if specified and the iface option for matching a specific ip or name is not used
  304. ifaces, err := net.Interfaces()
  305. if err != nil {
  306. return nil, fmt.Errorf("error listing all interfaces: %s", err)
  307. }
  308. // Check IP
  309. for _, ifaceToMatch := range ifaces {
  310. ifaceIP, err := ip.GetIfaceIP4Addr(&ifaceToMatch)
  311. if err != nil {
  312. // Skip if there is no IPv4 address
  313. continue
  314. }
  315. matched, err := regexp.MatchString(ifregex, ifaceIP.String())
  316. if err != nil {
  317. return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String())
  318. }
  319. if matched {
  320. ifaceAddr = ifaceIP
  321. iface = &ifaceToMatch
  322. break
  323. }
  324. }
  325. // Check Name
  326. if iface == nil && ifaceAddr == nil {
  327. for _, ifaceToMatch := range ifaces {
  328. matched, err := regexp.MatchString(ifregex, ifaceToMatch.Name)
  329. if err != nil {
  330. return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceToMatch.Name)
  331. }
  332. if matched {
  333. iface = &ifaceToMatch
  334. break
  335. }
  336. }
  337. }
  338. // Check that nothing was matched
  339. if iface == nil {
  340. return nil, fmt.Errorf("Could not match pattern %s to any of the available network interfaces", ifregex)
  341. }
  342. } else {
  343. log.Info("Determining IP address of default interface")
  344. if iface, err = ip.GetDefaultGatewayIface(); err != nil {
  345. return nil, fmt.Errorf("failed to get default interface: %s", err)
  346. }
  347. }
  348. if ifaceAddr == nil {
  349. ifaceAddr, err = ip.GetIfaceIP4Addr(iface)
  350. if err != nil {
  351. return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
  352. }
  353. }
  354. log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)
  355. if iface.MTU == 0 {
  356. return nil, fmt.Errorf("failed to determine MTU for %s interface", ifaceAddr)
  357. }
  358. var extAddr net.IP
  359. if len(opts.publicIP) > 0 {
  360. extAddr = net.ParseIP(opts.publicIP)
  361. if extAddr == nil {
  362. return nil, fmt.Errorf("invalid public IP address: %s", opts.publicIP)
  363. }
  364. log.Infof("Using %s as external address", extAddr)
  365. }
  366. if extAddr == nil {
  367. log.Infof("Defaulting external address to interface address (%s)", ifaceAddr)
  368. extAddr = ifaceAddr
  369. }
  370. return &backend.ExternalInterface{
  371. Iface: iface,
  372. IfaceAddr: ifaceAddr,
  373. ExtAddr: extAddr,
  374. }, nil
  375. }
  376. func WriteSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network) error {
  377. dir, name := filepath.Split(path)
  378. os.MkdirAll(dir, 0755)
  379. tempFile := filepath.Join(dir, "."+name)
  380. f, err := os.Create(tempFile)
  381. if err != nil {
  382. return err
  383. }
  384. // Write out the first usable IP by incrementing
  385. // sn.IP by one
  386. sn := bn.Lease().Subnet
  387. sn.IP += 1
  388. fmt.Fprintf(f, "FLANNEL_NETWORK=%s\n", nw)
  389. fmt.Fprintf(f, "FLANNEL_SUBNET=%s\n", sn)
  390. fmt.Fprintf(f, "FLANNEL_MTU=%d\n", bn.MTU())
  391. _, err = fmt.Fprintf(f, "FLANNEL_IPMASQ=%v\n", ipMasq)
  392. f.Close()
  393. if err != nil {
  394. return err
  395. }
  396. // rename(2) the temporary file to the desired location so that it becomes
  397. // atomically visible with the contents
  398. return os.Rename(tempFile, path)
  399. //TODO - is this safe? What if it's not on the same FS?
  400. }
  401. func mustRunHealthz() {
  402. address := net.JoinHostPort(opts.healthzIP, strconv.Itoa(opts.healthzPort))
  403. log.Infof("Start healthz server on %s", address)
  404. http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
  405. w.WriteHeader(http.StatusOK)
  406. w.Write([]byte("flanneld is running"))
  407. })
  408. if err := http.ListenAndServe(address, nil); err != nil {
  409. log.Errorf("Start healthz server error. %v", err)
  410. panic(err)
  411. }
  412. }
  413. func ReadSubnetFromSubnetFile(path string) ip.IP4Net {
  414. var prevSubnet ip.IP4Net
  415. if _, err := os.Stat(path); !os.IsNotExist(err) {
  416. prevSubnetVals, err := godotenv.Read(path)
  417. if err != nil {
  418. log.Errorf("Couldn't fetch previous subnet from subnet file at %s: %s", path, err)
  419. } else if prevSubnetString, ok := prevSubnetVals["FLANNEL_SUBNET"]; ok {
  420. err = prevSubnet.UnmarshalJSON([]byte(prevSubnetString))
  421. if err != nil {
  422. log.Errorf("Couldn't parse previous subnet from subnet file at %s: %s", path, err)
  423. }
  424. }
  425. }
  426. return prevSubnet
  427. }