Bladeren bron

Added new flag -iptables-resync allows you to pass an int (default 5) that is used in network/iptables SetupAndEnsureIPTables to control how often it resync's the iptables rules. We found on a larger cluster that having this value hardcoded to 5 seconds created too much contention on the iptables lock for kube-proxy to properly function

Brett Wagner 7 jaren geleden
bovenliggende
commit
454afbe127
4 gewijzigde bestanden met toevoegingen van 8 en 5 verwijderingen
  1. 1 0
      Documentation/configuration.md
  2. 4 2
      main.go
  3. 2 2
      network/iptables.go
  4. 1 1
      network/iptables_windows.go

+ 1 - 0
Documentation/configuration.md

@@ -55,6 +55,7 @@ The following configuration illustrates the use of most options with `udp` backe
 --kube-subnet-mgr: Contact the Kubernetes API for subnet assignment instead of etcd.
 --kube-subnet-mgr: Contact the Kubernetes API for subnet assignment instead of etcd.
 --iface="": interface to use (IP or name) for inter-host communication. Defaults to the interface for the default route on the machine. This can be specified multiple times to check each option in order. Returns the first match found.
 --iface="": interface to use (IP or name) for inter-host communication. Defaults to the interface for the default route on the machine. This can be specified multiple times to check each option in order. Returns the first match found.
 --iface-regex="": regex expression to match the first interface to use (IP or name) for inter-host communication. If unspecified, will default to the interface for the default route on the machine. This can be specified multiple times to check each regex in order. Returns the first match found. This option is superseded by the iface option and will only be used if nothing matches any option specified in the iface options.
 --iface-regex="": regex expression to match the first interface to use (IP or name) for inter-host communication. If unspecified, will default to the interface for the default route on the machine. This can be specified multiple times to check each regex in order. Returns the first match found. This option is superseded by the iface option and will only be used if nothing matches any option specified in the iface options.
+--iptables-resync=5: resync period for iptables rules, in seconds. Defaults to 5 seconds, if you see a large amount of contention for the iptables lock increasing this will probably help.
 --subnet-file=/run/flannel/subnet.env: filename where env variables (subnet and MTU values) will be written to.
 --subnet-file=/run/flannel/subnet.env: filename where env variables (subnet and MTU values) will be written to.
 --subnet-lease-renew-margin=60: subnet lease renewal margin, in minutes.
 --subnet-lease-renew-margin=60: subnet lease renewal margin, in minutes.
 --ip-masq=false: setup IP masquerade for traffic destined for outside the flannel network. Flannel assumes that the default policy is ACCEPT in the NAT POSTROUTING chain.
 --ip-masq=false: setup IP masquerade for traffic destined for outside the flannel network. Flannel assumes that the default policy is ACCEPT in the NAT POSTROUTING chain.

+ 4 - 2
main.go

@@ -95,6 +95,7 @@ type CmdLineOpts struct {
 	healthzPort            int
 	healthzPort            int
 	charonExecutablePath   string
 	charonExecutablePath   string
 	charonViciUri          string
 	charonViciUri          string
+	iptablesResyncSeconds  int
 }
 }
 
 
 var (
 var (
@@ -124,6 +125,7 @@ func init() {
 	flannelFlags.BoolVar(&opts.version, "version", false, "print version and exit")
 	flannelFlags.BoolVar(&opts.version, "version", false, "print version and exit")
 	flannelFlags.StringVar(&opts.healthzIP, "healthz-ip", "0.0.0.0", "the IP address for healthz server to listen")
 	flannelFlags.StringVar(&opts.healthzIP, "healthz-ip", "0.0.0.0", "the IP address for healthz server to listen")
 	flannelFlags.IntVar(&opts.healthzPort, "healthz-port", 0, "the port for healthz server to listen(0 to disable)")
 	flannelFlags.IntVar(&opts.healthzPort, "healthz-port", 0, "the port for healthz server to listen(0 to disable)")
+	flannelFlags.IntVar(&opts.iptablesResyncSeconds, "iptables-resyc", 5, "resync period for iptables rules, in seconds")
 
 
 	// glog will log to tmp files by default. override so all entries
 	// glog will log to tmp files by default. override so all entries
 	// can flow into journald (if running under systemd)
 	// can flow into journald (if running under systemd)
@@ -288,13 +290,13 @@ func main() {
 
 
 	// Set up ipMasq if needed
 	// Set up ipMasq if needed
 	if opts.ipMasq {
 	if opts.ipMasq {
-		go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()))
+		go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()), opts.iptablesResyncSeconds)
 	}
 	}
 
 
 	// Always enables forwarding rules. This is needed for Docker versions >1.13 (https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#container-communication-between-hosts)
 	// Always enables forwarding rules. This is needed for Docker versions >1.13 (https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#container-communication-between-hosts)
 	// In Docker 1.12 and earlier, the default FORWARD chain policy was ACCEPT.
 	// In Docker 1.12 and earlier, the default FORWARD chain policy was ACCEPT.
 	// In Docker 1.13 and later, Docker sets the default policy of the FORWARD chain to DROP.
 	// In Docker 1.13 and later, Docker sets the default policy of the FORWARD chain to DROP.
-	go network.SetupAndEnsureIPTables(network.ForwardRules(config.Network.String()))
+	go network.SetupAndEnsureIPTables(network.ForwardRules(config.Network.String()), opts.iptablesResyncSeconds)
 
 
 	if err := WriteSubnetFile(opts.subnetFile, config.Network, opts.ipMasq, bn); err != nil {
 	if err := WriteSubnetFile(opts.subnetFile, config.Network, opts.ipMasq, bn); err != nil {
 		// Continue, even though it failed.
 		// Continue, even though it failed.

+ 2 - 2
network/iptables.go

@@ -79,7 +79,7 @@ func ipTablesRulesExist(ipt IPTables, rules []IPTablesRule) (bool, error) {
 	return true, nil
 	return true, nil
 }
 }
 
 
-func SetupAndEnsureIPTables(rules []IPTablesRule) {
+func SetupAndEnsureIPTables(rules []IPTablesRule, resyncPeriod int) {
 	ipt, err := iptables.New()
 	ipt, err := iptables.New()
 	if err != nil {
 	if err != nil {
 		// if we can't find iptables, give up and return
 		// if we can't find iptables, give up and return
@@ -97,7 +97,7 @@ func SetupAndEnsureIPTables(rules []IPTablesRule) {
 			log.Errorf("Failed to ensure iptables rules: %v", err)
 			log.Errorf("Failed to ensure iptables rules: %v", err)
 		}
 		}
 
 
-		time.Sleep(5 * time.Second)
+		time.Sleep(time.Duration(resyncPeriod) * time.Second)
 	}
 	}
 }
 }
 
 

+ 1 - 1
network/iptables_windows.go

@@ -39,6 +39,6 @@ func ForwardRules(flannelNetwork string) []IPTablesRule {
 	return nil
 	return nil
 }
 }
 
 
-func SetupAndEnsureIPTables(rules []IPTablesRule) {
+func SetupAndEnsureIPTables(rules []IPTablesRule, resyncPeriod int) {
 
 
 }
 }