Merge pull request #840 from alvaroaleman/fix-712

Fix #712, allow overwriting the public IP of a Kubernetes node

Documentation/kubernetes.md

@@ -14,6 +14,10 @@ The `flannel` manifest defines four things:
 
 When you run pods, they will be allocated IP addresses from the pod network CIDR. No matter which node those pods end up on, they will be able to communicate with each other.
 
+# Annotations
+
+*  `flannel.alpha.coreos.com/public-ip-overwrite`: Allows to overwrite the public IP of a node. Useful if the public IP can not determined from the node, e.G. because it is behind a NAT
+
 ## Older versions of Kubernetes
 
 `kube-flannel.yaml` has some features that aren't compatible with older versions of Kubernetes, though flanneld itself should work with any version of Kubernetes.

dist/functional-test-k8s.sh

@@ -82,15 +82,15 @@ start_flannel() {
        docker exec flannel-e2e-test-flannel$host_num /bin/sh -c 'mkdir -p /etc/kube-flannel'
        echo $flannel_conf | docker exec -i flannel-e2e-test-flannel$host_num /bin/sh -c 'cat > /etc/kube-flannel/net-conf.json'
        docker exec -d flannel-e2e-test-flannel$host_num /opt/bin/flanneld --kube-subnet-mgr --kube-api-url $k8s_endpt
+       while ! docker exec flannel-e2e-test-flannel$host_num ls /run/flannel/subnet.env >/dev/null 2>&1; do
+         sleep 0.1
+       done
     done
 }
 
 create_ping_dest() {
     # add a dummy interface with $FLANNEL_SUBNET so we have a known working IP to ping
     for host_num in 1 2; do
-       while ! docker exec flannel-e2e-test-flannel$host_num ls /run/flannel/subnet.env >/dev/null 2>&1; do
-         sleep 0.1
-       done
 
        # Use declare to allow the host_num variable to be part of the ping_dest variable name. -g is needed to make it global
        declare -g ping_dest$host_num=$(docker "exec" --privileged flannel-e2e-test-flannel$host_num /bin/sh -c '\
@@ -119,6 +119,19 @@ test_host-gw() {
     pings
 }
 
+test_public-ip-overwrite(){
+  docker exec flannel-e2e-k8s-apiserver kubectl annotate node flannel1 \
+    flannel.alpha.coreos.com/public-ip-overwrite=172.18.0.2 >/dev/null 2>&1
+  start_flannel vxlan
+  assert_equals "172.18.0.2" \
+    "$(docker exec flannel-e2e-k8s-apiserver kubectl get node/flannel1 -o \
+    jsonpath='{.metadata.annotations.flannel\.alpha\.coreos\.com/public-ip}' 2>/dev/null)" \
+    "Overwriting public IP via annotation does not work"
+  # Remove annotation to not break all other tests
+  docker exec flannel-e2e-k8s-apiserver kubectl annotate node flannel1 \
+    flannel.alpha.coreos.com/public-ip-overwrite- >/dev/null 2>&1
+}
+
 pings() {
     # ping in both directions
 	assert "docker exec --privileged flannel-e2e-test-flannel1 /bin/ping -c 5 $ping_dest2" "Host 1 cannot ping host 2"

subnet/kube/kube.go

@@ -51,10 +51,11 @@ const (
 	resyncPeriod              = 5 * time.Minute
 	nodeControllerSyncTimeout = 10 * time.Minute
 
-	subnetKubeManagedAnnotation = "flannel.alpha.coreos.com/kube-subnet-manager"
-	backendDataAnnotation       = "flannel.alpha.coreos.com/backend-data"
-	backendTypeAnnotation       = "flannel.alpha.coreos.com/backend-type"
-	backendPublicIPAnnotation   = "flannel.alpha.coreos.com/public-ip"
+	subnetKubeManagedAnnotation        = "flannel.alpha.coreos.com/kube-subnet-manager"
+	backendDataAnnotation              = "flannel.alpha.coreos.com/backend-data"
+	backendTypeAnnotation              = "flannel.alpha.coreos.com/backend-type"
+	backendPublicIPAnnotation          = "flannel.alpha.coreos.com/public-ip"
+	backendPublicIPOverwriteAnnotation = "flannel.alpha.coreos.com/public-ip-overwrite"
 
 	netConfPath = "/etc/kube-flannel/net-conf.json"
 )
@@ -235,10 +236,15 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 	if n.Annotations[backendDataAnnotation] != string(bd) ||
 		n.Annotations[backendTypeAnnotation] != attrs.BackendType ||
 		n.Annotations[backendPublicIPAnnotation] != attrs.PublicIP.String() ||
-		n.Annotations[subnetKubeManagedAnnotation] != "true" {
+		n.Annotations[subnetKubeManagedAnnotation] != "true" ||
+		(n.Annotations[backendPublicIPOverwriteAnnotation] != "" && n.Annotations[backendPublicIPOverwriteAnnotation] != attrs.PublicIP.String()) {
 		n.Annotations[backendTypeAnnotation] = attrs.BackendType
 		n.Annotations[backendDataAnnotation] = string(bd)
-		n.Annotations[backendPublicIPAnnotation] = attrs.PublicIP.String()
+		if n.Annotations[backendPublicIPOverwriteAnnotation] != "" {
+			n.Annotations[backendPublicIPAnnotation] = n.Annotations[backendPublicIPOverwriteAnnotation]
+		} else {
+			n.Annotations[backendPublicIPAnnotation] = attrs.PublicIP.String()
+		}
 		n.Annotations[subnetKubeManagedAnnotation] = "true"
 
 		oldData, err := json.Marshal(cachedNode)