Merge pull request #845 from mkumatag/multiarch-e2e

Add support for multiarch e2e tests

Makefile

@@ -15,23 +15,18 @@ PACKAGES_EXPANDED=$(PACKAGES:%=github.com/coreos/flannel/%)
 
 # Set the (cross) compiler to use for different architectures
 ifeq ($(ARCH),amd64)
-	LIB_DIR=/lib/x86_64-linux-gnu
 	CC=gcc
 endif
 ifeq ($(ARCH),arm)
-	LIB_DIR=/usr/arm-linux-gnueabihf/lib
 	CC=arm-linux-gnueabihf-gcc
 endif
 ifeq ($(ARCH),arm64)
-	LIB_DIR=/usr/aarch64-linux-gnu/lib
 	CC=aarch64-linux-gnu-gcc
 endif
 ifeq ($(ARCH),ppc64le)
-	LIB_DIR=/usr/powerpc64le-linux-gnu/lib
 	CC=powerpc64le-linux-gnu-gcc
 endif
 ifeq ($(ARCH),s390x)
-	LIB_DIR=/usr/s390x-linux-gnu/lib
 	CC=s390x-linux-gnu-gcc
 endif
 
@@ -55,9 +50,10 @@ test: license-check gofmt
 	# Run the functional tests
 	make e2e-test
 
-e2e-test: bash_unit dist/flanneld-$(TAG)-$(ARCH).docker
-	./bash_unit dist/functional-test.sh
-	./bash_unit dist/functional-test-k8s.sh
+e2e-test: bash_unit dist/flanneld-e2e-$(TAG)-$(ARCH).docker
+	$(MAKE) -C images/iperf3 ARCH=$(ARCH)
+	FLANNEL_DOCKER_IMAGE=$(REGISTRY):$(TAG)-$(ARCH) ./bash_unit dist/functional-test.sh
+	FLANNEL_DOCKER_IMAGE=$(REGISTRY):$(TAG)-$(ARCH) ./bash_unit dist/functional-test-k8s.sh
 
 cover:
 	# A single package must be given - e.g. 'PACKAGES=pkg/ip make cover'
@@ -101,6 +97,19 @@ ifeq ($(ARCH),amd64)
 	docker build -f Dockerfile.$(ARCH) -t $(REGISTRY):$(TAG) .
 endif
 
+# This will build flannel natively using golang image
+dist/flanneld-e2e-$(TAG)-$(ARCH).docker:
+	# valid values for ARCH are [amd64 arm arm64 ppc64le s390x]
+	docker run -e GOARM=$(GOARM) \
+		-u $(shell id -u):$(shell id -g) \
+		-v $(CURDIR):/go/src/github.com/coreos/flannel:ro \
+		-v $(CURDIR)/dist:/go/src/github.com/coreos/flannel/dist \
+		golang:1.8.3 /bin/bash -c '\
+		cd /go/src/github.com/coreos/flannel && \
+		CGO_ENABLED=1 make -e dist/flanneld && \
+		mv dist/flanneld dist/flanneld-$(ARCH)'
+	docker build -f Dockerfile.$(ARCH) -t $(REGISTRY):$(TAG)-$(ARCH) .
+
 ## Create an ACI on disk for a specific arch and tag
 dist/flanneld-$(TAG)-$(ARCH).aci: dist/flanneld-$(TAG)-$(ARCH).docker
 	docker2aci dist/flanneld-$(TAG)-$(ARCH).docker

dist/functional-test-k8s.sh

@@ -1,15 +1,25 @@
 #!/bin/bash
+
+ARCH="${ARCH:-amd64}"
 ETCD_IMG="${ETCD_IMG:-quay.io/coreos/etcd:v3.2.7}"
 ETCD_LOCATION="${ETCD_LOCATION:-etcd}"
 FLANNEL_NET="${FLANNEL_NET:-10.10.0.0/16}"
 TAG=`git describe --tags --dirty`
 FLANNEL_DOCKER_IMAGE="${FLANNEL_DOCKER_IMAGE:-quay.io/coreos/flannel:$TAG}"
 K8S_VERSION="${K8S_VERSION:-1.7.6}"
+HYPERKUBE_IMG="gcr.io/google_containers/hyperkube-${ARCH}"
 
 docker_ip=$(ip -o -f inet addr show docker0 | grep -Po 'inet \K[\d.]+')
 etcd_endpt="http://$docker_ip:2379"
 k8s_endpt="http://$docker_ip:8080"
 
+# Set the proper imagename according to architecture
+if [[ ${ARCH} == "ppc64le" ]]; then
+    ETCD_IMG+="-ppc64le"
+elif [[ ${ARCH} == "arm64" ]]; then
+    ETCD_IMG+="-arm64"
+fi
+
 setup_suite() {
     # Run etcd, killing any existing one that was running
 
@@ -20,12 +30,12 @@ setup_suite() {
 
     # Start a kubernetes API server
     docker rm -f flannel-e2e-k8s-apiserver >/dev/null 2>/dev/null
-    docker run -d --net=host --name flannel-e2e-k8s-apiserver gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION \
+    docker run -d --net=host --name flannel-e2e-k8s-apiserver ${HYPERKUBE_IMG}:v$K8S_VERSION \
       /hyperkube apiserver --etcd-servers=$etcd_endpt \
       --service-cluster-ip-range=10.101.0.0/16 --insecure-bind-address=0.0.0.0 --allow-privileged >/dev/null
     sleep 1
 
-    while ! cat <<EOF |  docker run -i --rm --net=host gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION /hyperkube kubectl create -f - >/dev/null 2>/dev/null
+    while ! cat <<EOF |  docker run -i --rm --net=host ${HYPERKUBE_IMG}:v$K8S_VERSION /hyperkube kubectl create -f - >/dev/null 2>/dev/null
 apiVersion: v1
 kind: Node
 metadata:
@@ -39,7 +49,7 @@ do
     sleep 1
 done
 
-cat <<EOF |  docker run -i --rm --net=host gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION /hyperkube kubectl create -f - >/dev/null 2>/dev/null
+cat <<EOF |  docker run -i --rm --net=host ${HYPERKUBE_IMG}:v$K8S_VERSION /hyperkube kubectl create -f - >/dev/null 2>/dev/null
 apiVersion: v1
 kind: Node
 metadata:
@@ -117,5 +127,5 @@ pings() {
 
 test_manifest() {
     # This just tests that the API server accepts the manifest, not that it actually acts on it correctly.
-    assert "cat ../Documentation/kube-flannel.yml |  docker run -i --rm --net=host gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION /hyperkube kubectl create -f -"
+    assert "cat ../Documentation/kube-flannel.yml |  docker run -i --rm --net=host ${HYPERKUBE_IMG}:v$K8S_VERSION /hyperkube kubectl create -f -"
 }

dist/functional-test.sh

@@ -1,10 +1,23 @@
 #!/bin/bash
+
+ARCH="${ARCH:-amd64}"
 ETCD_IMG="${ETCD_IMG:-quay.io/coreos/etcd:v3.2.7}"
+# etcd might take a bit to come up - use a known etcd version so we know we have etcdctl available
+ETCDCTL_IMG="quay.io/coreos/etcd:v3.2.7"
 ETCD_LOCATION="${ETCD_LOCATION:-etcd}"
 FLANNEL_NET="${FLANNEL_NET:-10.10.0.0/16}"
 TAG=`git describe --tags --dirty`
 FLANNEL_DOCKER_IMAGE="${FLANNEL_DOCKER_IMAGE:-quay.io/coreos/flannel:$TAG}"
 
+# Set the proper imagename according to architecture
+if [[ ${ARCH} == "ppc64le" ]]; then
+    ETCD_IMG+="-ppc64le"
+    ETCDCTL_IMG+="-ppc64le"
+elif [[ ${ARCH} == "arm64" ]]; then
+    ETCD_IMG+="-arm64"
+    ETCDCTL_IMG+="-arm64"
+fi
+
 setup_suite() {
     # Run etcd, killing any existing one that was running
     docker_ip=$(ip -o -f inet addr show docker0 | grep -Po 'inet \K[\d.]+')
@@ -44,8 +57,7 @@ write_config_etcd() {
 	    flannel_conf="{ \"Network\": \"$FLANNEL_NET\", \"Backend\": { \"Type\": \"${backend}\" } }"
     fi
 
-	# etcd might take a bit to come up - use a known etcd version so we know we have etcdctl available
-	while ! docker run --rm -it quay.io/coreos/etcd:v3.2.7 etcdctl --endpoints=$etcd_endpt set /coreos.com/network/config "$flannel_conf" >/dev/null
+	while ! docker run --rm -it $ETCDCTL_IMG etcdctl --endpoints=$etcd_endpt set /coreos.com/network/config "$flannel_conf" >/dev/null
 	do
 		sleep 0.1
 	done
@@ -113,8 +125,8 @@ test_udp_perf() {
 perf() {
     # Perf test - run iperf server on flannel1 and client on flannel2
     docker rm -f flannel-e2e-test-flannel1-iperf 2>/dev/null
-    docker run -d --name flannel-e2e-test-flannel1-iperf --net=container:flannel-e2e-test-flannel1 mlabbe/iperf3
-    docker run --rm --net=container:flannel-e2e-test-flannel2 mlabbe/iperf3 -c $ping_dest1
+    docker run -d --name flannel-e2e-test-flannel1-iperf --net=container:flannel-e2e-test-flannel1 iperf3:latest
+    docker run --rm --net=container:flannel-e2e-test-flannel2 iperf3:latest -c $ping_dest1
 }
 
 test_multi() {

images/iperf3/Dockerfile

@@ -0,0 +1,11 @@
+FROM BASEIMAGE
+
+RUN apk add --update \
+    iperf3 \
+  && rm -rf /var/cache/apk/*
+
+EXPOSE 5201
+
+ENTRYPOINT ["/usr/bin/iperf3"]
+
+CMD ["--server","-p","5201"]

images/iperf3/Makefile

@@ -0,0 +1,22 @@
+IPERF_IMG ?= iperf3:latest
+
+ARCH ?= amd64
+
+TEMP_DIR := $(shell mktemp -d)
+
+ifeq ($(ARCH),amd64)
+        BASEIMAGE=alpine:3.6
+endif
+ifeq ($(ARCH),arm64)
+	BASEIMAGE=aarch64/alpine:3.6
+endif
+ifeq ($(ARCH),ppc64le)
+	BASEIMAGE=ppc64le/alpine:3.6
+endif
+
+all: container
+
+container:
+	cp ./* $(TEMP_DIR)
+	cd $(TEMP_DIR) && sed -i 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile
+	docker build --pull -t $(IPERF_IMG) -f $(TEMP_DIR)/Dockerfile $(TEMP_DIR)