test 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. #!/usr/bin/env bash
  2. #
  3. # Run all etcd tests
  4. # ./test
  5. # ./test -v
  6. #
  7. # Run tests for one package
  8. #
  9. # PKG=./wal ./test
  10. # PKG=snap ./test
  11. #
  12. # Run code coverage
  13. # COVERDIR=coverage PASSES=cov ./test
  14. set -e
  15. source ./build
  16. # build tests with vendored dependencies
  17. etcd_setup_gopath
  18. if [ -z "$PASSES" ]; then
  19. PASSES="fmt dep compile build unit"
  20. fi
  21. # Invoke ./cover for HTML output
  22. COVER=${COVER:-"-cover"}
  23. # Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
  24. IGNORE_PKGS="(cmd|vendor|etcdserverpb|rafttest|gopath.proto)"
  25. INTEGRATION_PKGS="(integration|e2e|contrib|functional-tester)"
  26. TEST_PKGS=`find . -name \*_test.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
  27. FORMATTABLE=`find . -name \*.go | while read a; do echo $(dirname $a)/"*.go"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
  28. TESTABLE_AND_FORMATTABLE=`echo "$TEST_PKGS" | egrep -v "$INTEGRATION_PKGS"`
  29. # TODO: 'client' pkg fails with gosimple from generated files
  30. # TODO: 'rafttest' is failing with unused
  31. GOSIMPLE_UNUSED_PATHS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | grep -v 'client'`
  32. if [ -z "$GOARCH" ]; then
  33. GOARCH=$(go env GOARCH);
  34. fi
  35. # user has not provided PKG override
  36. if [ -z "$PKG" ]; then
  37. TEST=$TESTABLE_AND_FORMATTABLE
  38. FMT=$FORMATTABLE
  39. # user has provided PKG override
  40. else
  41. # strip out leading dotslashes and trailing slashes from PKG=./foo/
  42. TEST=${PKG/#./}
  43. TEST=${TEST/#\//}
  44. TEST=${TEST/%\//}
  45. # only run gofmt on packages provided by user
  46. FMT="$TEST"
  47. fi
  48. # split TEST into an array and prepend REPO_PATH to each local package
  49. split=(${TEST// / })
  50. TEST=${split[@]/#/${REPO_PATH}/}
  51. # determine whether target supports race detection
  52. if [ "$GOARCH" == "amd64" ]; then
  53. RACE="--race"
  54. fi
  55. function unit_pass {
  56. echo "Running unit tests..."
  57. # only -run=Test so examples can run in integration tests
  58. go test -timeout 3m ${COVER} ${RACE} -cpu 1,2,4 -run=Test $@ ${TEST}
  59. }
  60. function integration_pass {
  61. echo "Running integration tests..."
  62. go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/integration
  63. go test -timeout 1m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/client/integration
  64. go test -timeout 10m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
  65. go test -timeout 1m -v -cpu 1,2,4 $@ ${REPO_PATH}/contrib/raftexample
  66. go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
  67. }
  68. function cov_pass {
  69. echo "Running code coverage..."
  70. # install gocovmerge before running code coverage from github.com/wadey/gocovmerge
  71. # gocovmerge merges coverage files
  72. if ! which gocovmerge >/dev/null; then
  73. echo "gocovmerge not installed"
  74. exit 255
  75. fi
  76. if [ -z "$COVERDIR" ]; then
  77. echo "COVERDIR undeclared"
  78. exit 255
  79. fi
  80. mkdir -p "$COVERDIR"
  81. # PKGS_DELIM contains all the core etcd pkgs delimited by ',' which will be profiled for code coverage.
  82. # Integration tests will generate code coverage for those pkgs
  83. PKGS_DELIM=$(echo $TEST | sed 's/ /,/g')
  84. # TODO create coverage to e2e test
  85. PKGS=`echo "$TEST_PKGS" | egrep -v "(e2e|functional-tester)"`
  86. for t in ${PKGS}; do
  87. tf=`echo $t | tr / _`
  88. # uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
  89. go test -covermode=set -coverpkg $PKGS_DELIM -timeout 15m -run=Test -v -coverprofile "$COVERDIR/${tf}.coverprofile" ${REPO_PATH}/$t
  90. done
  91. gocovmerge "$COVERDIR"/*.coverprofile >"$COVERDIR"/cover.out
  92. }
  93. function e2e_pass {
  94. echo "Running e2e tests..."
  95. go test -timeout 10m -v -cpu 1,2,4 $@ ${REPO_PATH}/e2e
  96. }
  97. function integration_e2e_pass {
  98. echo "Running integration and e2e tests..."
  99. go test -timeout 10m -v -cpu 1,2,4 $@ ${REPO_PATH}/e2e &
  100. e2epid="$!"
  101. go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/integration &
  102. intpid="$!"
  103. wait $e2epid
  104. wait $intpid
  105. go test -timeout 1m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/client/integration
  106. go test -timeout 10m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
  107. go test -timeout 1m -v -cpu 1,2,4 $@ ${REPO_PATH}/contrib/raftexample
  108. go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
  109. }
  110. function grpcproxy_pass {
  111. go test -timeout 15m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 $@ ${REPO_PATH}/integration
  112. }
  113. function release_pass {
  114. rm -f ./bin/etcd-last-release
  115. # to grab latest patch release; bump this up for every minor release
  116. UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.0.*" | head -1)
  117. if [ -n "$MANUAL_VER" ]; then
  118. # in case, we need to test against different version
  119. UPGRADE_VER=$MANUAL_VER
  120. fi
  121. local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
  122. echo "Downloading $file"
  123. set +e
  124. curl --fail -L https://github.com/coreos/etcd/releases/download/$UPGRADE_VER/$file -o /tmp/$file
  125. local result=$?
  126. set -e
  127. case $result in
  128. 0) ;;
  129. 22) return 0
  130. ;;
  131. *) exit $result
  132. ;;
  133. esac
  134. tar xzvf /tmp/$file -C /tmp/ --strip-components=1
  135. mkdir -p ./bin
  136. mv /tmp/etcd ./bin/etcd-last-release
  137. }
  138. function fmt_pass {
  139. toggle_failpoints disable
  140. echo "Checking gofmt..."
  141. fmtRes=$(gofmt -l -s -d $FMT)
  142. if [ -n "${fmtRes}" ]; then
  143. echo -e "gofmt checking failed:\n${fmtRes}"
  144. exit 255
  145. fi
  146. echo "Checking govet..."
  147. vetRes=$(go vet $TEST)
  148. if [ -n "${vetRes}" ]; then
  149. echo -e "govet checking failed:\n${vetRes}"
  150. exit 255
  151. fi
  152. echo "Checking 'go tool vet -shadow'..."
  153. for path in $FMT; do
  154. if [ "${path##*.}" != "go" ]; then
  155. path="${path}/*.go"
  156. fi
  157. vetRes=$(go tool vet -shadow ${path})
  158. if [ -n "${vetRes}" ]; then
  159. echo -e "govet -shadow checking ${path} failed:\n${vetRes}"
  160. exit 255
  161. fi
  162. done
  163. if which goword >/dev/null; then
  164. echo "Checking goword..."
  165. # get all go files to process
  166. gofiles=`find $FMT -iname '*.go' 2>/dev/null`
  167. # ignore tests and protobuf files
  168. gofiles=`echo ${gofiles} | sort | uniq | sed "s/ /\n/g" | egrep -v "(\\_test.go|\\.pb\\.go)"`
  169. # only check for broken exported godocs
  170. gowordRes=`goword -use-spell=false ${gofiles} | grep godoc-export | sort`
  171. if [ ! -z "$gowordRes" ]; then
  172. echo -e "goword checking failed:\n${gowordRes}"
  173. exit 255
  174. fi
  175. else
  176. echo "Skipping goword..."
  177. fi
  178. if which gosimple >/dev/null; then
  179. echo "Checking gosimple..."
  180. for path in $GOSIMPLE_UNUSED_PATHS; do
  181. simplResult=`gosimple ${path} 2>&1 || true`
  182. if [ -n "${simplResult}" ]; then
  183. echo -e "gosimple checking ${path} failed:\n${simplResult}"
  184. exit 255
  185. fi
  186. done
  187. else
  188. echo "Skipping gosimple..."
  189. fi
  190. if which unused >/dev/null; then
  191. echo "Checking unused..."
  192. for path in $GOSIMPLE_UNUSED_PATHS; do
  193. unusedResult=`unused ${path} 2>&1 || true`
  194. if [ -n "${unusedResult}" ]; then
  195. echo -e "unused checking ${path} failed:\n${unusedResult}"
  196. exit 255
  197. fi
  198. done
  199. else
  200. echo "Skipping unused..."
  201. fi
  202. echo "Checking for license header..."
  203. licRes=$(for file in $(find . -type f -iname '*.go' ! -path './cmd/*' ! -path './gopath.proto/*'); do
  204. head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" || echo -e " ${file}"
  205. done;)
  206. if [ -n "${licRes}" ]; then
  207. echo -e "license header checking failed:\n${licRes}"
  208. exit 255
  209. fi
  210. echo "Checking commit titles..."
  211. git log --oneline `git merge-base HEAD master`...HEAD | while read l; do
  212. commitMsg=`echo "$l" | cut -f2- -d' '`
  213. if [[ "$commitMsg" == Merge* ]]; then
  214. # ignore "Merge pull" commits
  215. continue
  216. fi
  217. if [[ "$commitMsg" == Revert* ]]; then
  218. # ignore revert commits
  219. continue
  220. fi
  221. pkgPrefix=`echo "$commitMsg" | cut -f1 -d':'`
  222. spaceCommas=`echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0`
  223. commaSpaces=`echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0`
  224. if [[ `echo $commitMsg | grep -c ":..*"` == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
  225. echo "$l"...
  226. echo "Expected commit title format '<package>{\", \"<package>}: <description>'"
  227. echo "Got: $l"
  228. exit 255
  229. fi
  230. done
  231. }
  232. function dep_pass {
  233. echo "Checking package dependencies..."
  234. # don't pull in etcdserver package
  235. pushd clientv3 >/dev/null
  236. badpkg="(etcdserver|mvcc)"
  237. deps=`go list -f '{{ .Deps }}' | sed 's/ /\n/g' | egrep "${badpkg}" | egrep -v "${badpkg}/" || echo ""`
  238. popd >/dev/null
  239. if [ ! -z "$deps" ]; then
  240. echo -e "clientv3 has masked dependencies:\n${deps}"
  241. exit 255
  242. fi
  243. }
  244. function compile_pass {
  245. echo "Checking build..."
  246. go build -v ./tools/...
  247. }
  248. # fail fast on static tests
  249. function build_pass {
  250. GO_BUILD_FLAGS="-a -v" etcd_build
  251. }
  252. for pass in $PASSES; do
  253. ${pass}_pass $@
  254. done
  255. echo "Success"