util.sh 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. #!/bin/bash
  2. # Copyright 2014 The Kubernetes Authors.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. kube::util::sortable_date() {
  16. date "+%Y%m%d-%H%M%S"
  17. }
  18. kube::util::wait_for_url() {
  19. local url=$1
  20. local prefix=${2:-}
  21. local wait=${3:-1}
  22. local times=${4:-30}
  23. which curl >/dev/null || {
  24. kube::log::usage "curl must be installed"
  25. exit 1
  26. }
  27. local i
  28. for i in $(seq 1 $times); do
  29. local out
  30. if out=$(curl -gfs $url 2>/dev/null); then
  31. kube::log::status "On try ${i}, ${prefix}: ${out}"
  32. return 0
  33. fi
  34. sleep ${wait}
  35. done
  36. kube::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each"
  37. return 1
  38. }
  39. # returns a random port
  40. kube::util::get_random_port() {
  41. awk -v min=1024 -v max=65535 'BEGIN{srand(); print int(min+rand()*(max-min+1))}'
  42. }
  43. # use netcat to check if the host($1):port($2) is free (return 0 means free, 1 means used)
  44. kube::util::test_host_port_free() {
  45. local host=$1
  46. local port=$2
  47. local success=0
  48. local fail=1
  49. which nc >/dev/null || {
  50. kube::log::usage "netcat isn't installed, can't verify if ${host}:${port} is free, skipping the check..."
  51. return ${success}
  52. }
  53. if [ ! $(nc -vz "${host}" "${port}") ]; then
  54. kube::log::status "${host}:${port} is free, proceeding..."
  55. return ${success}
  56. else
  57. kube::log::status "${host}:${port} is already used"
  58. return ${fail}
  59. fi
  60. }
  61. # Example: kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
  62. # See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
  63. kube::util::trap_add() {
  64. local trap_add_cmd
  65. trap_add_cmd=$1
  66. shift
  67. for trap_add_name in "$@"; do
  68. local existing_cmd
  69. local new_cmd
  70. # Grab the currently defined trap commands for this trap
  71. existing_cmd=`trap -p "${trap_add_name}" | awk -F"'" '{print $2}'`
  72. if [[ -z "${existing_cmd}" ]]; then
  73. new_cmd="${trap_add_cmd}"
  74. else
  75. new_cmd="${existing_cmd};${trap_add_cmd}"
  76. fi
  77. # Assign the test
  78. trap "${new_cmd}" "${trap_add_name}"
  79. done
  80. }
  81. # Opposite of kube::util::ensure-temp-dir()
  82. kube::util::cleanup-temp-dir() {
  83. rm -rf "${KUBE_TEMP}"
  84. }
  85. # Create a temp dir that'll be deleted at the end of this bash session.
  86. #
  87. # Vars set:
  88. # KUBE_TEMP
  89. kube::util::ensure-temp-dir() {
  90. if [[ -z ${KUBE_TEMP-} ]]; then
  91. KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX)
  92. kube::util::trap_add kube::util::cleanup-temp-dir EXIT
  93. fi
  94. }
  95. # This figures out the host platform without relying on golang. We need this as
  96. # we don't want a golang install to be a prerequisite to building yet we need
  97. # this info to figure out where the final binaries are placed.
  98. kube::util::host_platform() {
  99. local host_os
  100. local host_arch
  101. case "$(uname -s)" in
  102. Darwin)
  103. host_os=darwin
  104. ;;
  105. Linux)
  106. host_os=linux
  107. ;;
  108. *)
  109. kube::log::error "Unsupported host OS. Must be Linux or Mac OS X."
  110. exit 1
  111. ;;
  112. esac
  113. case "$(uname -m)" in
  114. x86_64*)
  115. host_arch=amd64
  116. ;;
  117. i?86_64*)
  118. host_arch=amd64
  119. ;;
  120. amd64*)
  121. host_arch=amd64
  122. ;;
  123. aarch64*)
  124. host_arch=arm64
  125. ;;
  126. arm64*)
  127. host_arch=arm64
  128. ;;
  129. arm*)
  130. host_arch=arm
  131. ;;
  132. i?86*)
  133. host_arch=x86
  134. ;;
  135. s390x*)
  136. host_arch=s390x
  137. ;;
  138. ppc64le*)
  139. host_arch=ppc64le
  140. ;;
  141. *)
  142. kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le."
  143. exit 1
  144. ;;
  145. esac
  146. echo "${host_os}/${host_arch}"
  147. }
  148. kube::util::find-binary() {
  149. local lookfor="${1}"
  150. local host_platform="$(kube::util::host_platform)"
  151. local locations=(
  152. "${KUBE_ROOT}/_output/bin/${lookfor}"
  153. "${KUBE_ROOT}/_output/dockerized/bin/${host_platform}/${lookfor}"
  154. "${KUBE_ROOT}/_output/local/bin/${host_platform}/${lookfor}"
  155. "${KUBE_ROOT}/platforms/${host_platform}/${lookfor}"
  156. )
  157. local bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
  158. echo -n "${bin}"
  159. }
  160. # Run all known doc generators (today gendocs and genman for kubectl)
  161. # $1 is the directory to put those generated documents
  162. kube::util::gen-docs() {
  163. local dest="$1"
  164. # Find binary
  165. gendocs=$(kube::util::find-binary "gendocs")
  166. genkubedocs=$(kube::util::find-binary "genkubedocs")
  167. genman=$(kube::util::find-binary "genman")
  168. genyaml=$(kube::util::find-binary "genyaml")
  169. genfeddocs=$(kube::util::find-binary "genfeddocs")
  170. mkdir -p "${dest}/docs/user-guide/kubectl/"
  171. "${gendocs}" "${dest}/docs/user-guide/kubectl/"
  172. mkdir -p "${dest}/docs/admin/"
  173. "${genkubedocs}" "${dest}/docs/admin/" "kube-apiserver"
  174. "${genkubedocs}" "${dest}/docs/admin/" "kube-controller-manager"
  175. "${genkubedocs}" "${dest}/docs/admin/" "kube-proxy"
  176. "${genkubedocs}" "${dest}/docs/admin/" "kube-scheduler"
  177. "${genkubedocs}" "${dest}/docs/admin/" "kubelet"
  178. # We don't really need federation-apiserver and federation-controller-manager
  179. # binaries to generate the docs. We just pass their names to decide which docs
  180. # to generate. The actual binary for running federation is hyperkube.
  181. "${genfeddocs}" "${dest}/docs/admin/" "federation-apiserver"
  182. "${genfeddocs}" "${dest}/docs/admin/" "federation-controller-manager"
  183. mkdir -p "${dest}/docs/man/man1/"
  184. "${genman}" "${dest}/docs/man/man1/"
  185. mkdir -p "${dest}/docs/yaml/kubectl/"
  186. "${genyaml}" "${dest}/docs/yaml/kubectl/"
  187. # create the list of generated files
  188. pushd "${dest}" > /dev/null
  189. touch .generated_docs
  190. find . -type f | cut -sd / -f 2- | LC_ALL=C sort > .generated_docs
  191. popd > /dev/null
  192. }
  193. # Puts a placeholder for every generated doc. This makes the link checker work.
  194. kube::util::set-placeholder-gen-docs() {
  195. local list_file="${KUBE_ROOT}/.generated_docs"
  196. if [ -e ${list_file} ]; then
  197. # remove all of the old docs; we don't want to check them in.
  198. while read file; do
  199. if [[ "${list_file}" != "${KUBE_ROOT}/${file}" ]]; then
  200. cp "${KUBE_ROOT}/hack/autogenerated_placeholder.txt" "${KUBE_ROOT}/${file}"
  201. fi
  202. done <"${list_file}"
  203. # The .generated_docs file lists itself, so we don't need to explicitly
  204. # delete it.
  205. fi
  206. }
  207. # Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT
  208. # must be set.
  209. kube::util::remove-gen-docs() {
  210. if [ -e "${KUBE_ROOT}/.generated_docs" ]; then
  211. # remove all of the old docs; we don't want to check them in.
  212. while read file; do
  213. rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
  214. done <"${KUBE_ROOT}/.generated_docs"
  215. # The .generated_docs file lists itself, so we don't need to explicitly
  216. # delete it.
  217. fi
  218. }
  219. # Takes a path $1 to traverse for md files to append the ga-beacon tracking
  220. # link to, if needed. If $2 is set, just print files that are missing
  221. # the link.
  222. kube::util::gen-analytics() {
  223. local path="$1"
  224. local dryrun="${2:-}"
  225. local mdfiles dir link
  226. # find has some strange inconsistencies between darwin/linux. The
  227. # path to search must end in '/' for linux, but darwin will put an extra
  228. # slash in results if there is a trailing '/'.
  229. if [[ $( uname ) == 'Linux' ]]; then
  230. dir="${path}/"
  231. else
  232. dir="${path}"
  233. fi
  234. # We don't touch files in special dirs, and the kubectl docs are
  235. # autogenerated by gendocs.
  236. # Don't descend into .directories
  237. mdfiles=($( find "${dir}" -name "*.md" -type f \
  238. -not -path '*/\.*' \
  239. -not -path "${path}/vendor/*" \
  240. -not -path "${path}/staging/*" \
  241. -not -path "${path}/third_party/*" \
  242. -not -path "${path}/_gopath/*" \
  243. -not -path "${path}/_output/*" \
  244. -not -path "${path}/docs/user-guide/kubectl/kubectl*" ))
  245. for f in "${mdfiles[@]}"; do
  246. link=$(kube::util::analytics-link "${f#${path}/}")
  247. if grep -q -F -x "${link}" "${f}"; then
  248. continue
  249. elif [[ -z "${dryrun}" ]]; then
  250. echo -e "\n\n${link}" >> "${f}"
  251. else
  252. echo "$f"
  253. fi
  254. done
  255. }
  256. # Prints analytics link to append to a file at path $1.
  257. kube::util::analytics-link() {
  258. local path="$1"
  259. echo "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/${path}?pixel)]()"
  260. }
  261. # Takes a group/version and returns the path to its location on disk, sans
  262. # "pkg". E.g.:
  263. # * default behavior: extensions/v1beta1 -> apis/extensions/v1beta1
  264. # * default behavior for only a group: experimental -> apis/experimental
  265. # * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned
  266. # * Very special handling for when both group and version are "": / -> api
  267. kube::util::group-version-to-pkg-path() {
  268. local group_version="$1"
  269. # Special cases first.
  270. # TODO(lavalamp): Simplify this by moving pkg/api/v1 and splitting pkg/api,
  271. # moving the results to pkg/apis/api.
  272. case "${group_version}" in
  273. # both group and version are "", this occurs when we generate deep copies for internal objects of the legacy v1 API.
  274. __internal)
  275. echo "api"
  276. ;;
  277. v1)
  278. echo "api/v1"
  279. ;;
  280. unversioned)
  281. echo "api/unversioned"
  282. ;;
  283. *)
  284. echo "apis/${group_version%__internal}"
  285. ;;
  286. esac
  287. }
  288. # Takes a group/version and returns the swagger-spec file name.
  289. # default behavior: extensions/v1beta1 -> extensions_v1beta1
  290. # special case for v1: v1 -> v1
  291. kube::util::gv-to-swagger-name() {
  292. local group_version="$1"
  293. case "${group_version}" in
  294. v1)
  295. echo "v1"
  296. ;;
  297. *)
  298. echo "${group_version%/*}_${group_version#*/}"
  299. ;;
  300. esac
  301. }
  302. # Fetches swagger spec from apiserver.
  303. # Assumed vars:
  304. # SWAGGER_API_PATH: Base path for swaggerapi on apiserver. Ex:
  305. # http://localhost:8080/swaggerapi.
  306. # SWAGGER_ROOT_DIR: Root dir where we want to to save the fetched spec.
  307. # VERSIONS: Array of group versions to include in swagger spec.
  308. kube::util::fetch-swagger-spec() {
  309. for ver in ${VERSIONS}; do
  310. # fetch the swagger spec for each group version.
  311. if [[ ${ver} == "v1" ]]; then
  312. SUBPATH="api"
  313. else
  314. SUBPATH="apis"
  315. fi
  316. SUBPATH="${SUBPATH}/${ver}"
  317. SWAGGER_JSON_NAME="$(kube::util::gv-to-swagger-name ${ver}).json"
  318. curl -w "\n" -fs "${SWAGGER_API_PATH}${SUBPATH}" > "${SWAGGER_ROOT_DIR}/${SWAGGER_JSON_NAME}"
  319. # fetch the swagger spec for the discovery mechanism at group level.
  320. if [[ ${ver} == "v1" ]]; then
  321. continue
  322. fi
  323. SUBPATH="apis/"${ver%/*}
  324. SWAGGER_JSON_NAME="${ver%/*}.json"
  325. curl -w "\n" -fs "${SWAGGER_API_PATH}${SUBPATH}" > "${SWAGGER_ROOT_DIR}/${SWAGGER_JSON_NAME}"
  326. done
  327. # fetch swagger specs for other discovery mechanism.
  328. curl -w "\n" -fs "${SWAGGER_API_PATH}" > "${SWAGGER_ROOT_DIR}/resourceListing.json"
  329. curl -w "\n" -fs "${SWAGGER_API_PATH}version" > "${SWAGGER_ROOT_DIR}/version.json"
  330. curl -w "\n" -fs "${SWAGGER_API_PATH}api" > "${SWAGGER_ROOT_DIR}/api.json"
  331. curl -w "\n" -fs "${SWAGGER_API_PATH}apis" > "${SWAGGER_ROOT_DIR}/apis.json"
  332. curl -w "\n" -fs "${SWAGGER_API_PATH}logs" > "${SWAGGER_ROOT_DIR}/logs.json"
  333. }
  334. # Returns the name of the upstream remote repository name for the local git
  335. # repo, e.g. "upstream" or "origin".
  336. kube::util::git_upstream_remote_name() {
  337. git remote -v | grep fetch |\
  338. grep -E 'github.com[/:]kubernetes/kubernetes|k8s.io/kubernetes' |\
  339. head -n 1 | awk '{print $1}'
  340. }
  341. # Checks whether there are any files matching pattern $2 changed between the
  342. # current branch and upstream branch named by $1.
  343. # Returns 1 (false) if there are no changes, 0 (true) if there are changes
  344. # detected.
  345. kube::util::has_changes_against_upstream_branch() {
  346. local -r git_branch=$1
  347. local -r pattern=$2
  348. local full_branch
  349. full_branch="$(kube::util::git_upstream_remote_name)/${git_branch}"
  350. echo "Checking for '${pattern}' changes against '${full_branch}'"
  351. # make sure the branch is valid, otherwise the check will pass erroneously.
  352. if ! git describe "${full_branch}" >/dev/null; then
  353. # abort!
  354. exit 1
  355. fi
  356. # notice this uses ... to find the first shared ancestor
  357. if git diff --name-only "${full_branch}...HEAD" | grep "${pattern}" > /dev/null; then
  358. return 0
  359. fi
  360. # also check for pending changes
  361. if git status --porcelain | grep "${pattern}" > /dev/null; then
  362. echo "Detected '${pattern}' uncommitted changes."
  363. return 0
  364. fi
  365. echo "No '${pattern}' changes detected."
  366. return 1
  367. }
  368. # ex: ts=2 sw=2 et filetype=sh