GNUmakefile 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
  1. SHELL = bash
  2. GOGOVERSION?=$(shell grep github.com/gogo/protobuf go.mod | awk '{print $$2}')
  3. GOTOOLS = \
  4. github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs \
  5. github.com/hashicorp/go-bindata/go-bindata \
  6. golang.org/x/tools/cmd/cover \
  7. golang.org/x/tools/cmd/stringer \
  8. github.com/gogo/protobuf/protoc-gen-gofast@$(GOGOVERSION) \
  9. github.com/hashicorp/protoc-gen-go-binary \
  10. github.com/vektra/mockery/cmd/mockery
  11. GOTAGS ?=
  12. GOMODULES ?= ./... ./api/... ./sdk/...
  13. GOFILES ?= $(shell go list $(GOMODULES) | grep -v /vendor/)
  14. ifeq ($(origin GOTEST_PKGS_EXCLUDE), undefined)
  15. GOTEST_PKGS ?= $(GOMODULES)
  16. else
  17. GOTEST_PKGS=$(shell go list $(GOMODULES) | sed 's/github.com\/hashicorp\/consul/./' | egrep -v "^($(GOTEST_PKGS_EXCLUDE))$$")
  18. endif
  19. GOOS?=$(shell go env GOOS)
  20. GOARCH?=$(shell go env GOARCH)
  21. GOPATH=$(shell go env GOPATH)
  22. MAIN_GOPATH=$(shell go env GOPATH | cut -d: -f1)
  23. ASSETFS_PATH?=agent/bindata_assetfs.go
  24. # Get the git commit
  25. GIT_COMMIT?=$(shell git rev-parse --short HEAD)
  26. GIT_DIRTY?=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true)
  27. GIT_DESCRIBE?=$(shell git describe --tags --always --match "v*")
  28. GIT_IMPORT=github.com/hashicorp/consul/version
  29. GOLDFLAGS=-X $(GIT_IMPORT).GitCommit=$(GIT_COMMIT)$(GIT_DIRTY) -X $(GIT_IMPORT).GitDescribe=$(GIT_DESCRIBE)
  30. PROTOFILES?=$(shell find . -name '*.proto' | grep -v 'vendor/')
  31. PROTOGOFILES=$(PROTOFILES:.proto=.pb.go)
  32. PROTOGOBINFILES=$(PROTOFILES:.proto=.pb.binary.go)
  33. ifeq ($(FORCE_REBUILD),1)
  34. NOCACHE=--no-cache
  35. else
  36. NOCACHE=
  37. endif
  38. DOCKER_BUILD_QUIET?=1
  39. ifeq (${DOCKER_BUILD_QUIET},1)
  40. QUIET=-q
  41. else
  42. QUIET=
  43. endif
  44. CONSUL_DEV_IMAGE?=consul-dev
  45. GO_BUILD_TAG?=consul-build-go
  46. UI_BUILD_TAG?=consul-build-ui
  47. BUILD_CONTAINER_NAME?=consul-builder
  48. CONSUL_IMAGE_VERSION?=latest
  49. ################
  50. # CI Variables #
  51. ################
  52. CI_DEV_DOCKER_NAMESPACE?=hashicorpdev
  53. CI_DEV_DOCKER_IMAGE_NAME?=consul
  54. CI_DEV_DOCKER_WORKDIR?=bin/
  55. ################
  56. TEST_MODCACHE?=1
  57. TEST_BUILDCACHE?=1
  58. # You can only use as many CPUs as you have allocated to docker
  59. ifdef TEST_DOCKER_CPUS
  60. TEST_DOCKER_RESOURCE_CONSTRAINTS=--cpus $(TEST_DOCKER_CPUS)
  61. TEST_PARALLELIZATION=-e GOMAXPROCS=$(TEST_DOCKER_CPUS)
  62. else
  63. TEST_DOCKER_RESOURCE_CONSTRAINTS=
  64. TEST_PARALLELIZATION=
  65. endif
  66. ifeq ($(TEST_MODCACHE), 1)
  67. TEST_MODCACHE_VOL=-v $(MAIN_GOPATH)/pkg/mod:/go/pkg/mod
  68. else
  69. TEST_MODCACHE_VOL=
  70. endif
  71. ifeq ($(TEST_BUILDCACHE), 1)
  72. TEST_BUILDCACHE_VOL=-v $(shell go env GOCACHE):/root/.caches/go-build
  73. else
  74. TEST_BUILDCACHE_VOL=
  75. endif
  76. DIST_TAG?=1
  77. DIST_BUILD?=1
  78. DIST_SIGN?=1
  79. ifdef DIST_VERSION
  80. DIST_VERSION_ARG=-v "$(DIST_VERSION)"
  81. else
  82. DIST_VERSION_ARG=
  83. endif
  84. ifdef DIST_RELEASE_DATE
  85. DIST_DATE_ARG=-d "$(DIST_RELEASE_DATE)"
  86. else
  87. DIST_DATE_ARG=
  88. endif
  89. ifdef DIST_PRERELEASE
  90. DIST_REL_ARG=-r "$(DIST_PRERELEASE)"
  91. else
  92. DIST_REL_ARG=
  93. endif
  94. PUB_GIT?=1
  95. PUB_WEBSITE?=1
  96. ifeq ($(PUB_GIT),1)
  97. PUB_GIT_ARG=-g
  98. else
  99. PUB_GIT_ARG=
  100. endif
  101. ifeq ($(PUB_WEBSITE),1)
  102. PUB_WEBSITE_ARG=-w
  103. else
  104. PUB_WEBSITE_ARG=
  105. endif
  106. export GO_BUILD_TAG
  107. export UI_BUILD_TAG
  108. export BUILD_CONTAINER_NAME
  109. export GIT_COMMIT
  110. export GIT_DIRTY
  111. export GIT_DESCRIBE
  112. export GOTAGS
  113. export GOLDFLAGS
  114. # Allow skipping docker build during integration tests in CI since we already
  115. # have a built binary
  116. ENVOY_INTEG_DEPS?=dev-docker
  117. ifdef SKIP_DOCKER_BUILD
  118. ENVOY_INTEG_DEPS=noop
  119. endif
  120. DEV_PUSH?=0
  121. ifeq ($(DEV_PUSH),1)
  122. DEV_PUSH_ARG=
  123. else
  124. DEV_PUSH_ARG=--no-push
  125. endif
  126. # all builds binaries for all targets
  127. all: bin
  128. # used to make integration dependencies conditional
  129. noop: ;
  130. bin: tools
  131. @$(SHELL) $(CURDIR)/build-support/scripts/build-local.sh
  132. # dev creates binaries for testing locally - these are put into ./bin and $GOPATH
  133. dev: changelogfmt dev-build
  134. dev-build:
  135. @$(SHELL) $(CURDIR)/build-support/scripts/build-local.sh -o $(GOOS) -a $(GOARCH)
  136. dev-docker: linux
  137. @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)"
  138. @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null
  139. @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)"
  140. @docker build $(NOCACHE) $(QUIET) -t '$(CONSUL_DEV_IMAGE)' --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) $(CURDIR)/pkg/bin/linux_amd64 -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile
  141. # In CircleCI, the linux binary will be attached from a previous step at bin/. This make target
  142. # should only run in CI and not locally.
  143. ci.dev-docker:
  144. @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)"
  145. @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null
  146. @echo "Building Consul Development container - $(CI_DEV_DOCKER_IMAGE_NAME)"
  147. @docker build $(NOCACHE) $(QUIET) -t '$(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):$(GIT_COMMIT)' \
  148. --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
  149. --label COMMIT_SHA=$(CIRCLE_SHA1) \
  150. --label PULL_REQUEST=$(CIRCLE_PULL_REQUEST) \
  151. --label CIRCLE_BUILD_URL=$(CIRCLE_BUILD_URL) \
  152. $(CI_DEV_DOCKER_WORKDIR) -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile
  153. @echo $(DOCKER_PASS) | docker login -u="$(DOCKER_USER)" --password-stdin
  154. @echo "Pushing dev image to: https://cloud.docker.com/u/hashicorpdev/repository/docker/hashicorpdev/consul"
  155. @docker push $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):$(GIT_COMMIT)
  156. ifeq ($(CIRCLE_BRANCH), master)
  157. @docker tag $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):$(GIT_COMMIT) $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):latest
  158. @docker push $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):latest
  159. endif
  160. changelogfmt:
  161. @echo "--> Making [GH-xxxx] references clickable..."
  162. @sed -E 's|([^\[])\[GH-([0-9]+)\]|\1[[GH-\2](https://github.com/hashicorp/consul/issues/\2)]|g' CHANGELOG.md > changelog.tmp && mv changelog.tmp CHANGELOG.md
  163. # linux builds a linux package independent of the source platform
  164. linux:
  165. @$(SHELL) $(CURDIR)/build-support/scripts/build-local.sh -o linux -a amd64
  166. # dist builds binaries for all platforms and packages them for distribution
  167. dist:
  168. @$(SHELL) $(CURDIR)/build-support/scripts/release.sh -t '$(DIST_TAG)' -b '$(DIST_BUILD)' -S '$(DIST_SIGN)' $(DIST_VERSION_ARG) $(DIST_DATE_ARG) $(DIST_REL_ARG)
  169. verify:
  170. @$(SHELL) $(CURDIR)/build-support/scripts/verify.sh
  171. publish:
  172. @$(SHELL) $(CURDIR)/build-support/scripts/publish.sh $(PUB_GIT_ARG) $(PUB_WEBSITE_ARG)
  173. dev-tree:
  174. @$(SHELL) $(CURDIR)/build-support/scripts/dev.sh $(DEV_PUSH_ARG)
  175. cov:
  176. go test $(GOMODULES) -coverprofile=coverage.out
  177. go tool cover -html=coverage.out
  178. test: other-consul dev-build vet test-install-deps test-internal
  179. test-install-deps:
  180. go test -tags '$(GOTAGS)' -i $(GOTEST_PKGS)
  181. update-vendor:
  182. @echo "--> Running go mod vendor"
  183. @go mod vendor
  184. @echo "--> Removing vendoring of our own nested modules"
  185. @rm -rf vendor/github.com/hashicorp/consul
  186. @grep -v "hashicorp/consul/" < vendor/modules.txt > vendor/modules.txt.new
  187. @mv vendor/modules.txt.new vendor/modules.txt
  188. test-internal:
  189. @echo "--> Running go test"
  190. @rm -f test.log exit-code
  191. @# Dump verbose output to test.log so we can surface test names on failure but
  192. @# hide it from travis as it exceeds their log limits and causes job to be
  193. @# terminated (over 4MB and over 10k lines in the UI). We need to output
  194. @# _something_ to stop them terminating us due to inactivity...
  195. { go test -v $(GOTEST_FLAGS) -tags '$(GOTAGS)' $(GOTEST_PKGS) 2>&1 ; echo $$? > exit-code ; } | tee test.log | egrep '^(ok|FAIL|panic:|--- FAIL|--- PASS)'
  196. @echo "Exit code: $$(cat exit-code)"
  197. @# This prints all the race report between ====== lines
  198. @awk '/^WARNING: DATA RACE/ {do_print=1; print "=================="} do_print==1 {print} /^={10,}/ {do_print=0}' test.log || true
  199. @grep -A10 'panic: ' test.log || true
  200. @# Prints all the failure output until the next non-indented line - testify
  201. @# helpers often output multiple lines for readability but useless if we can't
  202. @# see them. Un-intuitive order of matches is necessary. No || true because
  203. @# awk always returns true even if there is no match and it breaks non-bash
  204. @# shells locally.
  205. @awk '/^[^[:space:]]/ {do_print=0} /--- SKIP/ {do_print=1} do_print==1 {print}' test.log
  206. @awk '/^[^[:space:]]/ {do_print=0} /--- FAIL/ {do_print=1} do_print==1 {print}' test.log
  207. @grep '^FAIL' test.log || true
  208. @if [ "$$(cat exit-code)" == "0" ] ; then echo "PASS" ; exit 0 ; else exit 1 ; fi
  209. test-race:
  210. $(MAKE) GOTEST_FLAGS=-race
  211. # Run tests with config for CI so `make test` can still be local-dev friendly.
  212. test-ci: other-consul dev-build vet test-install-deps
  213. @ if ! GOTEST_FLAGS="-short -timeout 8m -p 3 -parallel 4" make test-internal; then \
  214. echo " ============"; \
  215. echo " Retrying 1/2"; \
  216. echo " ============"; \
  217. if ! GOTEST_FLAGS="-timeout 9m -p 1 -parallel 1" make test-internal; then \
  218. echo " ============"; \
  219. echo " Retrying 2/2"; \
  220. echo " ============"; \
  221. GOTEST_FLAGS="-timeout 9m -p 1 -parallel 1" make test-internal; \
  222. fi \
  223. fi
  224. test-flake: other-consul vet test-install-deps
  225. @$(SHELL) $(CURDIR)/build-support/scripts/test-flake.sh --pkg "$(FLAKE_PKG)" --test "$(FLAKE_TEST)" --cpus "$(FLAKE_CPUS)" --n "$(FLAKE_N)"
  226. test-docker: linux go-build-image
  227. @# -ti run in the foreground showing stdout
  228. @# --rm removes the container once its finished running
  229. @# GO_MODCACHE_VOL - args for mapping in the go module cache
  230. @# GO_BUILD_CACHE_VOL - args for mapping in the go build cache
  231. @# All the env vars are so we pass through all the relevant bits of information
  232. @# Needed for running the tests
  233. @# We map in our local linux_amd64 bin directory as thats where the linux dep
  234. @# target dropped the binary. We could build the binary in the container too
  235. @# but that might take longer as caching gets weird
  236. @# Lastly we map the source dir here to the /consul workdir
  237. @echo "Running tests within a docker container"
  238. @docker run -ti --rm \
  239. -e 'GOTEST_FLAGS=$(GOTEST_FLAGS)' \
  240. -e 'GOTEST_PKGS=$(GOTEST_PKGS)' \
  241. -e 'GOTAGS=$(GOTAGS)' \
  242. -e 'GIT_COMMIT=$(GIT_COMMIT)' \
  243. -e 'GIT_DIRTY=$(GIT_DIRTY)' \
  244. -e 'GIT_DESCRIBE=$(GIT_DESCRIBE)' \
  245. $(TEST_PARALLELIZATION) \
  246. $(TEST_DOCKER_RESOURCE_CONSTRAINTS) \
  247. $(TEST_MODCACHE_VOL) \
  248. $(TEST_BUILDCACHE_VOL) \
  249. -v $(MAIN_GOPATH)/bin/linux_amd64/:/go/bin \
  250. -v $(shell pwd):/consul \
  251. $(GO_BUILD_TAG) \
  252. make test-internal
  253. other-consul:
  254. @echo "--> Checking for other consul instances"
  255. @if ps -ef | grep 'consul agent' | grep -v grep ; then \
  256. echo "Found other running consul agents. This may affect your tests." ; \
  257. exit 1 ; \
  258. fi
  259. cover:
  260. go test $(GOFILES) --cover
  261. format:
  262. @echo "--> Running go fmt"
  263. @go fmt $(GOFILES)
  264. vet:
  265. @echo "--> Running go vet"
  266. @go vet -tags '$(GOTAGS)' $(GOFILES); if [ $$? -eq 1 ]; then \
  267. echo ""; \
  268. echo "Vet found suspicious constructs. Please check the reported constructs"; \
  269. echo "and fix them if necessary before submitting the code for review."; \
  270. exit 1; \
  271. fi
  272. # If you've run "make ui" manually then this will get called for you. This is
  273. # also run as part of the release build script when it verifies that there are no
  274. # changes to the UI assets that aren't checked in.
  275. static-assets:
  276. @go-bindata-assetfs -pkg agent -prefix pkg -o $(ASSETFS_PATH) ./pkg/web_ui/...
  277. @go fmt $(ASSETFS_PATH)
  278. # Build the static web ui and build static assets inside a Docker container
  279. ui: ui-docker static-assets-docker
  280. tools:
  281. go get -v $(GOTOOLS)
  282. version:
  283. @echo -n "Version: "
  284. @$(SHELL) $(CURDIR)/build-support/scripts/version.sh
  285. @echo -n "Version + release: "
  286. @$(SHELL) $(CURDIR)/build-support/scripts/version.sh -r
  287. @echo -n "Version + git: "
  288. @$(SHELL) $(CURDIR)/build-support/scripts/version.sh -g
  289. @echo -n "Version + release + git: "
  290. @$(SHELL) $(CURDIR)/build-support/scripts/version.sh -r -g
  291. docker-images: go-build-image ui-build-image
  292. go-build-image:
  293. @echo "Building Golang build container"
  294. @docker build $(NOCACHE) $(QUIET) --build-arg 'GOTOOLS=$(GOTOOLS)' -t $(GO_BUILD_TAG) - < build-support/docker/Build-Go.dockerfile
  295. ui-build-image:
  296. @echo "Building UI build container"
  297. @docker build $(NOCACHE) $(QUIET) -t $(UI_BUILD_TAG) - < build-support/docker/Build-UI.dockerfile
  298. static-assets-docker: go-build-image
  299. @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh static-assets
  300. consul-docker: go-build-image
  301. @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh consul
  302. ui-docker: ui-build-image
  303. @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh ui
  304. test-envoy-integ: $(ENVOY_INTEG_DEPS)
  305. @$(SHELL) $(CURDIR)/test/integration/connect/envoy/run-tests.sh
  306. proto-delete:
  307. @echo "Removing $(PROTOGOFILES)"
  308. -@rm $(PROTOGOFILES)
  309. @echo "Removing $(PROTOGOBINFILES)"
  310. -@rm $(PROTOGOBINFILES)
  311. proto-rebuild: proto-delete proto
  312. proto: $(PROTOGOFILES) $(PROTOGOBINFILES)
  313. @echo "Generated all protobuf Go files"
  314. %.pb.go %.pb.binary.go: %.proto
  315. @$(SHELL) $(CURDIR)/build-support/scripts/proto-gen.sh --grpc --import-replace "$<"
  316. .PHONY: all ci bin dev dist cov test test-ci test-internal test-install-deps cover format vet ui static-assets tools
  317. .PHONY: docker-images go-build-image ui-build-image static-assets-docker consul-docker ui-docker
  318. .PHONY: version proto proto-rebuild proto-delete test-envoy-integ