yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: easier build docker 

Signed-off-by: zhanghongtong <rory-z@outlook.com>
This commit is contained in:
zhanghongtong 2021-08-02 15:50:17 +08:00 committed by Rory Z
parent 3d0cb11682
commit b69059d442
6 changed files with 81 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
.github/workflows/build_packages.yaml vendored
View File

@ -216,6 +216,8 @@ jobs:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
erl_otp:
- 23.2.7.2-emqx-2
arch:
- amd64
- arm64
@ -295,7 +297,7 @@ jobs:
done
- name: build emqx packages
env:
ERL_OTP: erl23.2.7.2-emqx-2
ERL_OTP: erl${{ matrix.erl_otp }}
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
@ -336,17 +338,14 @@ jobs:
docker:
runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
arch:
- [amd64, x86_64]
- [arm64v8, aarch64]
- [arm32v7, arm]
erl_otp:
- 23.2.7.2-emqx-2
steps:
- uses: actions/download-artifact@v2
@ -355,22 +354,49 @@ jobs:
path: .
- name: unzip source code
run: unzip -q source.zip
- name: build emqx docker image
env:
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch[0] }}
QEMU_ARCH: ${{ matrix.arch[1] }}
run: |
sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
cd source
sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker
cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
- uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/')
- uses: docker/setup-buildx-action@v1
- uses: docker/setup-qemu-action@v1
with:
name: ${{ matrix.profile }}
path: source/_packages/${{ matrix.profile }}/.
image: tonistiigi/binfmt:latest
platforms: all
- name: build emqx docker image
if: github.event_name != 'release'
env:
ERL_OTP: erl${{ matrix.erl_otp }}
PROFILE: ${{ matrix.profile }}
run: |
cd source
PKG_VSN="$(./pkg-vsn.sh)"
docker buildx build --no-cache \
--platform=linux/amd64,linux/arm64 \
--build-arg PKG_VSN=$PKG_VSN \
--build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-alpine \
--build-arg RUN_FROM=alpine:3.14 \
--build-arg EMQX_NAME=$PROFILE \
--tag emqx/$PROFILE:$PKG_VSN \
-f deploy/docker/Dockerfile .
- uses: docker/login-action@v1
if: github.event_name == 'release'
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: build emqx docker image
if: github.event_name == 'release'
env:
ERL_OTP: erl${{ matrix.erl_otp }}
PROFILE: ${{ matrix.profile }}
run: |
cd source
PKG_VSN="$(./pkg-vsn.sh)"
docker buildx build --no-cache \
--platform=linux/amd64,linux/arm64 \
--build-arg PKG_VSN=$PKG_VSN \
--build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-alpine \
--build-arg RUN_FROM=alpine:3.14 \
--build-arg EMQX_NAME=$PROFILE \
--tag emqx/$PROFILE:$PKG_VSN \
-f deploy/docker/Dockerfile \
--push .
delete-artifact:
runs-on: ubuntu-20.04

8
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -24,14 +24,16 @@ jobs:
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
make deps-emqx-ee
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
else
echo "PROFILE=emqx" >> $GITHUB_ENV
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
fi
- name: make emqx image
run: make docker
run: make $PROFILE-docker
- name: run emqx
timeout-minutes: 5
run: |
@ -79,11 +81,13 @@ jobs:
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
make deps-emqx-ee
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
else
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
echo "PROFILE=emqx" >> $GITHUB_ENV
fi
- name: make emqx image
run: make docker
run: make $PROFILE-docker
- name: install k3s
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"

9
Makefile
View File

@ -153,7 +153,14 @@ run: $(PROFILE) quickrun
quickrun:
./_build/$(PROFILE)/rel/emqx/bin/emqx console
include docker.mk
## docker target is to create docker instructions
.PHONY: $(REL_PROFILES:%=%-docker)
define gen-docker-target
$1-docker: $(COMMON_DEPS)
@$(BUILD) $1 docker
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
conf-segs:
@scripts/merge-config.escript

17
build
View File

@ -120,6 +120,20 @@ make_zip() {
(cd "${tard}" && zip -qr - emqx) > "${zipball}"
}
make_docker() {
## Build Docker image
echo "DOCKER BUILD: Build Docker image."
echo "DOCKER BUILD: build version -> $PKG_VSN."
echo "DOCKER BUILD: docker repo -> emqx/$PROFILE "
docker build --no-cache \
--build-arg PKG_VSN="$PKG_VSN" \
--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine \
--build-arg EMQX_NAME="$PROFILE" \
--tag "emqx/$PROFILE:$PKG_VSN" \
-f deploy/docker/Dockerfile .
}
log "building artifact=$ARTIFACT for profile=$PROFILE"
case "$ARTIFACT" in
@ -140,6 +154,9 @@ case "$ARTIFACT" in
make -C "deploy/packages/${PKGERDIR}" clean
EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
;;
docker)
make_docker
;;
*)
log "Unknown artifact $ARTIFACT"
exit 1

10
deploy/docker/Dockerfile
View File

@ -1,10 +1,7 @@
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-amd64
ARG RUN_FROM=alpine:3.12
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine
ARG RUN_FROM=alpine:3.14
FROM ${BUILD_FROM} AS builder
ARG QEMU_ARCH=x86_64
COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
RUN apk add --no-cache \
git \
curl \
@ -43,10 +40,9 @@ LABEL org.label-schema.docker.dockerfile="Dockerfile" \
org.label-schema.vcs-url="https://github.com/emqx/emqx" \
maintainer="Raymond M Mouthaan <raymondmmouthaan@gmail.com>, Huang Rui <vowstar@gmail.com>, EMQ X Team <support@emqx.io>"
ARG QEMU_ARCH=x86_64
ARG EMQX_NAME=emqx
COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
COPY deploy/docker/docker-entrypoint.sh /usr/bin/
COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx
RUN ln -s /opt/emqx/bin/* /usr/local/bin/

186
docker.mk
View File

@ -1,186 +0,0 @@
#!/usr/bin/make -f
# -*- makefile -*-
## default globals
TARGET ?= emqx/emqx
QEMU_ARCH ?= x86_64
ARCH ?= amd64
QEMU_VERSION ?= v5.0.0-2
OS ?= alpine
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee)
ARCH_LIST := amd64 arm64v8 arm32v7
EMQX_NAME := emqx-ee
else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge)
ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
EMQX_NAME := emqx-edge
else
ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
EMQX_NAME := emqx
endif
.PHONY: docker
docker: docker-build docker-tag docker-save
.PHONY: docker-prepare
docker-prepare:
## Prepare the machine before any code installation scripts
# @echo "PREPARE: Setting up dependencies."
# @apt update -y
# @apt install --only-upgrade docker-ce -y
## Update docker configuration to enable docker manifest command
@echo "PREPARE: Updating docker configuration"
@mkdir -p $$HOME/.docker
# enable experimental to use docker manifest command
@echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json
# enable experimental
@echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json
@service docker restart
.PHONY: docker-build
docker-build:
## Build Docker image
@echo "DOCKER BUILD: Build Docker image."
@echo "DOCKER BUILD: build version -> $(PKG_VSN)."
@echo "DOCKER BUILD: arch - $(ARCH)."
@echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)."
@echo "DOCKER BUILD: docker repo - $(TARGET) "
@echo "DOCKER BUILD: emqx name - $(EMQX_NAME)."
## Prepare qemu to build images other then x86_64 on travis
@echo "PREPARE: Qemu" \
&& docker run --rm --privileged multiarch/qemu-user-static:register --reset
@mkdir -p tmp \
&& cd tmp \
&& curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \
&& tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \
&& cd -
@docker build --no-cache \
--build-arg PKG_VSN=$(PKG_VSN) \
--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-$(ARCH) \
--build-arg RUN_FROM=$(ARCH)/alpine:3.12 \
--build-arg EMQX_NAME=$(EMQX_NAME) \
--build-arg QEMU_ARCH=$(QEMU_ARCH) \
--tag $(TARGET):build-$(OS)-$(ARCH) \
-f deploy/docker/Dockerfile .
.PHONY: docker-tag
docker-tag:
@echo "DOCKER TAG: Tag Docker image."
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \
docker tag $(TARGET):build-$(OS)-$${arch} $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \
if [ $${arch} = amd64 ]; then \
docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \
echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \
fi; \
fi; \
done
.PHONY: docker-save
docker-save:
@echo "DOCKER SAVE: Save Docker image."
@mkdir -p _packages/$(EMQX_NAME)
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \
zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \
mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \
fi
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker save $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \
fi; \
done
.PHONY: docker-push
docker-push:
@echo "DOCKER PUSH: Push Docker image.";
@echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN).";
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
docker push $(TARGET):$(PKG_VSN); \
docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \
docker push $(TARGET):latest; \
fi;
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker push $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
fi; \
done
.PHONY: docker-manifest-list
docker-manifest-list:
version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \
latest="docker manifest create --amend $(TARGET):latest"; \
for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \
version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
fi; \
done; \
eval $$version; \
eval $$latest;
for arch in $(ARCH_LIST); do \
case $${arch} in \
"amd64") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
fi; \
;; \
"arm64v8") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
fi; \
;; \
"arm32v7") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
fi; \
;; \
"i386") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
fi; \
;; \
"s390x") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
fi; \
;; \
esac; \
done;
docker manifest inspect $(TARGET):$(PKG_VSN)
docker manifest push $(TARGET):$(PKG_VSN);
docker manifest inspect $(TARGET):latest
docker manifest push $(TARGET):latest;
.PHONY: docker-clean
docker-clean:
@echo "DOCKER CLEAN: Clean Docker image."
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)); fi
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \
fi \
done