diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index aa93c8ba5..032011056 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -338,17 +338,6 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} - arch: - - [amd64, x86_64] - - [arm64v8, aarch64] - - [arm32v7, arm] - - [i386, i386] - - [s390x, s390x] - exclude: - - profile: emqx-ee - arch: [i386, i386] - - profile: emqx-ee - arch: [s390x, s390x] steps: - uses: actions/download-artifact@v2 @@ -357,22 +346,72 @@ jobs: path: . - name: unzip source code run: unzip -q source.zip - - name: build emqx docker image - env: - PROFILE: ${{ matrix.profile }} - ARCH: ${{ matrix.arch[0] }} - QEMU_ARCH: ${{ matrix.arch[1] }} - run: | - sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - cd source - sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker - cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - - - uses: actions/upload-artifact@v1 - if: startsWith(github.ref, 'refs/tags/') + - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-qemu-action@v1 with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/. + image: tonistiigi/binfmt:latest + platforms: all + - uses: docker/metadata-action@v3 + id: meta + with: + images: ${{ github.repository_owner }}/${{ matrix.profile }} + flavor: | + latest=${{ !github.event.release.prerelease }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=match,pattern=[v|e](.*),group=1 + labels: + org.opencontainers.image.otp.version=${{ matrix.otp }} + - uses: docker/login-action@v1 + if: github.event_name == 'release' + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - uses: docker/build-push-action@v2 + if: matrix.profile != 'emqx-ee' + with: + push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} + pull: true + no-cache: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine + RUN_FROM=alpine:3.12 + EMQX_NAME=${{ matrix.profile }} + file: source/deploy/docker/Dockerfile + context: source + - uses: docker/build-push-action@v2 + if: matrix.profile == 'emqx-ee' + with: + push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} + pull: true + no-cache: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine + RUN_FROM=alpine:3.12 + EMQX_NAME=${{ matrix.profile }} + file: source/deploy/docker/Dockerfile.enterprise + context: source + - uses: aws-actions/configure-aws-credentials@v1 + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Push image to aws ecr + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' + run: | + version=${GITHUB_REF##*/} + docker pull emqx/emqx:${version#v} + docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push public.ecr.aws/emqx/emqx:${version#v} delete-artifact: runs-on: ubuntu-20.04 @@ -452,15 +491,6 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - - name: push docker image to docker hub - if: github.event_name == 'release' && matrix.profile == 'emqx' - run: | - set -e -x -u - sudo make docker-prepare - cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin - sudo TARGET=emqx/${{ matrix.profile }} make docker-push - sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - name: push docker image to aws ecr if: github.event_name == 'release' run: | diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index 06fd01c4b..c5e855a73 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -36,18 +36,20 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee make clean - fi - make docker - echo "::set-output name=version::$(./pkg-vsn.sh)" - if [ -f EMQX_ENTERPRISE ]; then + make emqx-ee-docker echo "::set-output name=imgname::emqx-ee" + echo "::set-output name=version::$(./pkg-vsn.sh)" + docker save emqx/emqx-ee:$(./pkg-vsn.sh) -o emqx.tar else + make emqx-docker echo "::set-output name=imgname::emqx" + echo "::set-output name=version::$(./pkg-vsn.sh)" + docker save emqx/emqx:$(./pkg-vsn.sh) -o emqx.tar fi - uses: actions/upload-artifact@v2 with: - name: emqx-docker-image-zip - path: _packages/${{ steps.build_docker.outputs.imgname }}/${{ steps.build_docker.outputs.imgname }}-docker-${{ steps.build_docker.outputs.version }}.zip + name: emqx-docker-image + path: emqx.tar webhook: runs-on: ubuntu-latest @@ -63,15 +65,10 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: emqx-docker-image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname}} - version: ${{ needs.build.outputs.version }} - run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + run: docker load < /tmp/emqx.tar - name: docker compose up timeout-minutes: 5 env: @@ -166,15 +163,10 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: emqx-docker-image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} - run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + run: docker load < /tmp/emqx.tar - name: docker compose up timeout-minutes: 5 env: @@ -276,15 +268,10 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: emqx-docker-image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} - run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + run: docker load < /tmp/emqx.tar - name: docker compose up timeout-minutes: 5 env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 908d1fe33..9efbc0bb1 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -17,7 +17,7 @@ jobs: id: install_erlang with: otp-version: 23.2 - - name: prepare + - name: make docker run: | if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials @@ -26,12 +26,12 @@ jobs: make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV + make emqx-ee-docker else echo "TARGET=emqx/emqx" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV + make emqx-docker fi - - name: make emqx image - run: make docker - name: run emqx timeout-minutes: 5 run: | @@ -79,11 +79,11 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + make emqx-ee-docker else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + make emqx-docker fi - - name: make emqx image - run: make docker - name: install k3s env: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" diff --git a/Makefile b/Makefile index 176d7c145..4fa86eda3 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,12 @@ REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine +export EMQX_DEFAULT_RUNNER = alpine:3.12 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 +export DOCKERFILE := deploy/docker/Dockerfile ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif @@ -148,11 +151,18 @@ $1: $1-rel endef $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt)))) +## docker target is to create docker instructions +.PHONY: $(REL_PROFILES:%=%-docker) +define gen-docker-target +$1-docker: $(COMMON_DEPS) + @$(BUILD) $1 docker +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) + .PHONY: run run: $(PROFILE) quickrun .PHONY: quickrun quickrun: - ./_build/$(PROFILE)/rel/emqx/bin/emqx console - -include docker.mk + ./_build/$(PROFILE)/rel/emqx/bin/emqx console \ No newline at end of file diff --git a/build b/build index d941aa40a..46cd60bc9 100755 --- a/build +++ b/build @@ -128,6 +128,19 @@ make_zip() { (cd "${tard}" && zip -qr - emqx) > "${zipball}" } +## This function builds the default docker image based on alpine:3.14 (by default) +make_docker() { + EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" + EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" + set -x + docker build --no-cache --pull \ + --build-arg BUILD_FROM="${EMQX_BUILDER}" \ + --build-arg RUN_FROM="${EMQX_RUNNER}" \ + --build-arg EMQX_NAME="$PROFILE" \ + --tag "emqx/$PROFILE:${PKG_VSN}" \ + -f "${DOCKERFILE}" . +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -148,6 +161,10 @@ case "$ARTIFACT" in make -C "deploy/packages/${PKGERDIR}" clean EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" ;; + + docker) + make_docker + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 0206dc031..c79508ec1 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -2,9 +2,6 @@ ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine ARG RUN_FROM=alpine:3.12 FROM ${BUILD_FROM} AS builder -ARG QEMU_ARCH=x86_64 -COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/ - RUN apk add --no-cache \ git \ curl \ @@ -43,10 +40,9 @@ LABEL org.label-schema.docker.dockerfile="Dockerfile" \ org.label-schema.vcs-url="https://github.com/emqx/emqx" \ maintainer="EMQ X Team " -ARG QEMU_ARCH=x86_64 ARG EMQX_NAME=emqx -COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/ +COPY deploy/docker/docker-entrypoint.sh /usr/bin/ COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx RUN ln -s /opt/emqx/bin/* /usr/local/bin/ diff --git a/docker.mk b/docker.mk deleted file mode 100644 index 7189ecc24..000000000 --- a/docker.mk +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile -*- - -## default globals. -## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise) -## or the TARGET varialbe can be set beforehand to force a different name -TARGET ?= emqx/$(PROFILE) -QEMU_ARCH ?= x86_64 -ARCH ?= amd64 -QEMU_VERSION ?= v5.0.0-2 -OS ?= alpine -export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) - -ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee) - ARCH_LIST := amd64 arm64v8 arm32v7 - EMQX_NAME := emqx-ee -else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge) - ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x - EMQX_NAME := emqx-edge -else - ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x - EMQX_NAME := emqx -endif - -.PHONY: docker -docker: docker-build docker-tag docker-save - -.PHONY: docker-prepare -docker-prepare: - ## Prepare the machine before any code installation scripts - # @echo "PREPARE: Setting up dependencies." - # @apt update -y - # @apt install --only-upgrade docker-ce -y - - ## Update docker configuration to enable docker manifest command - @echo "PREPARE: Updating docker configuration" - @mkdir -p $$HOME/.docker - - # enable experimental to use docker manifest command - @echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json - # enable experimental - @echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json - @service docker restart - -.PHONY: docker-build -docker-build: - ## Build Docker image - @echo "DOCKER BUILD: Build Docker image." - @echo "DOCKER BUILD: build version -> $(PKG_VSN)." - @echo "DOCKER BUILD: arch - $(ARCH)." - @echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)." - @echo "DOCKER BUILD: docker repo - $(TARGET) " - @echo "DOCKER BUILD: emqx name - $(EMQX_NAME)." - - ## Prepare qemu to build images other then x86_64 on travis - @echo "PREPARE: Qemu" \ - && docker run --rm --privileged multiarch/qemu-user-static:register --reset - - @mkdir -p tmp \ - && cd tmp \ - && curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \ - && tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \ - && cd - - - @docker build --no-cache \ - --build-arg PKG_VSN=$(PKG_VSN) \ - --build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine \ - --build-arg RUN_FROM=$(ARCH)/alpine:3.12 \ - --build-arg EMQX_NAME=$(EMQX_NAME) \ - --build-arg QEMU_ARCH=$(QEMU_ARCH) \ - --tag $(TARGET):build-$(OS)-$(ARCH) \ - -f deploy/docker/Dockerfile . - -.PHONY: docker-tag -docker-tag: - @echo "DOCKER TAG: Tag Docker image." - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \ - docker tag $(TARGET):build-$(OS)-$${arch} $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \ - echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \ - if [ $${arch} = amd64 ]; then \ - docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \ - echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \ - fi; \ - fi; \ - done - -.PHONY: docker-save -docker-save: - @echo "DOCKER SAVE: Save Docker image." - - @mkdir -p _packages/$(EMQX_NAME) - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \ - docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \ - zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \ - mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \ - fi - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker save $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \ - zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \ - mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \ - fi; \ - done - -.PHONY: docker-push -docker-push: - @echo "DOCKER PUSH: Push Docker image."; - @echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN)."; - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \ - docker push $(TARGET):$(PKG_VSN); \ - docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \ - docker push $(TARGET):latest; \ - fi; - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker push $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \ - fi; \ - done - -.PHONY: docker-manifest-list -docker-manifest-list: - version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \ - latest="docker manifest create --amend $(TARGET):latest"; \ - for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \ - version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \ - latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \ - fi; \ - done; \ - eval $$version; \ - eval $$latest; - - for arch in $(ARCH_LIST); do \ - case $${arch} in \ - "amd64") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \ - fi; \ - ;; \ - "arm64v8") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \ - fi; \ - ;; \ - "arm32v7") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \ - fi; \ - ;; \ - "i386") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \ - fi; \ - ;; \ - "s390x") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \ - fi; \ - ;; \ - esac; \ - done; - - docker manifest inspect $(TARGET):$(PKG_VSN) - docker manifest push $(TARGET):$(PKG_VSN); - docker manifest inspect $(TARGET):latest - docker manifest push $(TARGET):latest; - -.PHONY: docker-clean -docker-clean: - @echo "DOCKER CLEAN: Clean Docker image." - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)); fi - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \ - fi \ - done