Merge pull request #6620 from Rory-Z/chore/fix-docker-build-error

Chore/fix docker build error
2022-01-04 17:50:44 +08:00 · 2022-01-04 17:50:44 +08:00 · 7cd80c25fa
parent 38a46e1af3 514634e541
commit 7cd80c25fa
7 changed files with 115 additions and 263 deletions
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@ -338,17 +338,6 @@ jobs:
    strategy:
      matrix:
        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        arch:
-          - [amd64, x86_64]
-          - [arm64v8, aarch64]
-          - [arm32v7, arm]
-          - [i386, i386]
-          - [s390x, s390x]
-        exclude:
-          - profile: emqx-ee
-            arch: [i386, i386]
-          - profile: emqx-ee
-            arch: [s390x, s390x]

    steps:
    - uses: actions/download-artifact@v2
@ -357,22 +346,72 @@ jobs:
        path: .
    - name: unzip source code
      run: unzip -q source.zip
-    - name: build emqx docker image
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
+    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-qemu-action@v1
      with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/.
+        image: tonistiigi/binfmt:latest
+        platforms: all
+    - uses: docker/metadata-action@v3
+      id: meta
+      with:
+        images: ${{ github.repository_owner }}/${{ matrix.profile }}
+        flavor: |
+          latest=${{ !github.event.release.prerelease }}
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=match,pattern=[v|e](.*),group=1
+        labels:
+          org.opencontainers.image.otp.version=${{ matrix.otp }}
+    - uses: docker/login-action@v1
+      if: github.event_name == 'release'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - uses: docker/build-push-action@v2
+      if: matrix.profile != 'emqx-ee'
+      with:
+        push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile
+        context: source
+    - uses: docker/build-push-action@v2
+      if: matrix.profile == 'emqx-ee'
+      with:
+        push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile.enterprise
+        context: source
+    - uses: aws-actions/configure-aws-credentials@v1
+      if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: Push image to aws ecr
+      if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
+      run: |
+        version=${GITHUB_REF##*/}
+        docker pull emqx/emqx:${version#v}
+        docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v}
+        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+        docker push public.ecr.aws/emqx/emqx:${version#v}

  delete-artifact:
    runs-on: ubuntu-20.04
@ -452,15 +491,6 @@ jobs:
             -X POST \
             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
             ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: push docker image to docker hub
-      if: github.event_name == 'release' && matrix.profile == 'emqx'
-      run: |
-        set -e -x -u
-        sudo make docker-prepare
-        cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
-        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-push
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
    - name: push docker image to aws ecr
      if: github.event_name == 'release'
      run: |
--- a/.github/workflows/run_automate_tests.yaml
+++ b/.github/workflows/run_automate_tests.yaml
@ -36,18 +36,20 @@ jobs:
          echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
          make deps-emqx-ee
          make clean
-        fi
-        make docker
-        echo "::set-output name=version::$(./pkg-vsn.sh)"
-        if [ -f EMQX_ENTERPRISE ]; then
+          make emqx-ee-docker
          echo "::set-output name=imgname::emqx-ee"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx-ee:$(./pkg-vsn.sh) -o emqx.tar
        else
+          make emqx-docker
          echo "::set-output name=imgname::emqx"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx:$(./pkg-vsn.sh) -o emqx.tar
        fi
    - uses: actions/upload-artifact@v2
      with:
-        name: emqx-docker-image-zip
-        path: _packages/${{ steps.build_docker.outputs.imgname }}/${{ steps.build_docker.outputs.imgname }}-docker-${{ steps.build_docker.outputs.version }}.zip
+        name: emqx-docker-image
+        path: emqx.tar

  webhook:
    runs-on: ubuntu-latest
@ -63,15 +65,10 @@ jobs:
    - uses: actions/checkout@v2
    - uses: actions/download-artifact@v2
      with:
-        name: emqx-docker-image-zip
+        name: emqx-docker-image
        path: /tmp
    - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname}}
-        version: ${{ needs.build.outputs.version }}
-      run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+      run: docker load < /tmp/emqx.tar
    - name: docker compose up
      timeout-minutes: 5
      env:
@ -166,15 +163,10 @@ jobs:
    - uses: actions/checkout@v2
    - uses: actions/download-artifact@v2
      with:
-        name: emqx-docker-image-zip
+        name: emqx-docker-image
        path: /tmp
    - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
-      run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+      run: docker load < /tmp/emqx.tar
    - name: docker compose up
      timeout-minutes: 5
      env:
@ -276,15 +268,10 @@ jobs:
    - uses: actions/checkout@v2
    - uses: actions/download-artifact@v2
      with:
-        name: emqx-docker-image-zip
+        name: emqx-docker-image
        path: /tmp
    - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
-      run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+      run: docker load < /tmp/emqx.tar
    - name: docker compose up
      timeout-minutes: 5
      env:
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@ -17,7 +17,7 @@ jobs:
          id: install_erlang
          with:
            otp-version: 23.2
-        - name: prepare
+        - name: make docker
          run: |
            if make emqx-ee --dry-run > /dev/null 2>&1; then
              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
@ -26,12 +26,12 @@ jobs:
              make deps-emqx-ee
              echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-ee-docker
            else
              echo "TARGET=emqx/emqx" >> $GITHUB_ENV
              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-docker
            fi
-        - name: make emqx image
-          run: make docker
        - name: run emqx
          timeout-minutes: 5
          run: |
@ -79,11 +79,11 @@ jobs:
              echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
              make deps-emqx-ee
              echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              make emqx-ee-docker
            else
              echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              make emqx-docker
            fi
-        - name: make emqx image
-          run: make docker
        - name: install k3s
          env:
            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
--- a/16
+++ b/16
@ -3,9 +3,12 @@ REBAR_VERSION = 3.14.3-emqx-8
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine
+export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DESC ?= EMQ X
 export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3
+export DOCKERFILE := deploy/docker/Dockerfile
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 endif
@ -148,11 +151,18 @@ $1: $1-rel
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))

+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
+
 .PHONY: run
 run: $(PROFILE) quickrun

 .PHONY: quickrun
 quickrun:
-	./_build/$(PROFILE)/rel/emqx/bin/emqx console
-
-include docker.mk
+	./_build/$(PROFILE)/rel/emqx/bin/emqx console
--- a/17
+++ b/17
@ -128,6 +128,19 @@ make_zip() {
    (cd "${tard}" && zip -qr - emqx) > "${zipball}"
 }

+## This function builds the default docker image based on alpine:3.14 (by default)
+make_docker() {
+    EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
+    EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
+    set -x
+    docker build --no-cache --pull \
+       --build-arg BUILD_FROM="${EMQX_BUILDER}" \
+       --build-arg RUN_FROM="${EMQX_RUNNER}" \
+       --build-arg EMQX_NAME="$PROFILE" \
+       --tag "emqx/$PROFILE:${PKG_VSN}" \
+       -f "${DOCKERFILE}" .
+}
+
 log "building artifact=$ARTIFACT for profile=$PROFILE"

 case "$ARTIFACT" in
@ -148,6 +161,10 @@ case "$ARTIFACT" in
        make -C "deploy/packages/${PKGERDIR}" clean
        EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
        ;;
+
+    docker)
+        make_docker
+        ;;
    *)
        log "Unknown artifact $ARTIFACT"
        exit 1
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@ -2,9 +2,6 @@ ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
 ARG RUN_FROM=alpine:3.12
 FROM ${BUILD_FROM} AS builder

-ARG QEMU_ARCH=x86_64
-COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
-
 RUN apk add --no-cache \
    git \
    curl \
@ -43,10 +40,9 @@ LABEL org.label-schema.docker.dockerfile="Dockerfile" \
    org.label-schema.vcs-url="https://github.com/emqx/emqx" \
    maintainer="EMQ X Team <support@emqx.io>"

-ARG QEMU_ARCH=x86_64
 ARG EMQX_NAME=emqx

-COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
+COPY deploy/docker/docker-entrypoint.sh /usr/bin/
 COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx

 RUN ln -s /opt/emqx/bin/* /usr/local/bin/
--- a/docker.mk
+++ b/docker.mk
@ -1,188 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-## default globals.
-## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise)
-## or the TARGET varialbe can be set beforehand to force a different name
-TARGET ?= emqx/$(PROFILE)
-QEMU_ARCH ?= x86_64
-ARCH ?= amd64
-QEMU_VERSION ?= v5.0.0-2
-OS ?= alpine
-export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-
-ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee)
-	ARCH_LIST := amd64 arm64v8 arm32v7
-	EMQX_NAME := emqx-ee
-else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge)
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx-edge
-else
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx
-endif
-
-.PHONY: docker
-docker: docker-build docker-tag docker-save
-
-.PHONY: docker-prepare
-docker-prepare:
-	## Prepare the machine before any code installation scripts
-	# @echo "PREPARE: Setting up dependencies."
-	# @apt update -y
-	# @apt install --only-upgrade docker-ce -y
-
-	## Update docker configuration to enable docker manifest command
-	@echo "PREPARE: Updating docker configuration"
-	@mkdir -p $$HOME/.docker
-
-	# enable experimental to use docker manifest command
-	@echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json
-	# enable experimental
-	@echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json
-	@service docker restart
-
-.PHONY: docker-build
-docker-build:
-	## Build Docker image
-	@echo "DOCKER BUILD: Build Docker image."
-	@echo "DOCKER BUILD: build version -> $(PKG_VSN)."
-	@echo "DOCKER BUILD: arch - $(ARCH)."
-	@echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)."
-	@echo "DOCKER BUILD: docker repo - $(TARGET) "
-	@echo "DOCKER BUILD: emqx name - $(EMQX_NAME)."
-
-	## Prepare qemu to build images other then x86_64 on travis
-	@echo "PREPARE: Qemu" \
-	&& docker run --rm --privileged multiarch/qemu-user-static:register --reset
-
-	@mkdir -p tmp \
-	&& cd tmp \
-	&& curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& cd -
-
-	@docker build --no-cache \
-		--build-arg PKG_VSN=$(PKG_VSN)  \
-		--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine  \
-		--build-arg RUN_FROM=$(ARCH)/alpine:3.12 \
-		--build-arg EMQX_NAME=$(EMQX_NAME) \
-		--build-arg QEMU_ARCH=$(QEMU_ARCH) \
-		--tag $(TARGET):build-$(OS)-$(ARCH) \
-		-f deploy/docker/Dockerfile .
-
-.PHONY: docker-tag
-docker-tag:
-	@echo "DOCKER TAG: Tag Docker image."
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \
-			docker tag $(TARGET):build-$(OS)-$${arch}  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-			echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \
-			if [ $${arch} = amd64 ]; then \
-				docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \
-				echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \
-			fi; \
-		fi; \
-	done
-
-.PHONY: docker-save
-docker-save:
-	@echo "DOCKER SAVE: Save Docker image."
-
-	@mkdir -p _packages/$(EMQX_NAME)
-
-	@if [ -n  "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \
-		zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \
-		mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \
-	fi
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker save  $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \
-		fi; \
-	done
-
-.PHONY: docker-push
-docker-push:
-	@echo "DOCKER PUSH: Push Docker image.";
-	@echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN).";
-
-	@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker push $(TARGET):$(PKG_VSN); \
-		docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \
-		docker push $(TARGET):latest; \
-	fi;
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker push  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-		fi; \
-	done
-
-.PHONY: docker-manifest-list
-docker-manifest-list:
-	version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \
-	latest="docker manifest create --amend $(TARGET):latest"; \
-	for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \
-			version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-			latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-		fi; \
-	done; \
-	eval $$version; \
-	eval $$latest;
-
-	for arch in $(ARCH_LIST); do \
-		case $${arch} in \
-			"amd64") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-				fi; \
-				;; \
-			"arm64v8") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-				fi; \
-				;; \
-			"arm32v7") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-				fi; \
-				;; \
-			"i386") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-				fi; \
-				;; \
-			"s390x") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-				fi; \
-				;; \
-		esac; \
-	done;
-
-	docker manifest inspect $(TARGET):$(PKG_VSN)
-	docker manifest push $(TARGET):$(PKG_VSN);
-	docker manifest inspect $(TARGET):latest
-	docker manifest push $(TARGET):latest;
-
-.PHONY: docker-clean
-docker-clean:
-	@echo "DOCKER CLEAN: Clean Docker image."
-
-	@if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)); fi
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \
-		fi \
-	done